From 1cbcc5f84be17b893c4dc8bffbd5f4f66ff4efab Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 8 Sep 2026 11:48:18 -0600 Subject: [PATCH] release: validate before asking for external approval run input validation, tag creation, and testing before seeking external approval. The goal is to maximize the chances that the release will be successful before bringing in an approver --- .github/workflows/release.yml | 273 ++++++++++++++++++++++------------ CONTRIBUTING.md | 55 ++++--- scripts/release.sh | 7 +- 3 files changed, 209 insertions(+), 126 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b08d1e..9b5a6af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,13 +4,14 @@ # commit SHA to release, and this will: # 1. Validate the version (semver) and the SHA. # 2. Verify the SHA is reachable from origin/main. -# 3. Run the full CI gate (format, build, test) on the pinned SHA. -# 4. Create and push the annotated tag vX.Y.Z pointing at the SHA +# 3. Run the full CI gate (release config, format, build, test) on the +# exact commit that will be released. +# 4. Pack all NuGet packages and save them as a workflow artifact. +# 5. Request approval for stable releases. +# 6. Create and push the annotated tag vX.Y.Z pointing at the tested SHA # (using GITHUB_TOKEN). -# 5. Check out the tag and re-run the CI gate at the tag. -# 6. Pack all NuGet packages with the released version. -# 7. Create the GitHub Release and upload the .nupkg / .snupkg files. -# 8. Publish to NuGet.org via OIDC trusted publishing. +# 7. Create the GitHub Release and upload the tested .nupkg / .snupkg files. +# 8. Publish the tested packages to NuGet.org via OIDC trusted publishing. # # The releaser must supply an explicit commit SHA (not a branch name) so # that commits which land on main during the environment approval gate @@ -22,15 +23,18 @@ # uploads use --clobber, and dotnet nuget push uses --skip-duplicate, so # the workflow is safe to re-run. # -# Environment gating: +# Approval gating: # -# - Stable releases (e.g. v1.2.3) run in the protected `release` +# - Validation, CI, and package creation run before either GitHub +# Environment is entered, so a reviewer is only asked to approve a +# release that has already passed its preflight checks. +# - Stable releases (e.g. v1.2.3) publish through the protected `release` # GitHub Environment, which requires reviewer approval before any # tag is pushed or any artifact is published. -# - Prereleases (any version containing `-`, e.g. v1.2.3-beta.1) run -# in the `release-prerelease` Environment, which holds the same -# publish secrets but does NOT require reviewer approval. This -# keeps iteration on prereleases fast. +# - Prereleases (any version containing `-`, e.g. v1.2.3-beta.1) publish +# through the `release-prerelease` Environment, which holds the same +# publish secrets but does NOT require reviewer approval. This keeps +# iteration on prereleases fast. # # Both environments must be configured in repo settings (Settings -> # Environments) with the NuGet publish secrets (NUGET_USER). Only the @@ -49,120 +53,112 @@ on: required: true type: string -permissions: - contents: write - id-token: write # Required for NuGet OIDC trusted publishing - jobs: - release: - name: Release + validate: + name: Validate, test, and pack # we want to run ubuntu-latest but we'll pin to a specific version so workflow is reproducable runs-on: ubuntu-24.04 - # Gate stable releases behind the protected `release` GitHub - # Environment (required reviewers). Prereleases -- any semver with - # a `-` suffix, e.g. v1.2.3-beta.1 -- run in `release-prerelease`, - # which holds the same publish secrets but has no approval gate so - # iteration on prereleases stays fast. - # - # The validation step below enforces the semver shape - # vX.Y.Z(-prerelease)?, so this `contains` check is safe: stable - # versions never contain `-`, prereleases always do. - environment: ${{ contains(inputs.version, '-') && 'release-prerelease' || 'release' }} + permissions: + contents: read + outputs: + tag-exists: ${{ steps.release-state.outputs.exists }} + release-sha: ${{ steps.release-state.outputs.release-sha }} + nupkg: ${{ steps.find-artifacts.outputs.nupkg }} + snupkg: ${{ steps.find-artifacts.outputs.snupkg }} + openai_nupkg: ${{ steps.find-artifacts.outputs.openai_nupkg }} + openai_snupkg: ${{ steps.find-artifacts.outputs.openai_snupkg }} + anthropic_nupkg: ${{ steps.find-artifacts.outputs.anthropic_nupkg }} + anthropic_snupkg: ${{ steps.find-artifacts.outputs.anthropic_snupkg }} + agentframework_nupkg: ${{ steps.find-artifacts.outputs.agentframework_nupkg }} + agentframework_snupkg: ${{ steps.find-artifacts.outputs.agentframework_snupkg }} + azureopenai_nupkg: ${{ steps.find-artifacts.outputs.azureopenai_nupkg }} + azureopenai_snupkg: ${{ steps.find-artifacts.outputs.azureopenai_snupkg }} steps: - name: Validate inputs + env: + VERSION: ${{ inputs.version }} + INPUT_SHA: ${{ inputs.sha }} run: | - V="${{ inputs.version }}" - if [[ ! "$V" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]; then + if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?$ ]]; then echo "Error: version must be semver (e.g. v1.2.3 or v1.2.3-beta.1)" >&2 exit 1 fi - SHA="${{ inputs.sha }}" - if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: sha must be a full 40-character lowercase commit SHA. Got: '$SHA'" >&2 + if [[ ! "$INPUT_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "Error: sha must be a full 40-character lowercase commit SHA. Got: '$INPUT_SHA'" >&2 echo "Tip: copy the SHA from the commit page on GitHub (use the 'Copy full SHA' button)." >&2 exit 1 fi - - name: Checkout + - name: Checkout chosen commit uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ inputs.sha }} fetch-depth: 0 - name: Verify SHA is reachable from main + env: + INPUT_SHA: ${{ inputs.sha }} run: | - SHA="${{ inputs.sha }}" git fetch origin main --quiet - if ! git merge-base --is-ancestor "$SHA" origin/main; then - echo "Error: commit $SHA is not an ancestor of origin/main." >&2 + if ! git merge-base --is-ancestor "$INPUT_SHA" origin/main; then + echo "Error: commit $INPUT_SHA is not an ancestor of origin/main." >&2 echo "Releases must be cut from commits that have landed on main." >&2 exit 1 fi - echo "Commit $SHA is reachable from origin/main." + echo "Commit $INPUT_SHA is reachable from origin/main." - - name: Determine whether tag already exists - id: tag-state + - name: Resolve the exact release commit + id: release-state + env: + VERSION: ${{ inputs.version }} + INPUT_SHA: ${{ inputs.sha }} run: | - TAG="${{ inputs.version }}" git fetch --tags --quiet - if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then - echo "exists=true" >> "$GITHUB_OUTPUT" - echo "Tag '$TAG' already exists; will publish from the existing tag." - elif git ls-remote --tags origin | grep -q "refs/tags/${TAG}$"; then + if git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + RELEASE_SHA=$(git rev-list -n 1 "refs/tags/$VERSION") echo "exists=true" >> "$GITHUB_OUTPUT" - echo "Tag '$TAG' exists on origin but not locally; fetching." - git fetch origin "refs/tags/$TAG:refs/tags/$TAG" + echo "Tag '$VERSION' already exists; validating and packing commit $RELEASE_SHA." else + RELEASE_SHA="$INPUT_SHA" echo "exists=false" >> "$GITHUB_OUTPUT" - echo "Tag '$TAG' does not exist yet; will create at $SHA." + echo "Tag '$VERSION' does not exist yet; validating and packing commit $RELEASE_SHA." fi + echo "release-sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + + - name: Create local candidate tag + if: steps.release-state.outputs.exists == 'false' + env: + VERSION: ${{ inputs.version }} + RELEASE_SHA: ${{ steps.release-state.outputs.release-sha }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$VERSION" -m "Release $VERSION" "$RELEASE_SHA" + + - name: Checkout exact release commit + env: + RELEASE_SHA: ${{ steps.release-state.outputs.release-sha }} + run: git checkout --detach "$RELEASE_SHA" - name: Set up .NET 8.0 uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 with: dotnet-version: '8.0.x' - - name: Restore dependencies (pre-tag, on chosen ref) - if: steps.tag-state.outputs.exists == 'false' - run: dotnet restore - - - name: Check code formatting (pre-tag, on chosen ref) - if: steps.tag-state.outputs.exists == 'false' - run: dotnet format --verify-no-changes - - - name: Run CI (pre-tag, on chosen ref) - if: steps.tag-state.outputs.exists == 'false' - run: | - dotnet build --no-restore --configuration Release - dotnet test --no-build --configuration Release --verbosity normal - - - name: Configure git identity - if: steps.tag-state.outputs.exists == 'false' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + - name: Verify release configuration + run: ./scripts/verify-release-config.sh - - name: Create and push tag - if: steps.tag-state.outputs.exists == 'false' - run: | - TAG="${{ inputs.version }}" - SHA="${{ inputs.sha }}" - git tag -a "$TAG" -m "Release $TAG" "$SHA" - git push origin "$TAG" - - - name: Checkout tag - run: git checkout "${{ inputs.version }}" - - - name: Restore dependencies (at tag) + - name: Restore dependencies run: dotnet restore - - name: Check code formatting (at tag) + - name: Check code formatting run: dotnet format --verify-no-changes - - name: Run CI (at tag) - run: | - dotnet build --no-restore --configuration Release - dotnet test --no-build --configuration Release --verbosity normal + - name: Build + run: dotnet build --no-restore --configuration Release + + - name: Run tests + run: dotnet test --no-build --configuration Release --verbosity normal - name: Pack NuGet packages run: | @@ -218,6 +214,18 @@ jobs: AZUREOPENAI_NUPKG=$(find ./artifacts -name "Braintrust.Sdk.AzureOpenAI.${VERSION}.nupkg" | head -1) AZUREOPENAI_SNUPKG=$(find ./artifacts -name "Braintrust.Sdk.AzureOpenAI.${VERSION}.snupkg" | head -1) + for package in \ + "$NUPKG" \ + "$OPENAI_NUPKG" \ + "$ANTHROPIC_NUPKG" \ + "$AGENTFRAMEWORK_NUPKG" \ + "$AZUREOPENAI_NUPKG"; do + if [[ -z "$package" || ! -f "$package" ]]; then + echo "Error: expected NuGet package was not produced: '$package'" >&2 + exit 1 + fi + done + echo "nupkg=$NUPKG" >> $GITHUB_OUTPUT if [[ -n "$SNUPKG" ]]; then echo "snupkg=$SNUPKG" >> $GITHUB_OUTPUT @@ -261,6 +269,73 @@ jobs: echo " AzureOpenAI symbols package: $AZUREOPENAI_SNUPKG" fi + - name: Save tested release packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-packages + path: | + artifacts/*.nupkg + artifacts/*.snupkg + if-no-files-found: error + + release: + name: Approve and publish + needs: validate + # we want to run ubuntu-latest but we'll pin to a specific version so workflow is reproducable + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: write + id-token: write # Required for NuGet OIDC trusted publishing + # Gate only publishing behind the protected `release` GitHub + # Environment (required reviewers). Validation, tests, and packaging + # have already succeeded in the `validate` job. + environment: ${{ contains(inputs.version, '-') && 'release-prerelease' || 'release' }} + steps: + - name: Checkout tested commit + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ needs.validate.outputs.release-sha }} + fetch-depth: 0 + + - name: Create tag or verify existing tag + env: + VERSION: ${{ inputs.version }} + RELEASE_SHA: ${{ needs.validate.outputs.release-sha }} + TAG_EXISTED: ${{ needs.validate.outputs.tag-exists }} + run: | + git fetch --tags --quiet + if git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + TAG_SHA=$(git rev-list -n 1 "refs/tags/$VERSION") + if [[ "$TAG_SHA" != "$RELEASE_SHA" ]]; then + echo "Error: tag '$VERSION' changed after validation." >&2 + echo "Validated commit: $RELEASE_SHA" >&2 + echo "Current tag commit: $TAG_SHA" >&2 + exit 1 + fi + echo "Tag '$VERSION' already points at the validated commit; skipping creation." + else + if [[ "$TAG_EXISTED" == "true" ]]; then + echo "Error: tag '$VERSION' was deleted after validation; refusing to recreate it." >&2 + exit 1 + fi + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$VERSION" -m "Release $VERSION" "$RELEASE_SHA" + git push origin "$VERSION" + fi + + - name: Set up .NET 8.0 + uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 + with: + dotnet-version: '8.0.x' + + - name: Download tested release packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-packages + path: ./artifacts + - name: Create or update GitHub Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -278,16 +353,16 @@ jobs: # Upload artifacts if they exist, clobbering any partial uploads from a prior run. for artifact in \ - "${{ steps.find-artifacts.outputs.nupkg }}" \ - "${{ steps.find-artifacts.outputs.snupkg }}" \ - "${{ steps.find-artifacts.outputs.openai_nupkg }}" \ - "${{ steps.find-artifacts.outputs.openai_snupkg }}" \ - "${{ steps.find-artifacts.outputs.anthropic_nupkg }}" \ - "${{ steps.find-artifacts.outputs.anthropic_snupkg }}" \ - "${{ steps.find-artifacts.outputs.agentframework_nupkg }}" \ - "${{ steps.find-artifacts.outputs.agentframework_snupkg }}" \ - "${{ steps.find-artifacts.outputs.azureopenai_nupkg }}" \ - "${{ steps.find-artifacts.outputs.azureopenai_snupkg }}"; do + "${{ needs.validate.outputs.nupkg }}" \ + "${{ needs.validate.outputs.snupkg }}" \ + "${{ needs.validate.outputs.openai_nupkg }}" \ + "${{ needs.validate.outputs.openai_snupkg }}" \ + "${{ needs.validate.outputs.anthropic_nupkg }}" \ + "${{ needs.validate.outputs.anthropic_snupkg }}" \ + "${{ needs.validate.outputs.agentframework_nupkg }}" \ + "${{ needs.validate.outputs.agentframework_snupkg }}" \ + "${{ needs.validate.outputs.azureopenai_nupkg }}" \ + "${{ needs.validate.outputs.azureopenai_snupkg }}"; do if [[ -n "$artifact" && -f "$artifact" ]]; then gh release upload "$TAG" "$artifact" --clobber fi @@ -302,11 +377,11 @@ jobs: - name: Publish to NuGet.org run: | for NUPKG in \ - "${{ steps.find-artifacts.outputs.nupkg }}" \ - "${{ steps.find-artifacts.outputs.openai_nupkg }}" \ - "${{ steps.find-artifacts.outputs.anthropic_nupkg }}" \ - "${{ steps.find-artifacts.outputs.agentframework_nupkg }}" \ - "${{ steps.find-artifacts.outputs.azureopenai_nupkg }}"; do + "${{ needs.validate.outputs.nupkg }}" \ + "${{ needs.validate.outputs.openai_nupkg }}" \ + "${{ needs.validate.outputs.anthropic_nupkg }}" \ + "${{ needs.validate.outputs.agentframework_nupkg }}" \ + "${{ needs.validate.outputs.azureopenai_nupkg }}"; do if [[ -z "$NUPKG" || ! -f "$NUPKG" ]]; then echo "Error: NuGet package not found: $NUPKG" exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e6625b..a99ec6d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,29 +15,33 @@ from a local checkout. - `version` — semver tag like `v1.2.3` (or `v1.2.3-beta.1` for a prerelease). - `sha` — the full 40-char commit SHA from step 1. -4. Click **Run workflow**. The job will pause on the protected +4. Click **Run workflow**. Before requesting approval, the workflow + validates the inputs, verifies the SHA is reachable from + `origin/main`, runs CI on the exact commit that will ship, and packs + all NuGet packages. +5. After those checks pass, the publish job pauses on the protected `release` environment until a required reviewer approves it. -5. After approval, the workflow validates the inputs, verifies the SHA - is reachable from `origin/main`, runs CI on the pinned SHA, creates - and pushes the annotated tag `vX.Y.Z` pointing at that SHA, checks - out the tag and re-runs CI, packs all NuGet packages, creates the - GitHub Release with the `.nupkg`/`.snupkg` files attached, and - publishes to NuGet.org via OIDC trusted publishing. +6. After approval, the workflow creates and pushes the annotated tag + `vX.Y.Z` pointing at the tested commit, creates the GitHub Release + with the tested `.nupkg`/`.snupkg` files attached, and publishes + those packages to NuGet.org via OIDC trusted publishing. ### Why the SHA is required (and not just a branch name) -The workflow pauses at the environment approval gate. During that -pause, new commits can land on `main`. If we tagged "whatever `main` -is right now" at publish time, those just-landed commits would be -silently included in the release. +The publish job pauses at the environment approval gate only after the +exact release commit has passed CI and its NuGet packages have been +built. During that pause, new commits can land on `main`. If we tagged +"whatever `main` is right now" at publish time, those just-landed +commits would be silently included in the release. Requiring the releaser to pin an explicit commit SHA at dispatch time -makes the released contents reviewable: what gets approved is exactly -what ships, regardless of how long the approval takes. +makes the released contents reviewable: what gets approved is the +tested commit and packages that will ship, regardless of how long the +approval takes. ### Approval gate and secrets -The release job runs in one of two GitHub Environments depending on +The publish job runs in one of two GitHub Environments depending on whether the version is a stable release or a prerelease: - **`release`** — used for stable versions (e.g. `v1.2.3`). @@ -50,13 +54,15 @@ whether the version is a stable release or a prerelease: secrets** but **no required reviewers**, so prerelease iteration is fast. -The workflow picks the environment dynamically from the `version` -input via `contains(inputs.version, '-')`. The input validation step -enforces the semver shape `vX.Y.Z(-prerelease)?`, so the check is -safe: stable versions never contain `-`, prereleases always do. +The workflow picks the publish environment dynamically from the +`version` input via `contains(inputs.version, '-')`. The prerequisite +validation job enforces the semver shape `vX.Y.Z(-prerelease)?`, so the +check is safe: stable versions never contain `-`, prereleases always +do. Secrets are scoped to each environment, so they are only accessible to -jobs that have entered that environment. +the publish job after it has entered that environment. The preflight +job does not have access to publish secrets. If you are cutting a stable release and forget to leave off the prerelease suffix, the workflow will silently take the ungated path. @@ -71,11 +77,12 @@ the tag already exists. On re-run, the workflow: -- Detects the existing tag and skips the tag-creation step. -- Re-runs CI at the tag. -- Re-packs all NuGet packages. -- Updates the existing GitHub Release and re-uploads assets with - `--clobber` so partial uploads from the prior run are replaced. +- Detects the existing tag, then re-runs CI and re-packs all NuGet + packages from that tag before requesting approval. +- Skips the tag-creation step after approval. +- Updates the existing GitHub Release and re-uploads the newly tested + assets with `--clobber` so partial uploads from the prior run are + replaced. - Re-pushes to NuGet.org with `--skip-duplicate`, so packages that already made it through on the previous attempt are not treated as failures. diff --git a/scripts/release.sh b/scripts/release.sh index 0be8781..a581aae 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -4,9 +4,10 @@ # # The CANONICAL release path is the gated GitHub Actions workflow: # Actions -> Release -> Run workflow -# That workflow runs in the protected `release` environment, which -# requires reviewer approval and holds the NuGet publish secrets. See -# CONTRIBUTING.md ("Releasing") for the end-to-end flow. +# Its publish job runs in the protected `release` environment, which +# requires reviewer approval and holds the NuGet publish secrets. Input +# validation, tests, and package creation finish before approval is requested. +# See CONTRIBUTING.md ("Releasing") for the end-to-end flow. # # This script is kept as a local fallback for testing tag creation # (e.g. with --skip-push) and for emergencies where the Actions UI is