From ccd7227de9c9a87a2d1177b7286044189c4d2839 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Mon, 19 Oct 2026 14:26:09 +0200 Subject: [PATCH] Fix paginated release API collection --- .github/workflows/apt-publish.yml | 2 +- .github/workflows/release-sign.yml | 4 +-- tools/test_release.py | 46 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/apt-publish.yml b/.github/workflows/apt-publish.yml index 49f0b1d7..27776418 100644 --- a/.github/workflows/apt-publish.yml +++ b/.github/workflows/apt-publish.yml @@ -62,7 +62,7 @@ jobs: PY if [[ "$BOOTSTRAP" == true ]]; then gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/deployments?environment=github-pages&per_page=100" \ - --jq '[.[][]]' > .apt-input/deployments.json + | python3 -c 'import json, sys; print(json.dumps([deployment for page in json.load(sys.stdin) for deployment in page]))' > .apt-input/deployments.json python3 -c 'import json; assert json.load(open(".apt-input/deployments.json")) == [], "bootstrap refused: prior Pages deployments exist"' fi - name: Download the published complete signed release diff --git a/.github/workflows/release-sign.yml b/.github/workflows/release-sign.yml index c3dc679d..0c3dacbc 100644 --- a/.github/workflows/release-sign.yml +++ b/.github/workflows/release-sign.yml @@ -41,7 +41,7 @@ jobs: gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID" > .release-input/run.json attempt=$(python3 -c 'import json; r=json.load(open(".release-input/run.json")); n=r["run_attempt"]; assert type(n) is int and n>0; print(n)') gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/attempts/$attempt/jobs?per_page=100" \ - --jq '[.[] | .jobs[]]' > .release-input/jobs.json + | python3 -c 'import json, sys; print(json.dumps([job for page in json.load(sys.stdin) for job in page["jobs"]]))' > .release-input/jobs.json python3 tools/release.py gates --run .release-input/run.json --jobs .release-input/jobs.json \ --environment .release-input/environment.json \ --version "$GITHUB_REF_NAME" --source "$GITHUB_SHA" --out .release-input/gates.json @@ -72,7 +72,7 @@ jobs: python3 tools/release.py stage --packages .release-input/packages --directory .release --version "$GITHUB_REF_NAME" cp .release-input/gates.json .release/gates.json gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$CI_RUN_ID/artifacts?per_page=100" \ - --jq '[.[] | .artifacts[]]' > .release-input/artifacts.json + | python3 -c 'import json, sys; print(json.dumps([artifact for page in json.load(sys.stdin) for artifact in page["artifacts"]]))' > .release-input/artifacts.json python3 tools/release.py evidence --directory .release --artifacts .release-input/artifacts.json python3 tools/release_inventory.py generate --directory .release --version "$GITHUB_REF_NAME" \ --source "$GITHUB_SHA" --builder "$builder" --inputs .release-input/build/build-inputs.json diff --git a/tools/test_release.py b/tools/test_release.py index aab3a12b..45166b81 100644 --- a/tools/test_release.py +++ b/tools/test_release.py @@ -30,6 +30,52 @@ def job_fixture(): class ReleaseTests(unittest.TestCase): + def test_workflow_api_pagination_preserves_all_pages_and_failures(self): + workflows = Path(__file__).resolve().parents[1] / '.github/workflows' + cases = [('release-sign.yml', '/jobs?', 'jobs', '.release-input/jobs.json'), + ('release-sign.yml', '/artifacts?', 'artifacts', '.release-input/artifacts.json'), + ('apt-publish.yml', '/deployments?', None, '.apt-input/deployments.json')] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name in ('.release-input', '.apt-input'): + (root / name).mkdir() + # Match gh's option contract while keeping the workflow test offline. + command = root / 'gh' + command.write_text('''#!/usr/bin/env python3 +import os, sys +if '--slurp' not in sys.argv or '--paginate' not in sys.argv: + sys.exit(2) +if '--jq' in sys.argv or '--template' in sys.argv: + sys.exit(2) +sys.stdout.write(os.environ['GH_FIXTURE_PAGES']) +sys.exit(int(os.environ.get('GH_FIXTURE_EXIT', '0'))) +''') + command.chmod(0o755) + environment = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH'], + GITHUB_REPOSITORY='example/repository', CI_RUN_ID='123', attempt='1') + for workflow, endpoint, field, output in cases: + lines = (workflows / workflow).read_text().replace('\\\n', '').splitlines() + commands = [line.strip() for line in lines + if 'gh api --paginate' in line and endpoint in line] + self.assertEqual(len(commands), 1, workflow) + pages = [[{'id': 1}], [], [{'id': 2}]] + if field: + pages = [{field: page} for page in pages] + for label, payload, exit_code, success in ( + ('multiple pages', json.dumps(pages), '0', True), + ('no pages', '[]', '0', True), + ('invalid JSON', 'invalid', '0', False), + ('API failure', json.dumps(pages), '17', False)): + with self.subTest(workflow=workflow, endpoint=endpoint, case=label): + result = subprocess.run(['bash', '-euo', 'pipefail', '-c', commands[0]], + cwd=root, env=dict(environment, GH_FIXTURE_PAGES=payload, + GH_FIXTURE_EXIT=exit_code), + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode == 0, success, result.stderr) + if success: + expected = [] if label == 'no pages' else [{'id': 1}, {'id': 2}] + self.assertEqual(json.loads((root / output).read_text()), expected) + def test_signing_environment_requires_independent_review(self): configured = {'name': 'release-signing', 'protection_rules': [ {'type': 'required_reviewers', 'prevent_self_review': True,