Skip to content

Commit 919fa6a

Browse files
etretr
committed
FEATURE-ci-v3-integration: fix native compiler and shell portability
1 parent eba2aa9 commit 919fa6a

8 files changed

Lines changed: 371 additions & 34 deletions

File tree

‎.github/workflows/v3-package-validation.yml‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -116,18 +116,30 @@ jobs:
116116
mkdir -p "$RUNNER_TEMP/package-evidence"
117117
exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1
118118
export PATH="/c/Program Files/Git/cmd:$PATH"
119-
/mingw64/bin/python -c 'import subprocess; subprocess.run(["git", "--version"], check=True)'
119+
/mingw64/bin/python - <<'GIT_PROBE'
120+
import shutil, subprocess
121+
print('selected native Git:', shutil.which('git'), flush=True)
122+
for arguments in (['--version'], ['rev-parse', '--verify', 'HEAD^{commit}']):
123+
result = subprocess.run(['git'] + arguments, capture_output=True, text=True)
124+
print('git', *arguments, 'exit=' + str(result.returncode), flush=True)
125+
print(result.stdout[-2048:], end='', flush=True)
126+
print(result.stderr[-2048:], end='', flush=True)
127+
result.check_returncode()
128+
GIT_PROBE
120129
/mingw64/bin/python -m venv "$RUNNER_TEMP/package-evidence/clients"
121130
"$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/bin/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt
122131
- name: Build selected prerequisites
123-
run: /mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites"
132+
run: |
133+
export PATH="/c/Program Files/Git/cmd:$PATH"
134+
/mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites"
124135
- name: Bootstrap
125136
run: ./bootstrap
126137
- name: Installed matrix (native Python; IOCP and WSAPoll)
127-
run: >-
128-
/mingw64/bin/python scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix"
129-
--provider-prefix "$(cygpath -m "$RUNNER_TEMP/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)"
130-
--client-python "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/bin/python.exe")" --node node --compiler clang++ --cc clang --jobs 2
138+
run: |
139+
export PATH="/c/Program Files/Git/cmd:$PATH"
140+
/mingw64/bin/python scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" \
141+
--provider-prefix "$(cygpath -m "$RUNNER_TEMP/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)" \
142+
--client-python "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/bin/python.exe")" --node node --compiler clang++ --cc clang --jobs 2
131143
- if: always()
132144
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
133145
with:
@@ -146,21 +158,30 @@ jobs:
146158
release: '14.3'
147159
mem: 8192
148160
cpu: 2
149-
prepare: pkg install -y bash autoconf automake libtool pkgconf python3 gmake curl binutils perl5 llvm git node24
161+
prepare: pkg install -y bash autoconf automake libtool pkgconf python3 gmake curl binutils perl5 llvm19 git node24
150162
run: |
151163
export RUNNER_TEMP=/tmp/v3-package-${{ github.run_id }}-${{ github.run_attempt }}
152164
mkdir -p "$RUNNER_TEMP/package-evidence"
153165
trap 'cp -R "$RUNNER_TEMP/package-evidence" ./package-evidence' EXIT
154166
if [ "$(node --version)" != v24.15.0 ]; then
155167
mkdir -p "$RUNNER_TEMP/node-source"
168+
node_logs="$RUNNER_TEMP/package-evidence/node-source"
169+
mkdir -p "$node_logs"
156170
curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz"
157171
test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9
158172
tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source"
159173
(
160174
cd "$RUNNER_TEMP/node-source/node-v24.15.0"
161-
./configure --prefix="$RUNNER_TEMP/node"
162-
gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1
163-
gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1
175+
{
176+
node_llvm_bin="$(llvm-config19 --bindir)"
177+
export CC="$node_llvm_bin/clang" CXX="$node_llvm_bin/clang++"
178+
printf 'CC=%s\nCXX=%s\n' "$CC" "$CXX"
179+
"$CC" --version
180+
"$CXX" --version
181+
} > "$node_logs/compiler.log" 2>&1
182+
./configure --prefix="$RUNNER_TEMP/node" > "$node_logs/configure.log" 2>&1
183+
gmake -j2 > "$node_logs/build.log" 2>&1
184+
gmake install >> "$node_logs/build.log" 2>&1
164185
)
165186
export PATH="$RUNNER_TEMP/node/bin:$PATH"
166187
fi
@@ -173,6 +194,11 @@ jobs:
173194
python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" \
174195
--provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" \
175196
--client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --pkg-config-dir /usr/local/share/pkgconfig --compiler clang++ --cc clang --make gmake --jobs 2
197+
- name: Recover FreeBSD evidence
198+
if: always()
199+
run: |
200+
mkdir -p package-evidence
201+
scp -O -r freebsd:/tmp/v3-package-${{ github.run_id }}-${{ github.run_attempt }}/package-evidence/. package-evidence/
176202
- if: always()
177203
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
178204
with:

‎.github/workflows/v3-release-acceptance.yml‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -180,19 +180,32 @@ jobs:
180180
node-version: 24.15.0
181181
- name: Verify native Git and download immutable selected archive
182182
run: |-
183+
exec > >(tee "$RUNNER_TEMP/git-provenance.log") 2>&1
183184
export PATH="/c/Program Files/Git/cmd:$PATH"
184-
/mingw64/bin/python -c 'import subprocess; subprocess.run(["git","--version"],check=True)'
185+
/mingw64/bin/python - <<'GIT_PROBE'
186+
import shutil, subprocess
187+
print('selected native Git:', shutil.which('git'), flush=True)
188+
for arguments in (['--version'], ['rev-parse', '--verify', 'HEAD^{commit}']):
189+
result = subprocess.run(['git'] + arguments, capture_output=True, text=True)
190+
print('git', *arguments, 'exit=' + str(result.returncode), flush=True)
191+
print(result.stdout[-2048:], end='', flush=True)
192+
print(result.stderr[-2048:], end='', flush=True)
193+
result.check_returncode()
194+
GIT_PROBE
185195
/mingw64/bin/python scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected"
186196
- name: Selected provider and clients
187197
run: |-
188198
exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1
199+
export PATH="/c/Program Files/Git/cmd:$PATH"
189200
/mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites"
190201
/mingw64/bin/python -m venv "$RUNNER_TEMP/clients"
191202
"$(cygpath -m "$RUNNER_TEMP/clients/bin/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt
192203
- name: Complete native and installed TLS off/on proof
193-
run: /mingw64/bin/python scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz
194-
--provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$(cygpath -m "$RUNNER_TEMP/clients/bin/python.exe")"
195-
--node node --output "$RUNNER_TEMP/platform" --toolchain-prefix "$(cygpath -m /mingw64)"
204+
run: |-
205+
export PATH="/c/Program Files/Git/cmd:$PATH"
206+
/mingw64/bin/python scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz \
207+
--provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$(cygpath -m "$RUNNER_TEMP/clients/bin/python.exe")" \
208+
--node node --output "$RUNNER_TEMP/platform" --toolchain-prefix "$(cygpath -m /mingw64)"
196209
- id: proof
197210
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
198211
with:
@@ -209,6 +222,7 @@ jobs:
209222
${{ runner.temp }}/prerequisites/preparation.json
210223
${{ runner.temp }}/prerequisites/*.log
211224
${{ runner.temp }}/prerequisites-ci.log
225+
${{ runner.temp }}/git-provenance.log
212226
retention-days: 30
213227
if-no-files-found: warn
214228
compression-level: 0
@@ -241,23 +255,32 @@ jobs:
241255
release: '14.3'
242256
mem: 8192
243257
cpu: 2
244-
prepare: pkg install -y bash autoconf automake libtool pkgconf python3 gmake curl binutils perl5 llvm git
258+
prepare: pkg install -y bash autoconf automake libtool pkgconf python3 gmake curl binutils perl5 llvm19 git
245259
node24
246260
run: |-
247261
set -eu
248262
export RUNNER_TEMP=/tmp/v3-release-${{ github.run_id }}-${{ github.run_attempt }}
249-
mkdir -p "$RUNNER_TEMP"
250-
trap 'mkdir -p platform-evidence; cp -R "$RUNNER_TEMP/platform" platform-evidence/ 2>/dev/null || true; cp -R "$RUNNER_TEMP/prerequisites" platform-evidence/ 2>/dev/null || true' EXIT
263+
mkdir -p "$RUNNER_TEMP/platform-evidence"
264+
trap 'cp -R "$RUNNER_TEMP/platform" "$RUNNER_TEMP/platform-evidence/" 2>/dev/null || true; cp -R "$RUNNER_TEMP/prerequisites" "$RUNNER_TEMP/platform-evidence/" 2>/dev/null || true; cp -R "$RUNNER_TEMP/platform-evidence" ./platform-evidence' EXIT
251265
if [ "$(node --version)" != v24.15.0 ]; then
252266
mkdir -p "$RUNNER_TEMP/node-source"
267+
node_logs="$RUNNER_TEMP/platform-evidence/node-source"
268+
mkdir -p "$node_logs"
253269
curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz"
254270
test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9
255271
tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source"
256272
(
257273
cd "$RUNNER_TEMP/node-source/node-v24.15.0"
258-
./configure --prefix="$RUNNER_TEMP/node"
259-
gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1
260-
gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1
274+
{
275+
node_llvm_bin="$(llvm-config19 --bindir)"
276+
export CC="$node_llvm_bin/clang" CXX="$node_llvm_bin/clang++"
277+
printf 'CC=%s\nCXX=%s\n' "$CC" "$CXX"
278+
"$CC" --version
279+
"$CXX" --version
280+
} > "$node_logs/compiler.log" 2>&1
281+
./configure --prefix="$RUNNER_TEMP/node" > "$node_logs/configure.log" 2>&1
282+
gmake -j2 > "$node_logs/build.log" 2>&1
283+
gmake install >> "$node_logs/build.log" 2>&1
261284
)
262285
export PATH="$RUNNER_TEMP/node/bin:$PATH"
263286
fi
@@ -267,6 +290,11 @@ jobs:
267290
python3 -m venv "$RUNNER_TEMP/clients"
268291
"$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt
269292
python3 scripts/run-v3-release-ci.py platform --selection build/ci-selected-input/selected.json --archive build/ci-selected-input/libhttpserver-*.tar.gz --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$RUNNER_TEMP/clients/bin/python" --node node --make gmake --pkg-config-dir /usr/local/share/pkgconfig --output "$RUNNER_TEMP/platform"
293+
- name: Recover FreeBSD evidence
294+
if: always()
295+
run: |
296+
mkdir -p platform-evidence
297+
scp -O -r freebsd:/tmp/v3-release-${{ github.run_id }}-${{ github.run_attempt }}/platform-evidence/. platform-evidence/
270298
- id: proof
271299
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
272300
with:

‎scripts/check-v3-installed-consumer.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,14 @@ def package_lane(args, source, output, mode, inventory):
4444
if mode == 'yes':
4545
env['V3_TLS_CFLAGS'] = '-I' + (args.provider_prefix / 'include').as_posix()
4646
env['V3_TLS_LIBS'] = '-L' + (args.provider_prefix / 'lib').as_posix() + ' -lssl -lcrypto'
47-
command(['bash', (source / 'configure').as_posix(), '--enable-v3-tls=' + mode, '--disable-examples',
47+
bash = 'bash'
48+
if platform.system() == 'Windows':
49+
# CreateProcess searches System32 before PATH for an unqualified name.
50+
bash = shutil.which('bash', path=env.get('PATH', ''))
51+
if not bash:
52+
raise ValueError('bash is unavailable on the selected PATH')
53+
bash = str(Path(bash).resolve())
54+
command([bash, (source / 'configure').as_posix(), '--enable-v3-tls=' + mode, '--disable-examples',
4855
'--prefix=' + prefix.as_posix()], build, env)
4956
command([args.make, '-j' + str(args.jobs)], build, env)
5057
command([args.make, 'install'], build, env)

‎scripts/lib/release_ci.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@
22
from datetime import datetime
33
import hashlib
44
import json
5+
import os
6+
import platform
57
import re
6-
from pathlib import PurePosixPath, PureWindowsPath
8+
from pathlib import Path, PurePosixPath, PureWindowsPath
9+
import shutil
710
import stat
811
import subprocess
912
import zipfile
@@ -76,11 +79,19 @@ def validate_request(root, commit, version):
7679
try:
7780
actual = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD^{commit}'], cwd=root, text=True, stderr=subprocess.PIPE).strip()
7881
except (OSError, subprocess.CalledProcessError) as error:
79-
raise ValueError('selected request requires executable Git and a checkout') from error
82+
diagnostic = error.stderr if isinstance(error, subprocess.CalledProcessError) else str(error)
83+
detail = ': ' + diagnostic[-2048:].strip() if diagnostic else ''
84+
raise ValueError('selected request requires executable Git and a checkout' + detail) from error
8085
require(actual == commit, 'selected commit differs from checkout')
8186
status = subprocess.check_output(['git', 'status', '--porcelain=v1', '--untracked-files=all'], cwd=root, text=True)
8287
require(not status, 'selected checkout has modified/untracked source inputs')
83-
subprocess.run(['bash', str(root / 'scripts/validate-version.sh'), version], cwd=root, check=True)
88+
bash = 'bash'
89+
if platform.system() == 'Windows':
90+
# CreateProcess searches System32 before PATH for an unqualified name.
91+
bash = shutil.which('bash', path=os.environ.get('PATH', ''))
92+
require(bash, 'bash is unavailable on the selected PATH')
93+
bash = str(Path(bash).resolve())
94+
subprocess.run([bash, str(root / 'scripts/validate-version.sh'), version], cwd=root, check=True)
8495

8596

8697
def verify_download(archive, metadata, run, expected, expected_digest, destination):

0 commit comments

Comments
 (0)