Skip to content

Improve multibuild cache use in wheel builds #9976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
akx wants to merge 2 commits into from
+42 −1
Closed

Improve multibuild cache use in wheel builds #9976

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d9cb50b
wheels-dependencies.sh: warn on deps missing from bundle
akx Sep 9, 2026
6ba903e
Override builders for zlib-ng and openjpeg so they use cached files
akx Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion .github/workflows/wheels-dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,20 @@ if [[ -z "$IS_MACOS" ]]; then
fi

ARCHIVE_SDIR=pillow-depends-main
MISSING_DEPENDS_FILE=$(mktemp)

# Redeclare fetch_unpack as multibuild_fetch_unpack so we can still call it
eval "$(declare -f fetch_unpack | sed '1s/^fetch_unpack/multibuild_fetch_unpack/')"

function fetch_unpack {
local url=$1
local archive_fname=${2:-$(basename $url)}
if [[ ! -f "${ARCHIVE_SDIR}/${archive_fname}" ]]; then
echo "::warning title=Dependency missing from pillow-depends::$archive_fname is not in pillow-depends; downloading it from $url" >&2
echo " $archive_fname (from $url)" >> "$MISSING_DEPENDS_FILE"
fi
multibuild_fetch_unpack "$@"
}

VERSIONS_FILE="$PROJECTDIR/.github/dependencies.json"
_get_ver() { python3 -c "import json; print(json.load(open('$VERSIONS_FILE'))['$1'])"; }
Expand Down Expand Up @@ -144,7 +158,11 @@ function build_zlib_ng {
ORIGINAL_HOST_CONFIGURE_FLAGS=$HOST_CONFIGURE_FLAGS
unset HOST_CONFIGURE_FLAGS

build_github zlib-ng/zlib-ng $ZLIB_NG_VERSION --installnamedir=$BUILD_PREFIX/lib --zlib-compat
local out_dir=$(fetch_unpack https://github.com/zlib-ng/zlib-ng/archive/$ZLIB_NG_VERSION.tar.gz zlib-ng-$ZLIB_NG_VERSION.tar.gz)
(cd $out_dir \
&& ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS --installnamedir=$BUILD_PREFIX/lib --zlib-compat \
&& make -j4 \
&& make install)

HOST_CONFIGURE_FLAGS=$ORIGINAL_HOST_CONFIGURE_FLAGS
touch zlib-stamp
Expand Down Expand Up @@ -248,6 +266,20 @@ function build_zstd {
touch zstd-stamp
}

function build_openjpeg {
if [ -e openjpeg-stamp ]; then return; fi
build_zlib
build_libpng
build_tiff
build_lcms2
local cmake=$(get_modern_cmake)
local out_dir=$(fetch_unpack https://github.com/uclouvain/openjpeg/archive/v$OPENJPEG_VERSION.tar.gz openjpeg-$OPENJPEG_VERSION.tar.gz)
(cd $out_dir \
&& $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib $HOST_CMAKE_FLAGS . \
&& make -j4 install)
touch openjpeg-stamp
}

function build {
build_xz
if [ -z "$IS_ALPINE" ] && [ -z "$SANITIZER" ] && [ -z "$IS_MACOS" ]; then
Expand Down Expand Up @@ -398,6 +430,15 @@ fi

wrap_wheel_builder build

if [[ -f $MISSING_DEPENDS_FILE ]]; then
(
echo "# The following archives were not found in pillow-depends and were downloaded from upstream instead:"
echo '```'
cat "$MISSING_DEPENDS_FILE"
echo '```'
) >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
fi

# A safety catch for iOS. iOS can't use dynamic libraries, but clang will prefer
# to link dynamic libraries to static libraries. The only way to reliably
# prevent this is to not have dynamic libraries available in the first place.
Expand Down
Loading