diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index cbde3510..def316ef 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -1,60 +1,46 @@ +#!/usr/bin/env bash +set -ex +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/geolibs.sh" +BUILD_PREFIX="${BUILD_PREFIX:-${TMPDIR:-/tmp}/geostack}" +BUILD_DIR="${BUILD_DIR:-${TMPDIR:-/tmp}/geostack-build}" +mkdir -p "$BUILD_PREFIX" "$BUILD_DIR" +OS="$(uname -s)" -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - # webp, zstd, xz, libtiff cause a conflict with building webp and libtiff - # curl from brew requires zstd, use system curl - # if php is installed, brew tries to reinstall these after installing openblas - brew remove --ignore-dependencies zstd libtiff curl php -fi +# ========================= +# OS SETUP +# ========================= -echo "::group::Get code of project: $REPO_DIR" - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - if [[ "$REPO_DIR" == "rasterio" ]]; then - git clone https://github.com/rasterio/rasterio.git - cd rasterio - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "Fiona" ]]; then - git clone https://github.com/Toblerity/Fiona.git - cd Fiona - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "pyproj" ]]; then - git clone https://github.com/pyproj4/pyproj.git - cd pyproj - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "gdal" ]]; then - git clone https://github.com/OSGeo/gdal.git - cd gdal - git checkout ${BUILD_COMMIT} - # No such file or directory for GDAL 3.6.4 - ls swig/python - fi - if [[ "$REPO_DIR" == "psutil" ]]; then - git clone https://github.com/giampaolo/psutil.git - cd psutil - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "geopands" ]]; then - git clone https://github.com/geopandas/geopandas.git - cd geopandas - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "pyogrio" ]]; then - git clone https://github.com/geopandas/pyogrio.git - cd pyogrio - # setting git safe directory is required for properly building wheels when - # git >= 2.35.3 - git config --global --add safe.directory "*" - git checkout ${BUILD_COMMIT} - fi - if [[ "$REPO_DIR" == "geopandas" ]]; then - git clone https://github.com/geopandas/geopandas.git - cd geopandas - # setting git safe directory is required for properly building wheels when - # git >= 2.35.3 - git config --global --add safe.directory "*" - git checkout ${BUILD_COMMIT} - fi -echo "::endgroup::" +setup_linux() { + export CFLAGS="-fPIC ${CFLAGS:-}" + export CXXFLAGS="-fPIC ${CXXFLAGS:-}" +} +setup_macos() { + export MACOSX_DEPLOYMENT_TARGET="11.0" +} +setup_windows() { + export CMAKE_GENERATOR="Ninja" +} + +case "$OS" in + Linux) setup_linux ;; + Darwin) setup_macos ;; + MINGW*|MSYS*|CYGWIN*) setup_windows ;; + *) echo "Unsupported OS: $OS"; exit 1 ;; +esac + + +echo "BUILD_TARGET:: $BUILD_TARGET:" +exit 1 +case "$BUILD_TARGET:" in + gdal) + build_gdal + ;; + proj) + build_proj + ;; + *) + echo "Unknown target: $BUILD_TARGET:" + exit 1 + ;; +esac diff --git a/.github/workflows/code.sh b/.github/workflows/code.sh new file mode 100755 index 00000000..59a63739 --- /dev/null +++ b/.github/workflows/code.sh @@ -0,0 +1,50 @@ +echo "::group::Get code of project: $REPO_DIR" +if [[ "$REPO_DIR" == "rasterio" ]]; then + git clone https://github.com/rasterio/rasterio.git + cd rasterio + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "Fiona" ]]; then + git clone https://github.com/Toblerity/Fiona.git + cd Fiona + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "pyproj" ]]; then + git clone https://github.com/pyproj4/pyproj.git + cd pyproj + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "gdal" ]]; then + git clone https://github.com/OSGeo/gdal.git + cd gdal + git checkout ${BUILD_COMMIT} + # No such file or directory for GDAL 3.6.4 + ls swig/python +fi +if [[ "$REPO_DIR" == "psutil" ]]; then + git clone https://github.com/giampaolo/psutil.git + cd psutil + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "geopands" ]]; then + git clone https://github.com/geopandas/geopandas.git + cd geopandas + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "pyogrio" ]]; then + git clone https://github.com/geopandas/pyogrio.git + cd pyogrio + # setting git safe directory is required for properly building wheels when + # git >= 2.35.3 + git config --global --add safe.directory "*" + git checkout ${BUILD_COMMIT} +fi +if [[ "$REPO_DIR" == "geopandas" ]]; then + git clone https://github.com/geopandas/geopandas.git + cd geopandas + # setting git safe directory is required for properly building wheels when + # git >= 2.35.3 + git config --global --add safe.directory "*" + git checkout ${BUILD_COMMIT} +fi +echo "::endgroup::" diff --git a/.github/workflows/env_vars.sh b/.github/workflows/env_vars.sh new file mode 100644 index 00000000..9dd3fa8c --- /dev/null +++ b/.github/workflows/env_vars.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +SQLITE_VERSION=3530200 +CURL_VERSION=8.16.0 +LIBPNG_VERSION=1.6.50 +ZLIB_VERSION=1.2.11 +JPEG_VERSION=9f +GEOS_VERSION=3.14.0 +JSONC_VERSION=0.18 +PROJ_VERSION=9.6.0 +PROJ_DATA_VER=1.19.0 +GDAL_VERSION=3.10.3 +NGHTTP2_VERSION=1.46.0 +EXPAT_VERSION=2.2.6 +OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ +OPENSSL_ROOT=openssl-1.1.1w +OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 +PCRE_VERSION=10.44 +EXPAT_VERSION=2.6.3 +TIFF_VERSION=4.7.0 +PCRE_VERSION=10.44 +PYOGRIO_VERSION=v0.11.1 +export GDAL_CONFIG=/usr/local/bin/gdal-config +export PACKAGE_DATA=1 +#from PROJ 9.x +export PROJ_WHEEL=true +export PROJ_NETWORK=ON diff --git a/.github/workflows/geolibs.sh b/.github/workflows/geolibs.sh new file mode 100755 index 00000000..ba783c84 --- /dev/null +++ b/.github/workflows/geolibs.sh @@ -0,0 +1,472 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# import versions +source "${SCRIPT_DIR}/env_vars.sh" + +MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") +DOWNLOADS_SDIR=downloads +ARCHIVE_SDIR=/tmp/archives + +if [ $(uname) == "Darwin" ]; then + IS_MACOS=1; IS_OSX=1; +fi + +function check_sha256sum { + local fname=$1 + if [ -z "$fname" ]; then echo "Need path"; exit 1; fi + local sha256=$2 + if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi + echo "${sha256} ${fname}" > ${fname}.sha256 + if [ -n "$IS_MACOS" ]; then + shasum -a 256 -c ${fname}.sha256 + else + sha256sum -c ${fname}.sha256 + fi + rm ${fname}.sha256 +} + +function start_spinner { + if [ -n "$MB_SPINNER_PID" ]; then + return + fi + + >&2 echo "Building libraries..." + # Start a process that runs as a keep-alive + # to avoid travis quitting if there is no output + (while true; do + sleep 60 + >&2 echo "Still building..." + done) & + MB_SPINNER_PID=$! + disown +} + +function stop_spinner { + if [ ! -n "$MB_SPINNER_PID" ]; then + return + fi + + kill $MB_SPINNER_PID + unset MB_SPINNER_PID + + >&2 echo "Building libraries finished." +} + +function any_python { + for cmd in $PYTHON_EXE python3 python; do + if [ -n "$(type -t $cmd)" ]; then + echo $cmd + return + fi + done + echo "Could not find python or python3" + exit 1 +} + +function abspath { + # Can work with any Python; need not be our installed Python. + $(any_python) -c "import os.path; print(os.path.abspath('$1'))" +} + +function relpath { + # Path of first input relative to second (or $PWD if not specified) + # Can work with any Python; need not be our installed Python. + $(any_python) -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))" +} + +function realpath { + # Can work with any Python; need not be our installed Python. + $(any_python) -c "import os; print(os.path.realpath('$1'))" +} + +function lex_ver { + # Echoes dot-separated version string padded with zeros + # Thus: + # 3.2.1 -> 003002001 + # 3 -> 003000000 + echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' +} + +function unlex_ver { + # Reverses lex_ver to produce major.minor.micro + # Thus: + # 003002001 -> 3.2.1 + # 003000000 -> 3.0.0 + echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))" +} + +function strip_ver_suffix { + echo $(unlex_ver $(lex_ver $1)) +} + +function is_function { + # Echo "true" if input argument string is a function + # Allow errors during "set -e" blocks. + (set +e; $(declare -Ff "$1" > /dev/null) && echo true) +} + +function gh_clone { + git clone https://github.com/$1 +} + +# gh-clone was renamed to gh_clone, so we have this alias for +# backwards compatibility. +alias gh-clone=gh_clone + +function set_opts { + # Set options from input options string (in $- format). + local opts=$1 + local chars="exhmBH" + for (( i=0; i<${#chars}; i++ )); do + char=${chars:$i:1} + [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char + done +} + +function suppress { + # Run a command, show output only if return code not 0. + # Takes into account state of -e option. + # Compare + # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122 + # Set -e stuff agonized over in + # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell + local tmp=$(mktemp tmp.XXXXXXXXX) || return + local errexit_set + echo "Running $@" + if [[ $- = *e* ]]; then errexit_set=true; fi + set +e + ( if [[ -n $errexit_set ]]; then set -e; fi; "$@" > "$tmp" 2>&1 ) ; ret=$? + [ "$ret" -eq 0 ] || cat "$tmp" + rm -f "$tmp" + if [[ -n $errexit_set ]]; then set -e; fi + return "$ret" +} + +function expect_return { + # Run a command, succeeding (returning 0) only if the commend returns a specified code + # Parameters + # retcode expected return code (which may be zero) + # command the command called + # + # any further arguments are passed to the called command + # + # Returns 1 if called with less than 2 arguments + (( $# < 2 )) && echo "Must have at least 2 arguments" && return 1 + local retcode=$1 + local retval + ( "${@:2}" ) || retval=$? + [[ $retcode == ${retval:-0} ]] && return 0 + return ${retval:-1} +} + +function cmd_notexit { + # wraps a command, capturing its return code and preventing it + # from exiting the shell. Handles -e / +e modes. + # Parameters + # cmd - command + # any further parameters are passed to the wrapped command + # If called without an argument, it will exit the shell with an error + local cmd=$1 + if [ -z "$cmd" ];then echo "no command"; exit 1; fi + if [[ $- = *e* ]]; then errexit_set=true; fi + set +e + ("${@:1}") ; retval=$? + [[ -n $errexit_set ]] && set -e + return $retval +} + +function rm_mkdir { + # Remove directory if present, then make directory + local path=$1 + if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi + if [ -d "$path" ]; then rm -rf $path; fi + mkdir $path +} + +function untar { + local in_fname=$1 + if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi + local extension=${in_fname##*.} + case $extension in + tar) tar -xf $in_fname ;; + gz|tgz) tar -zxf $in_fname ;; + bz2) tar -jxf $in_fname ;; + zip) unzip -qq $in_fname ;; + xz) if [ -n "$IS_MACOS" ]; then + tar -xf $in_fname + else + if [[ ! $(type -P "unxz") ]]; then + echo xz must be installed to uncompress file; exit 1 + fi + unxz -c $in_fname | tar -xf - + fi ;; + *) echo Did not recognize extension $extension; exit 1 ;; + esac +} + +fetch_unpack() { + local url="$1" + local archive="$(basename "$url")" + mkdir -p "$BUILD_DIR" + cd "$BUILD_DIR" + [[ -f "$archive" ]] || curl -L -o "$archive" "$url" + # unpack + case "$archive" in + *.tar.gz|*.tgz) + tar xzf "$archive" + ;; + *.tar.bz2) + tar xjf "$archive" + ;; + *.tar.xz) + tar xJf "$archive" + ;; + *.zip) + unzip -q "$archive" + ;; + *) + echo "Unsupported archive: $archive" + exit 1 + ;; + esac + echo "$BUILD_DIR/${archive%.tar.*}" +} + + +function build_geos { + CFLAGS="$CFLAGS -g -O2" + CXXFLAGS="$CXXFLAGS -g -O2" + if [ -e geos-stamp ]; then return; fi + local cmake=cmake + fetch_unpack http://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 + (cd geos-${GEOS_VERSION} \ + && mkdir build && cd build \ + && $cmake .. \ + -DCMAKE_INSTALL_PREFIX:PATH=$BUILD_PREFIX \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_IPO=ON \ + -DBUILD_APPS:BOOL=OFF \ + -DBUILD_TESTING:BOOL=OFF \ + && $cmake --build . -j4 \ + && $cmake --install .) + touch geos-stamp +} +function build_jsonc { + if [ -e jsonc-stamp ]; then return; fi + local cmake=cmake + fetch_unpack https://s3.amazonaws.com/json-c_releases/releases/json-c-${JSONC_VERSION}.tar.gz + (cd json-c-${JSONC_VERSION} \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET . \ + && make -j4 \ + && make install) + if [ -n "$IS_OSX" ]; then + for lib in $(ls ${BUILD_PREFIX}/lib/libjson-c.5*.dylib); do + install_name_tool -id $lib $lib + done + for lib in $(ls ${BUILD_PREFIX}/lib/libjson-c.dylib); do + install_name_tool -id $lib $lib + done + fi + touch jsonc-stamp +} +function build_proj { + CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" + CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" + if [ -e proj-stamp ]; then return; fi + local cmake=cmake + build_sqlite + fetch_unpack http://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz + (cd proj-${PROJ_VERSION} \ + && mkdir build && cd build \ + && $cmake .. \ + -DCMAKE_INSTALL_PREFIX:PATH=$BUILD_PREFIX \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_IPO=ON \ + -DBUILD_APPS:BOOL=OFF \ + -DBUILD_TESTING:BOOL=OFF \ + && $cmake --build . -j4 \ + && $cmake --install .) + touch proj-stamp +} +function build_tiff { + if [ -e tiff-stamp ]; then return; fi + build_zlib + build_jpeg + build_xz + fetch_unpack https://download.osgeo.org/libtiff/tiff-${TIFF_VERSION}.tar.gz + (cd tiff-${TIFF_VERSION} \ + && mv VERSION VERSION.txt \ + && (patch -u --force < ../patches/libtiff-rename-VERSION.patch || true) \ + && ./configure --prefix=$BUILD_PREFIX \ + && make -j4 \ + && make install) + touch tiff-stamp +} +function build_sqlite { + if [ -e sqlite-stamp ]; then return; fi + fetch_unpack https://www.sqlite.org/2026/sqlite-autoconf-${SQLITE_VERSION}.tar.gz + (cd sqlite-autoconf-${SQLITE_VERSION} \ + && ./configure --prefix=$BUILD_PREFIX \ + && make -j4 \ + && make install) + touch sqlite-stamp +} +function build_expat { + if [ -e expat-stamp ]; then return; fi + if [ -n "$IS_OSX" ]; then + : + else + fetch_unpack https://github.com/libexpat/libexpat/releases/download/R_2_2_6/expat-${EXPAT_VERSION}.tar.bz2 + (cd expat-${EXPAT_VERSION} \ + && ./configure --prefix=$BUILD_PREFIX \ + && make -j4 \ + && make install) + fi + touch expat-stamp +} +function build_nghttp2 { + if [ -e nghttp2-stamp ]; then return; fi + fetch_unpack https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz + (cd nghttp2-${NGHTTP2_VERSION} \ + && ./configure --enable-lib-only --prefix=$BUILD_PREFIX \ + && make -j4 \ + && make install) + touch nghttp2-stamp +} +function build_openssl { + if [ -e openssl-stamp ]; then return; fi + OPENSSL_ROOT=$(fetch_unpack "${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz") + echo "${OPENSSL_ROOT}" + check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} + (cd ${OPENSSL_ROOT} \ + && ./config no-ssl2 -fPIC --prefix=$BUILD_PREFIX \ + && make -j4 \ + && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) + touch openssl-stamp +} +function build_curl { + if [ -e curl-stamp ]; then return; fi + CFLAGS="$CFLAGS -g -O2" + CXXFLAGS="$CXXFLAGS -g -O2" + build_openssl + build_nghttp2 + local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl --without-libidn2" + fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz + (cd curl-${CURL_VERSION} \ + && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib:$BUILD_PREFIX/lib64 ./configure $flags \ + && make -j4 \ + && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) + touch curl-stamp +} +function build_pcre2 { + build_simple pcre2 $PCRE_VERSION https://github.com/PCRE2Project/pcre2/releases/download/pcre2-${PCRE_VERSION} +} +function build_gdal { + if [ -e gdal-stamp ]; then return; fi + build_curl + build_jpeg + build_libpng + build_jsonc + build_tiff + build_proj + build_sqlite + build_expat + build_geos + build_pcre2 + CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" + CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" + if [ -n "$IS_OSX" ]; then + GEOS_CONFIG="-DGDAL_USE_GEOS=OFF" + PCRE2_LIB="$BUILD_PREFIX/lib/libpcre2-8.dylib" + else + GEOS_CONFIG="-DGDAL_USE_GEOS=ON" + PCRE2_LIB="$BUILD_PREFIX/lib/libpcre2-8.so" + fi + local cmake=cmake + fetch_unpack http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz + (cd gdal-${GDAL_VERSION} \ + && mkdir build \ + && cd build \ + && $cmake .. \ + -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ + -DCMAKE_INCLUDE_PATH=$BUILD_PREFIX/include \ + -DCMAKE_LIBRARY_PATH=$BUILD_PREFIX/lib \ + -DCMAKE_PROGRAM_PATH=$BUILD_PREFIX/bin \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ + -DOGR_BUILD_OPTIONAL_DRIVERS=ON \ + ${GEOS_CONFIG} \ + -DGDAL_USE_TIFF=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=ON \ + -DGDAL_ENABLE_DRIVER_GIF=ON \ + -DGDAL_ENABLE_DRIVER_GRIB=ON \ + -DGDAL_ENABLE_DRIVER_JPEG=ON \ + -DGDAL_USE_ICONV=ON \ + -DGDAL_USE_JSONC=ON \ + -DGDAL_USE_JSONC_INTERNAL=OFF \ + -DGDAL_USE_ZLIB=ON \ + -DGDAL_USE_ZLIB_INTERNAL=OFF \ + -DGDAL_ENABLE_DRIVER_PNG=ON \ + -DGDAL_ENABLE_DRIVER_OGCAPI=OFF \ + -DOGR_ENABLE_DRIVER_GPKG=ON \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_JAVA_BINDINGS=OFF \ + -DBUILD_CSHARP_BINDINGS=OFF \ + -DGDAL_USE_SFCGAL=OFF \ + -DGDAL_USE_XERCESC=OFF \ + -DGDAL_USE_LIBXML2=OFF \ + -DGDAL_USE_PCRE2=ON \ + -DPCRE2_INCLUDE_DIR=$BUILD_PREFIX/include \ + -DPCRE2-8_LIBRARY=$PCRE2_LIB \ + -DGDAL_USE_POSTGRESQL=OFF \ + -DGDAL_USE_ODBC=OFF \ + && $cmake --build . -j4 \ + && $cmake --install .) + if [ -n "$IS_OSX" ]; then + : + else + strip -v --strip-unneeded ${BUILD_PREFIX}/lib/libgdal.so.* || true + strip -v --strip-unneeded ${BUILD_PREFIX}/lib64/libgdal.so.* || true + fi + touch gdal-stamp +} +function pre_build { + # Any stuff that you need to do before you start building the wheels + # Runs in the root directory of this repository. + #if [ -n "$IS_OSX" ]; then + # # Update to latest zlib for OSX build + # build_new_zlib + #fi + local cmake=$(get_modern_cmake) + suppress build_openssl + suppress build_nghttp2 + if [ -n "$IS_OSX" ]; then + rm /usr/local/lib/libpng* || true + fi + #fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz + # Remove previously installed curl. + rm -rf /usr/local/lib/libcurl* || true + suppress build_curl + suppress build_jpeg + suppress build_jsonc + suppress build_tiff + suppress build_sqlite + suppress build_proj + suppress build_expat + suppress build_geos + if [ -n "$IS_OSX" ]; then + export LDFLAGS="${LDFLAGS} -Wl,-rpath,${BUILD_PREFIX}/lib" + fi + suppress build_gdal +} diff --git a/.github/workflows/linux_x86_64.yaml b/.github/workflows/linux_x86_64.yaml index 72252250..63279191 100644 --- a/.github/workflows/linux_x86_64.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -8,7 +8,6 @@ on: description: "tmate debugging enabled" required: false default: false - pull_request: env: WHEEL_SDIR: wheelhouse @@ -43,7 +42,7 @@ jobs: - name: get code of psutil run: | echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of psutil run: | sudo apt install python3-virtualenv @@ -84,7 +83,7 @@ jobs: - name: get code of pyproj run: | echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of pyproj run: | echo $PATH @@ -123,7 +122,7 @@ jobs: - name: get code of Fiona run: | echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of Fiona run: | echo $PATH @@ -161,7 +160,7 @@ jobs: - name: get code of rasterio run: | echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of rasterio run: | echo $PATH @@ -199,7 +198,7 @@ jobs: python-version: "3.12" - name: get code of GDAL run: | - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of GDAL run: | sudo apt install python3-virtualenv @@ -238,7 +237,7 @@ jobs: python-version: "3.12" - name: get code of pyogrio run: | - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of pyogrio run: | sudo apt install python3-virtualenv @@ -260,7 +259,7 @@ jobs: os-name: ["jammy"] env: REPO_DIR: geopandas - BUILD_COMMIT: v1.1.1 + BUILD_COMMIT: v1.1.3 PYOGRIO_VERISON: WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" @@ -277,7 +276,7 @@ jobs: python-version: "3.12" - name: get code of geopandas run: | - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of geopandas run: | sudo apt install python3-virtualenv diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 09790d57..72c95178 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -2,7 +2,6 @@ name: MacOS wheels ARM64 on: workflow_dispatch: - pull_request: env: WHEEL_SDIR: wheelhouse jobs: @@ -12,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] @@ -33,7 +32,7 @@ jobs: - name: get code of psutil run: | echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh @@ -48,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] @@ -76,7 +75,7 @@ jobs: run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of pyproj run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" @@ -93,7 +92,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] @@ -121,7 +120,7 @@ jobs: run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" echo $PATH - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of Fiona run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" @@ -138,7 +137,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] @@ -165,7 +164,7 @@ jobs: - name: Get code of gdal run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of GDAL run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" @@ -181,7 +180,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] @@ -209,7 +208,7 @@ jobs: - name: get code of pyogrio run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of pyogrio run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" @@ -226,14 +225,14 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] os-name: ["osx"] env: REPO_DIR: geopandas - BUILD_COMMIT: v1.1.1 + BUILD_COMMIT: v1.1.3 WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" @@ -254,7 +253,7 @@ jobs: - name: get code of geopandas run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - bash -x .github/workflows/build.sh + bash -x .github/workflows/code.sh - name: Build Wheel of geopandas run: | export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" @@ -271,7 +270,7 @@ jobs: strategy: fail-fast: false matrix: - os: [m1] + os: [macos-latest] python-version: ["3.11", "3.12", "3.13"] platform: ["arm64"] macos-target: ["13"] diff --git a/.github/workflows/psutil.yaml b/.github/workflows/psutil.yaml new file mode 100644 index 00000000..08b5f71b --- /dev/null +++ b/.github/workflows/psutil.yaml @@ -0,0 +1,49 @@ +--- +name: psutil wheels +on: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: "tmate debugging enabled" + required: false + default: false +# pull_request: +jobs: + build_psutil: + name: Build wheels of psutil on "${{ matrix.os }}, ${{ matrix.arch }}" + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, arch: x86_64 } + - { os: ubuntu-24.04-arm, arch: aarch64 } + - { os: macos-15, arch: x86_64 } + - { os: macos-15, arch: arm64 } + - { os: windows-2025, arch: AMD64 } + - { os: windows-11-arm, arch: ARM64 } + env: + REPO_DIR: psutil + BUILD_COMMIT: release-7.0.0 + steps: + - uses: actions/checkout@v6 + - name: get code of psutil + run: | + echo $PATH + bash -x .github/workflows/code.sh + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + - name: Build Wheel of psutil + uses: pypa/cibuildwheel@v4.0.0 + env: + CIBW_ARCHS: "${{ matrix.arch }}" + CIBW_BUILD: "cp311-* cp312-* cp313-*" + # This tells cibuildwheel to skip tests for all builds + CIBW_TEST_SKIP: "*" + with: + package-dir: ./psutil + output-dir: wheelhouse diff --git a/.github/workflows/pyproj.yaml b/.github/workflows/pyproj.yaml new file mode 100644 index 00000000..2db1301f --- /dev/null +++ b/.github/workflows/pyproj.yaml @@ -0,0 +1,52 @@ +--- +name: PyProj wheels +on: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: "tmate debugging enabled" + required: false + default: false + pull_request: +jobs: + build_pyproj: + name: Build wheels of pyproj on "${{ matrix.os }}, ${{ matrix.arch }}" + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, arch: x86_64 } + - { os: ubuntu-24.04-arm, arch: aarch64 } + - { os: macos-15, arch: x86_64 } + - { os: macos-15, arch: arm64 } + - { os: windows-2025, arch: AMD64 } + - { os: windows-11-arm, arch: ARM64 } + env: + REPO_DIR: pyproj + BUILD_COMMIT: 3.7.2 + steps: + - uses: actions/checkout@v6 + - name: get code of pyproj + run: | + echo $PATH + bash -x .github/workflows/code.sh + - name: Install Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + - name: Build Wheel of pyproj + uses: pypa/cibuildwheel@v4.0.0 + env: + REPO_DIR: pyproj + BUILD_TARGET: proj + CIBW_ARCHS: "${{ matrix.arch }}" + CIBW_BUILD: "cp311-* cp312-* cp313-*" + # This tells cibuildwheel to skip tests for all builds + CIBW_TEST_SKIP: "*" + CIBW_BEFORE_ALL: ./.github/workflows/build.sh + with: + package-dir: ./pyproj + output-dir: wheelhouse diff --git a/.github/workflows/wheel.sh b/.github/workflows/wheel.sh deleted file mode 100755 index 99097038..00000000 --- a/.github/workflows/wheel.sh +++ /dev/null @@ -1,18 +0,0 @@ -echo "::group::Build wheel" - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - before_install - echo "+++++++++++++++++++++++++++++++++++++++++" - echo "check python of venv after before_install" - echo "+++++++++++++++++++++++++++++++++++++++++" - which python - python3 -c "import sys; print(sys.version)" | awk -F \. {'print $1$2'} - echo $PIP_CMD - echo $PYTHON_EXE - echo $REPO_DIR - echo $PLAT - echo " ${PROJ_DIR} ${PROJ_DATA}" - echo " ${PROJ_WHEEL} ${PROJ_NETWORK}" - build_wheel $REPO_DIR $PLAT - ls -l "${GITHUB_WORKSPACE}/${WHEEL_SDIR}/" -echo "::endgroup::" diff --git a/config.sh b/config.sh deleted file mode 100644 index 24f9fee6..00000000 --- a/config.sh +++ /dev/null @@ -1,428 +0,0 @@ -# Custom utilities for Fiona wheels. -# -# Test for OSX with [ -n "$IS_OSX" ]. - -function fetch_unpack { - # Fetch input archive name from input URL - # Parameters - # url - URL from which to fetch archive - # archive_fname (optional) archive name - # - # Echos unpacked directory and file names. - # - # If `archive_fname` not specified then use basename from `url` - # If `archive_fname` already present at download location, use that instead. - local url=$1 - if [ -z "$url" ];then echo "url not defined"; exit 1; fi - local archive_fname=${2:-$(basename $url)} - local arch_sdir="${ARCHIVE_SDIR:-archives}" - # Make the archive directory in case it doesn't exist - mkdir -p $arch_sdir - local out_archive="${arch_sdir}/${archive_fname}" - # If the archive is not already in the archives directory, get it. - if [ ! -f "$out_archive" ]; then - # Source it from multibuild archives if available. - local our_archive="${MULTIBUILD_DIR}/archives/${archive_fname}" - if [ -f "$our_archive" ]; then - ln -s $our_archive $out_archive - else - # Otherwise download it. - curl -v --insecure -L $url > $out_archive - fi - fi - # Unpack archive, refreshing contents, echoing dir and file - # names. - rm_mkdir arch_tmp - install_rsync - (cd arch_tmp && \ - untar ../$out_archive && \ - ls -1d * && - rsync --delete -ah * ..) -} - -function build_geos { - if [ -e geos-stamp ]; then return; fi - CFLAGS="$CFLAGS -g -O2" - CXXFLAGS="$CXXFLAGS -g -O2" - local cmake=cmake - fetch_unpack http://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 - (cd geos-${GEOS_VERSION} \ - && mkdir build && cd build \ - && $cmake .. \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_INSTALL_PREFIX:PATH=$BUILD_PREFIX \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_IPO=ON \ - -DBUILD_APPS:BOOL=OFF \ - -DBUILD_TESTING:BOOL=OFF \ - && $cmake --build . -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch geos-stamp - #&& sudo $cmake --install .) -} - -function build_jsonc { - if [ -e jsonc-stamp ]; then return; fi - local cmake=cmake - fetch_unpack https://s3.amazonaws.com/json-c_releases/releases/json-c-${JSONC_VERSION}.tar.gz - (cd json-c-${JSONC_VERSION} \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET . \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - if [ -n "$IS_OSX" ]; then - for lib in $(ls ${BUILD_PREFIX}/lib/libjson-c.5*.dylib); do - sudo install_name_tool -id $lib $lib - done - for lib in $(ls ${BUILD_PREFIX}/lib/libjson-c.dylib); do - sudo install_name_tool -id $lib $lib - done - fi - touch jsonc-stamp -} - - -function build_tiff { - if [ -e tiff-stamp ]; then return; fi - build_zlib - build_jpeg - #if [ -n "$IS_OSX" ]; then brew install curl; else echo "compilation on ML" ; fi - ensure_xz - fetch_unpack https://download.osgeo.org/libtiff/tiff-${TIFF_VERSION}.tar.gz - (cd tiff-${TIFF_VERSION} \ - && mv VERSION VERSION.txt \ - && (patch -u --force < ../patches/libtiff-rename-VERSION.patch || true) \ - && ./configure --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch tiff-stamp -} - - -function build_proj { - build_sqlite - CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" - CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" - if [ -e proj-stamp ]; then return; fi - local cmake=cmake - echo "env: $PROJ_DIR and build prefix ${BUILD_PREFIX}" - echo "env: $PROJ_DATA and build prefix ${BUILD_PREFIX}" - # - fetch_unpack http://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz - (cd proj-${PROJ_VERSION} - mkdir build - cd build - cmake .. \ - -DCMAKE_INSTALL_PREFIX=$PROJ_DIR \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - -DCMAKE_PREFIX_PATH=$BUILD_PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_IPO=ON \ - -DBUILD_APPS:BOOL=ON \ - -DBUILD_TESTING:BOOL=OFF -# -DCMAKE_INSTALL_LIBDIR=lib -# -DBUILD_CCT:BOOL=ON \ -# -DBUILD_CS2CS:BOOL=ON \ -# -DBUILD_GEOD:BOOL=ON \ -# -DBUILD_GIE:BOOL=ON \ -# -DBUILD_GMOCK:BOOL=ON \ -# -DBUILD_PROJINFO:BOOL=ON \ -# -DBUILD_TESTING:BOOL=OFF - cmake --build . -j4 - (if [ -n "$IS_OSX" ]; then sudo cmake --install . ; else cmake --install .; fi)) - # https://github.com/OSGeo/PROJ-data - echo "fetch_unpack https://github.com/OSGeo/PROJ-data/archive/refs/tags/${PROJ_DATA_VER}.tar.gz " - ls -lrt - echo "ls -lrt ${PROJ_DATA}" - ls -lrt ${PROJ_DATA} - #projsync --all - touch proj-stamp -} - - -function build_sqlite { - if [ -e sqlite-stamp ]; then return; fi - fetch_unpack https://www.sqlite.org/2023/sqlite-autoconf-${SQLITE_VERSION}.tar.gz - (cd sqlite-autoconf-${SQLITE_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch sqlite-stamp -} - - -function build_expat { - if [ -e expat-stamp ]; then return; fi - fetch_unpack https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-${EXPAT_VERSION}.tar.bz2 - (cd expat-${EXPAT_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch expat-stamp -} - - -function build_nghttp2 { - if [ -e nghttp2-stamp ]; then return; fi - fetch_unpack https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz - (cd nghttp2-${NGHTTP2_VERSION} \ - && ./configure --enable-lib-only --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch nghttp2-stamp -} - - -function build_openssl { - if [ -e openssl-stamp ]; then return; fi - fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz - check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} - (cd ${OPENSSL_ROOT} \ - && ./config no-ssl2 -fPIC --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch openssl-stamp -} - - -function build_curl { - if [ -e curl-stamp ]; then return; fi - CFLAGS="$CFLAGS -g -O2" - CXXFLAGS="$CXXFLAGS -g -O2" - build_nghttp2 - build_openssl - local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl" - #local flags="--prefix=$BUILD_PREFIX --with-libz --with-ssl" - #fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz - (cd curl-${CURL_VERSION} \ - && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib:$BUILD_PREFIX/lib64 ./configure $flags \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) - touch curl-stamp -} - -function build_pcre2 { - build_simple pcre2 $PCRE_VERSION https://github.com/PCRE2Project/pcre2/releases/download/pcre2-${PCRE_VERSION} -} - -function build_gdal { - if [ -e gdal-stamp ]; then return; fi - build_curl - build_jpeg - build_libpng - build_jsonc - build_tiff - build_proj - build_sqlite - build_expat - build_geos - - EXPAT_PREFIX=$BUILD_PREFIX - CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" - CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" - - if [ -n "$IS_OSX" ]; then - GEOS_CONFIG="-DGDAL_USE_GEOS=OFF" - else - GEOS_CONFIG="-DGDAL_USE_GEOS=ON" - fi - - local cmake=cmake - fetch_unpack http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz - (cd gdal-${GDAL_VERSION} - mkdir build - cd build - # build using cmake - cmake .. \ - -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ - -DCMAKE_INCLUDE_PATH=$BUILD_PREFIX/include \ - -DCMAKE_LIBRARY_PATH=$BUILD_PREFIX/lib \ - -DCMAKE_PROGRAM_PATH=$BUILD_PREFIX/bin \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ - -DOGR_BUILD_OPTIONAL_DRIVERS=ON \ - ${GEOS_CONFIG} \ - -DGDAL_USE_TIFF=ON \ - -DGDAL_USE_TIFF_INTERNAL=OFF \ - -DGDAL_USE_GEOTIFF_INTERNAL=ON \ - -DGDAL_ENABLE_DRIVER_GIF=ON \ - -DGDAL_ENABLE_DRIVER_GRIB=ON \ - -DGDAL_ENABLE_DRIVER_JPEG=ON \ - -DGDAL_USE_ICONV=ON \ - -DGDAL_USE_JSONC=ON \ - -DGDAL_USE_JSONC_INTERNAL=OFF \ - -DGDAL_USE_ZLIB=ON \ - -DGDAL_USE_ZLIB_INTERNAL=OFF \ - -DGDAL_ENABLE_DRIVER_PNG=ON \ - -DGDAL_ENABLE_DRIVER_OGCAPI=OFF \ - -DOGR_ENABLE_DRIVER_GPKG=ON \ - -DBUILD_PYTHON_BINDINGS=OFF \ - -DBUILD_JAVA_BINDINGS=OFF \ - -DBUILD_CSHARP_BINDINGS=OFF \ - -DGDAL_USE_SFCGAL=OFF \ - -DGDAL_USE_XERCESC=OFF \ - -DGDAL_USE_LIBXML2=OFF \ - -DGDAL_USE_POSTGRESQL=OFF \ - -DGDAL_USE_ODBC=OFF - cmake --build . -j4 - (if [ -n "$IS_OSX" ]; then sudo cmake --install . ; else cmake --install .; fi)) - if [ -n "$IS_OSX" ]; then - : - else - strip -v --strip-unneeded ${BUILD_PREFIX}/lib/libgdal.so.* || true - strip -v --strip-unneeded ${BUILD_PREFIX}/lib64/libgdal.so.* || true - fi - touch gdal-stamp -} - - -function pre_build { - # Any stuff that you need to do before you start building the wheels - # Runs in the root directory of this repository. - #if [ -n "$IS_OSX" ]; then - # # Update to latest zlib for OSX build - # build_new_zlib - #fi - if [ -z "$IS_OSX" ]; then - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib - export LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib - fi - if [[ "$REPO_DIR" != "psutil" ]]; then - build_nghttp2 - build_openssl - # Remove previously installed curl. - #sudo rm -rf /usr/local/lib/libcurl* - if [ -n "$IS_OSX" ]; then sudo rm -rf /usr/local/lib/libcurl* ; else rm -rf /usr/local/lib/libcurl* ; fi - fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz - build_zlib - build_curl - build_sqlite - build_tiff - build_proj - if [[ "$REPO_DIR" != "pyproj" ]] ; then - build_jpeg - build_libpng - build_jsonc - build_expat - build_geos - build_gdal - fi - if [[ "$REPO_DIR" != "geopandas" ]] ; then - build_jpeg - build_libpng - build_jsonc - build_expat - build_geos - build_gdal - fi - if [ -n "$IS_OSX" ]; then - export LDFLAGS="${LDFLAGS} -Wl,-rpath,${BUILD_PREFIX}/lib" - if [[ "$REPO_DIR" == "pyproj" ]]; then - export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PROJ_DIR}/lib" - fi - fi - fi -} - -function run_tests { - if [ -n "$IS_OSX" ]; then - export PATH=$PATH:${BUILD_PREFIX}/bin - export LC_ALL=en_US.UTF-8 - export LANG=en_US.UTF-8 - else - export LC_ALL=C.UTF-8 - export LANG=C.UTF-8 - export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - apt-get update - apt-get install -y ca-certificates - fi - if [[ "$REPO_DIR" == "Fiona" ]]; then - unset GDAL_DATA - unset PROJ_LIB - unset PROJ_DATA - cp -R ../Fiona/tests ./tests - python -m pip install "shapely" $TEST_DEPENDS - GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]" - fio --version - fio env --formats - python ../test_fiona_issue383.py - fi - if [[ "$REPO_DIR" == "gdal" ]]; then - echo "Run import to test that numpy is included" - python3 -c 'from osgeo import gdal_array' - fi -} - - -function build_wheel_cmd { - local cmd=${1:-build_cmd} - local repo_dir=${2:-$REPO_DIR} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - #start_spinner - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - #stop_spinner - pip install -U pip - pip install -U build - echo "REPO_DIR:: $REPO_DIR" - - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $BUILD_DEPENDS - fi - if [ "$REPO_DIR" == "Fiona" ]; then - (cd $repo_dir && GDAL_VERSION=$GDAL_VERSION $cmd $wheelhouse) - fi - if [ "$REPO_DIR" == "gdal" ]; then - pip download GDAL==${GDAL_VERSION} - echo "Control the DATA" - tar xzvf gdal-${GDAL_VERSION}.tar.gz - cd gdal-${GDAL_VERSION} - $cmd $wheelhouse - fi - if [ "$REPO_DIR" == "pyproj" ]; then - (cd $repo_dir && $cmd $wheelhouse) - fi - if [ "$REPO_DIR" == "psutil" ]; then - (cd $repo_dir && $cmd $wheelhouse) - fi - if [ "$REPO_DIR" == "pyogrio" ]; then - # setting git safe directory is required for properly building wheels when - # git >= 2.35.3 - (cd $repo_dir && git config --global --add safe.directory "*" && $cmd $wheelhouse) - fi - if [ "$REPO_DIR" == "geopandas" ]; then - # setting git safe directory is required for properly building wheels when - # git >= 2.35.3 - (cd $repo_dir && git config --global --add safe.directory "*" && $cmd $wheelhouse) - fi - if [ "$REPO_DIR" == "rasterio" ]; then - (cd $repo_dir && git config --global --add safe.directory "*" && GDAL_VERSION=$GDAL_VERSION $cmd $wheelhouse) - fi - if [ -n "$IS_OSX" ]; then - pip install delocate - delocate-listdeps --all --depending $wheelhouse/*.whl - else # manylinux - pip install auditwheel - fi - repair_wheelhouse $wheelhouse -} - - -function build_cmd { - local abs_wheelhouse=$1 - python -vv -m build -o $abs_wheelhouse -} - - -function macos_arm64_native_build_setup { - # Setup native build for single arch arm_64 wheels - export PLAT="arm64" - # We don't want universal2 builds and only want an arm64 build - export _PYTHON_HOST_PLATFORM="macosx-13.0-arm64" - export ARCHFLAGS+=" -arch arm64" - $@ -} diff --git a/env_vars.sh b/env_vars.sh deleted file mode 100644 index 35002c2e..00000000 --- a/env_vars.sh +++ /dev/null @@ -1,45 +0,0 @@ -SQLITE_VERSION=3440000 -CURL_VERSION=8.4.0 -LIBPNG_VERSION=1.6.35 -ZLIB_VERSION=1.2.11 -JPEG_VERSION=9f -GEOS_VERSION=3.11.5 -JSONC_VERSION=0.18 -PROJ_VERSION=9.6.0 -PROJ_DATA_VER=1.19.0 -GDAL_VERSION=3.10.3 -NGHTTP2_VERSION=1.46.0 -EXPAT_VERSION=2.2.6 -OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ -OPENSSL_ROOT=openssl-1.1.1w -OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 -PCRE_VERSION=10.44 -EXPAT_VERSION=2.6.3 -TIFF_VERSION=4.7.0 -PCRE_VERSION=10.44 -PYOGRIO_VERSION=v0.11.1 -export MACOSX_DEPLOYMENT_TARGET=11.0 -if [ -z "$IS_OSX" ] || [ "$PLAT" == x86_64 ]; then - export MACOSX_DEPLOYMENT_TARGET=13.0 -fi -if [ -z "$IS_OSX" ] || [ "$PLAT" == arm_64 ]; then - export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" -fi -export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 -export GDAL_CONFIG=/usr/local/bin/gdal-config -export PACKAGE_DATA=1 -#from PROJ 9.x -export PROJ_DIR=/usr/local/ -export PROJ_DATA=${PROJ_DIR}share/proj -export PROJ_WHEEL=true -export PROJ_NETWORK=ON -#export SETUPTOOLS_USE_DISTUTILS=stdlib -if [[ "$REPO_DIR" == "pyproj" ]]; then - export PROJ_DIR=${GITHUB_WORKSPACE}/pyproj/pyproj/proj_dir - export PROJ_DATA=${PROJ_DIR}/share/proj - if [ -z "$IS_OSX" ]; then - echo "PROJ_DIR on ManyLinux " - export PROJ_DIR=/io/pyproj/pyproj/proj_dir - export PROJ_DATA=${PROJ_DIR}/share/proj - fi -fi diff --git a/multibuild.submodules/.appveyor.yml b/multibuild.submodules/.appveyor.yml deleted file mode 100644 index 6db3a67a..00000000 --- a/multibuild.submodules/.appveyor.yml +++ /dev/null @@ -1,91 +0,0 @@ -# https://www.appveyor.com/docs/windows-images-software - -image: Visual Studio 2022 - -environment: - global: - REPO_DIR: python-appveyor-demo - PACKAGE_NAME: python_appveyor_demo - BUILD_COMMIT: master - BUILD_DEPENDS: "cython" - TEST_DEPENDS: "nose" - - matrix: - - PYTHON: "C:\\Miniconda37" - PYTHON_VERSION: "3.7" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Miniconda37-x64" - PYTHON_VERSION: "3.7" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Miniconda38-x64" - PYTHON_VERSION: "3.8" - PYTHON_ARCH: "64" - -# We always use a 64-bit machine, but can build x86 distributions -# with the TARGET_ARCH variable. -platform: - - x64 - -matrix: - fast_finish: false - -install: - # Install miniconda and fix headers - - where python - - where py # On Windows, py might be a better way to find and run CPython - - py --list # py -3.10-32, -3.10-64 with same syntax for 3.{9-3}, 2.{7,6} - - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PYTHON%\Library\bin;%PATH% - - echo %PATH% - - where conda - - conda info - - # Check that we have the expected version and architecture for Python - - python --version - - python -c "import struct; print(struct.calcsize('P') * 8)" - - py --version - - py -c "import struct; print(struct.calcsize('P') * 8)" - - # clone a origsel/python-appveyor-demo - # this repo includes a simple package to test appveyor - - git clone https://github.com/ogrisel/python-appveyor-demo.git - -build_script: - # Install build requirements - - conda install --yes %BUILD_DEPENDS% - - # build wheel: - - cd %REPO_DIR% - - git checkout %BUILD_COMMIT% - - python setup.py bdist_wheel - - ls dist/* - -test_script: - # create test env - - conda create --yes -n test_env python=%PYTHON_VERSION% %TEST_DEPENDS% - - activate test_env - - # install from wheel - - pip install --no-index --find-links dist/ %PACKAGE_NAME% - - # run tests from install wheel - - cd .. - - python -m pyappveyordemo.tests.test_extension - - # Smoke test of install_python script - # Use C:\PythonXY, C:\PythonXY-x64, C:\PythonXYrcZ, or C:\PythonXYrcZ-x64 - - set PYTHON=C:\Python37 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python37-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python38-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python38 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python39-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python39 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python310-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python310 - - ps: .\install_python.ps1 diff --git a/multibuild.submodules/.github/workflows/lint_python.yml b/multibuild.submodules/.github/workflows/lint_python.yml deleted file mode 100644 index fe56fb21..00000000 --- a/multibuild.submodules/.github/workflows/lint_python.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: lint_python -on: [pull_request, push, workflow_dispatch] -jobs: - lint_python: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - run: pip install --upgrade pip wheel setuptools - - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear - flake8-comprehensions isort mypy pytest pyupgrade safety - - run: bandit --recursive --skip B101 . # B101 is assert statements - - run: black --check . || true - - run: codespell --ignore-words-list="commend" # --skip="*.css,*.js,*.lock" - - run: flake8 . --count --max-complexity=10 --max-line-length=88 - --show-source --statistics - - run: isort --check-only --profile black . - - run: pip install -r requirements.txt || pip install --editable . || true - - run: mkdir --parents --verbose .mypy_cache - - run: mypy --ignore-missing-imports --install-types --non-interactive . - - run: pytest . || pytest --doctest-modules . || true - - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true - - run: safety check diff --git a/multibuild.submodules/.github/workflows/test.yml b/multibuild.submodules/.github/workflows/test.yml deleted file mode 100644 index 0ff11f16..00000000 --- a/multibuild.submodules/.github/workflows/test.yml +++ /dev/null @@ -1,47 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the devel branch - push: - branches: [ devel ] - pull_request: - branches: [ devel ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] # [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.12"] # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"] - - # The type of runner that the job will run on - runs-on: ${{ matrix.os }} - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v6 - - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - # Runs a single command using the runner's Python - - name: Run a one-line script - shell: python - run: print("Hello, world!") - - # Runs a set of commands using the runner's shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. diff --git a/multibuild.submodules/.gitignore b/multibuild.submodules/.gitignore deleted file mode 100644 index 37d7cb51..00000000 --- a/multibuild.submodules/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -downloads/ -archives/ -*.orig -*.swp - -# lib_check downloads these -*-stamp -arb*/ -bzip2*/ -cfitsio/ -flex*/ -freetype*/ -giflib*/ -hdf5*/ -jpeg*/ -lcms2*/ -libaec*/ -libpng*/ -libwebp*/ -lzo*/ -mpir*/ -openssl*/ -pcre*/ -ragel*/ -szip*/ -swig*/ -tiff*/ -xz*/ -yaml*/ diff --git a/multibuild.submodules/.travis.yml b/multibuild.submodules/.travis.yml deleted file mode 100644 index b7a128cf..00000000 --- a/multibuild.submodules/.travis.yml +++ /dev/null @@ -1,234 +0,0 @@ -# Set TEST_BUILDS=1 to run library_builders tests - -language: generic - -cache: - directories: - - $HOME/.ccache - -env: - global: - # Always set Python version - - MB_PYTHON_VERSION=3.9 - -matrix: - include: - - os: linux - dist: trusty - language: python - # 64-bit builds - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - # 64-bit builds - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - - MB_ML_VER=2010 - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - - MB_ML_VER=_2_24 - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - # 32-bit builds - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - - PLAT=i686 - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - - PLAT=i686 - - MB_ML_VER=2010 - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - # Builds with caching - - os: linux - dist: xenial - env: - - TEST_BUILDS=1 - - USE_CCACHE=1 - # Build with pypy3 - - os: linux - env: - - MB_PYTHON_VERSION=pypy3.6-7.3 - - MB_ML_VER=2014 - - TEST_BUILDS=1 - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - # OSX builds - - os: osx - env: - - MB_PYTHON_VERSION=2.7.17 # 2.7.18 doesn't have a 10.6 build - - MB_PYTHON_OSX_VER=10.6 - - TEST_BUILDS=1 - - OPENSSL_ROOT=openssl-1.0.2u - - OPENSSL_HASH=ecd0c6ffb493dd06707d38b14bb4d8c2288bb7033735606569d8f90f89669d16 - - os: osx - env: - - MB_PYTHON_VERSION=2.7 - - TEST_BUILDS=1 - - os: osx - env: - - MB_PYTHON_VERSION=pypy3.6-7.3 - - MB_PYTHON_OSX_VER=10.9 - - TEST_BUILDS=1 - - os: osx - language: objective-c - env: - - MB_PYTHON_VERSION=2.7 - - VENV=venv - - os: osx - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - env: - - MB_PYTHON_VERSION=3.5 - - VENV=venv - # this is expected to fail - # 3.5 / 10.9 build doesn't exist - - os: osx - env: - - MB_PYTHON_VERSION=3.5 - - MB_PYTHON_OSX_VER=10.9 - - OSX_ENV_EXPECT_FAIL=true - - os: osx - env: - - MB_PYTHON_VERSION=3.6 - - MB_PYTHON_OSX_VER=10.6 - - VENV=venv - - os: osx - env: - - MB_PYTHON_VERSION=3.6 - - VENV=venv - - os: osx - env: - - MB_PYTHON_VERSION=3.7 - - VENV=venv - - os: osx - osx_image: xcode10.2 - env: - - MB_PYTHON_VERSION=3.7 - - VENV=venv - - USE_CCACHE=1 - - os: osx - env: - - MB_PYTHON_VERSION=3.8 - - VENV=venv - - os: osx - env: - - MB_PYTHON_VERSION=3.9 - - VENV=venv - - os: osx - env: - - MB_PYTHON_VERSION=pypy-7.3 - - TEST_BUILDS=1 - - VENV=venv - # Default OSX (xcode image) is 10.13 (xcode 9.4.1) as of 2018-08-03 - # See: https://docs.travis-ci.com/user/reference/osx/ - - os: osx - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.7 - - TEST_BUILDS=1 - - os: osx - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.7 - - MB_PYTHON_OSX_VER=10.9 - - TEST_BUILDS=1 - - os: osx - osx_image: xcode10 - env: - - MB_PYTHON_VERSION=3.7 - - os: osx - osx_image: xcode10 - env: - - MB_PYTHON_VERSION=3.8 - - os: osx - osx_image: xcode10 - env: - - MB_PYTHON_VERSION=3.9 - - os: osx - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode10 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode9.3 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode9.2 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode9.1 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode9 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode8.3 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode8.2 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode8.1 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode8 - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - osx_image: xcode7.3 - env: - - MB_PYTHON_VERSION=3.5 - # ARM aarch 64 builds - - os: linux - arch: arm64 - env: - - MB_PYTHON_VERSION=3.7 - - MB_ML_VER=2014 - - TEST_BUILDS=1 - - PLAT=aarch64 - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - # s390x builds - - os: linux - arch: s390x - env: - - MB_PYTHON_VERSION=3.7 - - MB_ML_VER=2014 - - TEST_BUILDS=1 - - PLAT=s390x - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - # ppc64le builds - - os: linux - arch: ppc64le - env: - - MB_PYTHON_VERSION=3.7 - - MB_ML_VER=2014 - - TEST_BUILDS=1 - - PLAT=ppc64le - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - -script: - - export ENV_VARS_PATH="tests/env_vars.sh" - - source tests/test_multibuild.sh diff --git a/multibuild.submodules/LICENSE b/multibuild.submodules/LICENSE deleted file mode 100644 index 338dd5cc..00000000 --- a/multibuild.submodules/LICENSE +++ /dev/null @@ -1,34 +0,0 @@ -.. _license: - -********************* -Copyright and License -********************* - -The multibuild package, including all examples, code snippets and attached -documentation is covered by the 2-clause BSD license. - - Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights - reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/multibuild.submodules/README.rst b/multibuild.submodules/README.rst deleted file mode 100644 index 464f8271..00000000 --- a/multibuild.submodules/README.rst +++ /dev/null @@ -1,614 +0,0 @@ -################################################ -Utilities for building on Travis CI and AppVeyor -################################################ - -************************************ -Update: Uploads, Rackspace, Anaconda -************************************ - -The original Multibuild default was to upload wheels to a Rackspace container, -where Rackspace kindly donated the hosting to the Scikit-learn team. We had -a URL pointing to the Rackspace container: `http://wheels.scipy.org`. - -Rackspace finally stopped subsidizing this container, and the Rackspace of -`http://wheels.scipy.org` is no more. Some projects using Multibuild have moved -to using https://anaconda.org/scipy-wheels-nightly/ for weekly uploads and -https://anaconda.org/multibuild-wheels-staging for staging wheels to PyPI. - -******************* -Uploads to Anaconda -******************* - -If you want to upload to Anaconda, and you don't need the extra storage space for nightly builds that Anaconda kindly donates to NumPy, SciPy etc, then you can do this with your own Anaconda organization. - -See https://github.com/MacPython/nipy-wheels for a simple example. - -* Make an account at Anaconda.org. -* Make an organization - for example we have used ``nipy``. -* Navigate to ``https://anaconda.org/nipy`` (but use your organization). -* Go to your account menu towards the top left. This should be labeled with your - organization name. -* Select "Settings", then "Access". -* Create an access token. Give it permission: "Allow write access to the API - site", and (not sure if this is necessary) "Allow uploads to PyPI - repositories". -* "View" your token. It may be of form - ``ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6``. -* Encrypt this to your repository, maybe using the ``travis`` command line tool - (``gem install travis``). Your command will be something like:: - - travis encrypt -r MacPython/nipy-wheels ANACONDA_SECRET=ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6 - - Note that ``MacPython/nipy-wheels`` is your Github organization/repository. The encryption only applies to Travis CI running against this repository. - -* Go to your `.travis.yml` file and add the output ``secure`` key. This will - look something like:: - - env: - global: - # Following generated with - # travis encrypt -r MacPython/nipy-wheels ANACONDA_SECRET= - # where has API write access to the anaconda.org - # organization named in $ANACONDA_ORG - - secure: "IqN7LjXWVBaijggUoB+ohjzFzH6nU0OyxznXEMgWoNxQJRiYXXKAt/Z5c4ldp9LUynefJO306M8foN4Gm8M8PNDlhjElzdOtIkGYtDKUXx7aXtrg8rPk1mzuM1F27er4Dbi7WFtpPClr8z8JKNNV50yeM1o2cXu4HgrPrRKgKk/2D8EQaPQlcOqul0O63D9AjVoW3EIG0aWEnZQQGfuGAPgyr0OS92LX2h1pcD2lNZHhqYmXmm5U0IwZmWL3Y0N7PO3VXcOCeIbiHAlJzhk4C4+86TT7DN+VhmfGyY/s61fOz47K+lEZLVqqeQki+HV75fti0XwYG7rjcSvDanNx+w2J/ogSLQpiNxZ0FZ+W8psXEaFUgFf7oXzRkW9gQ4KAsItEWHifq061ngr5AWLPLh+01LGP1Xg8wT5WEVUzBfD2uJPsy20DLcP9WGYa6cBNwtpqmUkdVgM3ZCPWlro7+v1kqxsKp91uh8SRKVlkD4mwbf0FnWxbNZ9v4Z9gs0pZoRclzL+/YcIcSTYAwiQRqaX7T0tpxaUZ0VYTMwCgpsufUX1idV1HV5+WKr9FUocoq+1RRW/JeXkisX9FRvem8cSGmnxB/hynlxoqzttCVMwtrKWPwxH4dHD+lavouho68Q7iBql1ZBZEhQy0O9NC1wr4Rg2CeDPZuzqVjmSPuXQ=" - - When Travis CI runs, this causes the ``ANACONDA_SECRET`` environment variable - to contain the API key above. - - Also add this to your global environment variables:: - - - ANACONDA_ORG="nipy" - -* To upload from Travis CI, add a clause like this to the end of your - ``.travis.yml``:: - - after_success: - # Upload wheels to Anaconda.org - - pip install git+https://github.com/Anaconda-Platform/anaconda-project - - pip install git+https://github.com/Anaconda-Platform/anaconda-client - - anaconda -t ${ANACONDA_SECRET} upload --force -u ${ANACONDA_ORG} ${TRAVIS_BUILD_DIR}/wheelhouse/*.whl - - ``wheelhouse/*.whl`` defines the files you want to upload. - -* You might also want to build and upload from AppVeyor. To encrypt the API - key above, go to https://ci.appveyor.com/account/matthew-brett/settings - (where ``matthew-brett`` is the account from which your AppVeyor job runs. - Click on "Encrypt YaML" on the left. Type in your API key value (e.g. - ``ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6``) as the value to encrypt. Click "Encrypt" and note the text it suggests. Now, at the top of your ``appveyor.yml`` file, add something like:: - - environment: - global: - ANACONDA_ORG: "nipy" - ANACONDA_SECRET: - secure: Ds0PkQD0b/QOfoNoiPuFJb01zg0Mq0dkAxIG2jRXocCAereSXdWw6XYaDrutHWme - - where ``secure:`` is the text suggested from "Encrypt" above, and ``nipy`` is your Anaconda organization. - - Finally, add a clause like the following to the end of your ``appveyor.yml`` file:: - - on_success: - # Upload the generated wheel package to Anaconda.org - - pip install git+https://github.com/Anaconda-Platform/anaconda-project - - pip install git+https://github.com/Anaconda-Platform/anaconda-client - - anaconda -t %ANACONDA_SECRET% upload --force -u %ANACONDA_ORG% nipy\dist\*.whl - - where ``nipy\dist\*.whl`` finds the files you want to upload. - -There's a simple example of these steps applied at -https://github.com/MacPython/nipy-wheels. - -Here is the NumPy code (for running on Travis CI) to upload to Anaconda: -https://github.com/MacPython/numpy-wheels/blob/master/.travis.yml#L99 - -For projects housed under the MacPython GitHub organization, you have access to -Anaconda upload tokens via the "Organization Secrets" -https://github.com/MacPython/numexpr-wheels/settings/secrets . You can use -these to move to GitHub Actions (they provide x86 machines for Windows, Linux -and Mac). Otherwise we (please raise an issue here) will need to negotiate -getting you tokens, or you can make your own, as above. - -********************** -Use Github for uploads -********************** - -Another option is to use GitHub for staging --- as do Cython `for Travis CI -`_ and -`for AppVeyor -`_. - -************ -Introduction -************ - -A set of scripts to automate builds of macOS and Manylinux1 wheels on the -`Travis CI `_ infrastructure, and also Windows -wheels on the `AppVeyor `_ infrastructure. - -The Travis CI scripts are designed to build *and test*: - -* 64-bit macOS wheels built for macOS 10.9+ -* 64/32-bit macOS wheels built for macOS 10.6+ -* 64-bit ``manylinuxX_x86_64`` wheels, both narrow and wide Unicode builds, - where `X` is any valid Manylinux version: `1`, `2010`, `2014`, `_2_24` or `_2_28`. -* 32-bit ``manylinuxX_i686`` wheels, both narrow and wide Unicode builds - -You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10 -and 3.11. - -The small innovation here is that you can test against Linux 32-bit builds, both -wide and narrow Unicode Python 2 builds, which was not easy on the default -Travis CI configurations. - -The AppVeyor setup is designed to build *and test*: - -* 64-bit Windows ``win_amd64`` wheels -* 32-bit Windows ``win32`` wheels - -You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10 -and 3.11. - -***************** -How does it work? -***************** - -Multibuild is a series of bash scripts that define bash functions to build and -test wheels. - -Configuration is by overriding the default build function, and defining a test -function. - -The bash scripts are layered, in the sense that they are composed of a number of scripts -which are sourced in sequence, each one potentially overriding previous ones. - -macOS -===== - -The following bash scripts are sourced in this order:: - - multibuild/common_utils.sh - multibuild/osx_utils.sh - env_vars.sh - multibuild/configure_build.sh - multibuild/library_builders.sh - config.sh - -See ``multibuild/travis_osx_steps.sh`` - -The macOS build / test phases run on the macOS VM started by Travis CI. -Therefore any environment variable defined in ``.travis.yml`` or the bash -shell scripts listed above are available for your build and test. - -Build options are controlled mainly by the following environment -variables: - -* ``MB_PYTHON_VER`` sets the Python version targeted: ``major.minor.patch`` - for CPython, or ``pypy-major.minor`` for PyPy. -* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C - extensions. For CPython targets it may be set to 10.6 or 10.9, provided a - corresponding Python build is available at `python.org - `_. It defaults to the highest - version available. It's ignored for PyPy targets. -* ``PLAT`` sets the architectures built for any C extensions: ``x86_64`` or - ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same - arches as the target Python version: 64-bit for CPython macOS 10.9 or PyPy, - and 64/32-bit for CPython 10.6. - -In most cases it's best to rely on the defaults for ``MB_PYTHON_OSX_VER`` and -``PLAT``, rather than setting them explicitly. Examples of exceptions to this -guideline include: - -* setting ``MB_PYTHON_OSX_VER=10.6`` to build a 10.6 64/32-bit CPython wheel - for Python 2.7 (default for 2.7 is 10.9 64-bit) -* setting ``MB_PYTHON_OSX_VER=10.6 and PLAT=x86_64`` to build a 10.6 64-bit - only wheel (10.6 would normally be 64/32-bit). Such a wheel would still have - a platform tag of ``macosx_10_6_intel`` , advertising support for both 64 and - 32-bit, but wouldn't work in 32-bit mode. This may be OK given how unlikely it - is that there is still anyone actually running Python on macOS in 32-bit - mode. - -The ``build_wheel`` function builds the wheel, and ``install_run`` -function installs and tests it. Look in ``multibuild/common_utils.sh`` for -default definitions of these functions. See below for more details, many of -which are common to macOS and Linux. - -Manylinux -========= - -The build phase is in a Manylinux Docker container, but the test phase is in -a clean container. - - -Build phase ------------ - -Specify the Manylinux version to build for with the ``MB_ML_VER`` environment -variable. The default version is ``2014``. Versions that are currently valid are: - -* ``1`` corresponding to manylinux1 (see `PEP 513 `_). -* ``2010`` corresponding to manylinux2010 (see `PEP 571 `_). -* ``2014`` corresponding to manylinux2014 and adds more architectures to ``PLAT`` - (see `PEP 599 `_). -* ``_2_24`` corresponding to manylinux_2_24 (see `PEP 600 `_). -* ``_2_28`` corresponding to manylinux_2_28 (see `PEP 600 `_). - -The environment variable specified which Manylinux docker container you are building in. - -The ``PLAT`` environment variable can be one of - -* ``x86_64``, for 64-bit x86 -* ``i686``, for 32-bit x86 -* ``s390x``, for 64-bit s390x -* ``ppc64le``, for PowerPC -* ``aarch64``, for ARM -* ``arm64``, for Apple silicon -* ``universal2``, for both Apple silicon and 64-bit x86 - -The default is ``x86_64``. Only ``x86_64`` and ``i686`` are valid on manylinux1 and manylinux2010. - -``multibuild/travis_linux_steps.sh`` defines the ``build_wheel`` function, -which starts up the Manylinux1 Docker container to run a wrapper script -``multibuild/docker_build_wrap.sh``, that (within the container) sources the -following bash scripts:: - - multibuild/common_utils.sh - multibuild/manylinux_utils.sh - env_vars.sh - multibuild/configure_build.sh - multibuild/library_builders.sh - config.sh - -See ``docker_build_wrap.sh`` to review the order of script sourcing. - -See the definition of ``build_multilinux`` in -``multibuild/travis_linux_steps.sh`` for the environment variables passed from -Travis CI to the Manylinux1 container. - -Once in the container, after sourcing the scripts above, the wrapper runs the -real ``build_wheel`` function, which now comes (by default) from -``multibuild/common_utils.sh``. - -Test phase ----------- - -Specify the version to test with the ``DOCKER_TEST_IMAGE`` environment -variable. The default version is dependent on ``MB_ML_LIBC`` and ``PLAT``. - -When ``MB_ML_LIBC`` is ``musllinux``: - -* ``multibuild/alpine3.18_x86_64``, when ``PLAT`` is ``x86_64`` -* ``multibuild/alpine3.18_arm64v8``, when ``PLAT`` is ``aarch64`` - -Otherwise: - -* ``multibuild/focal_x86_64``, when ``PLAT`` is ``x86_64`` -* ``matthewbrett/trusty:32`` when ``PLAT`` is ``i686`` (Yes, an older image for 32-bit) -* ``multibuild/focal_arm64v8`` when ``PLAT`` is ``aarch64`` -* ``multibuild/focal_ppc64le`` when ``PLAT`` is ``ppc64le`` -* ``multibuild/focal_s390x`` when ``PLAT`` is ``s390x`` - -Other valid values are any in https://hub.docker.com/orgs/multibuild/repositories, -using the correct platform code. Alternatively, you can use the substitution -pattern ``multibuild/focal_{PLAT}`` in the ``.travis.yml`` file. - -See ``multibuild/docker_test_wrap.sh``. - -``multibuild/travis_linux_steps.sh`` defines the ``install_run`` function, -which starts up the testing Docker container with the wrapper script -``multibuild/docker_test_wrap.sh``. The wrapper script sources the following -bash scripts:: - - multibuild/common_utils.sh - config.sh - -See ``docker_test_wrap.sh`` for script source order. - -See ``install_run`` in ``multibuild/travis_linux_steps.sh`` for the -environment variables passed into the container. - -It then (in the container) runs the real ``install_run`` command, which comes -(by default) from ``multibuild/common_utils.sh``. - -********************************* -Standard build and test functions -********************************* - -The standard build command is ``build_wheel``. This is a bash function. By -default the function that is run on macOS, and in the Manylinux container for -the build phase, is defined in ``multibuild/common_utils.sh``. You can -override the default function in the project ``config.sh`` file (see below). - -If you are building a wheel from PyPI, rather than from a source repository, -you can use the ``build_index_wheel`` command, again defined in -``multibuild/common_utils.sh``. - -Typically, you can get away with leaving the default ``build_wheel`` / -``build_index_wheel`` functions to do their thing, but you may need to define -a ``pre_build`` function in ``config.sh``. The default ``build_wheel`` and -``build_index_wheel`` functions will call the ``pre_build`` function, if -defined, before building the wheel, so ``pre_build`` is a good place to build -any required libraries. - -The standard test command is the bash function ``install_run``. The version -run on macOS and in the Linux testing container is also defined in -``multibuild/common_utils.sh``. Typically, you do not override this function, -but you in that case you will need to define a ``run_tests`` function, to run -your tests, returning a non-zero error code for failure. The default -``install_run`` implementation calls the ``run_tests`` function, which you -will likely define in ``config.sh``. See the examples below for examples of -less and more complicated builds, where the complicated builds override more -of the default implementations. - -******************** -To use these scripts -******************** - -* Make a repository for building wheels on Travis CI - e.g. - https://github.com/MacPython/astropy-wheels - or in your case maybe - ``https://github.com/your-org/your-project-wheels``; - -* Add this (here) repository as a submodule:: - - git submodule add https://github.com/multi-build/multibuild.git - -* Add your own project repository as another submodule:: - - git submodule add https://github.com/your-org/your-project.git - -* Create a ``.travis.yml`` file, something like this:: - - env: - global: - - REPO_DIR=your-project - # Commit from your-project that you want to build - - BUILD_COMMIT=v0.1.0 - # pip dependencies to _build_ your project - - BUILD_DEPENDS="cython numpy" - # pip dependencies to _test_ your project. Include any dependencies - # that you need, that are also specified in BUILD_DEPENDS, this will be - # a separate install. - # Now see the Uploads section for the stuff you need to - # upload your wheels after CI has built them. - - # You will likely prefer "language: generic" for travis configuration, - # rather than, say "language: python". Multibuild doesn't use - # Travis-provided Python but rather installs and uses its own, where the - # Python version is set from the MB_PYTHON_VERSION variable. You can still - # specify a language here if you need it for some unrelated logic and you - # can't use Multibuild-provided Python or other software present on a - # builder. - language: generic - - # For CPython macOS builds only, the minimum supported macOS version and - # architectures of any C extensions in the wheel are set with the variable - # MB_PYTHON_OSX_VER: 10.9 (64-bit only) or 10.6 (64/32-bit dual arch). By - # default this is set to the highest available for the Python version selected - # using MB_PYTHON_VERSION. You should only need to set this explicitly if you - # are building a 10.6 dual-arch build for a CPython version where both a 10.9 and - # 10.6 build are available (for example, 2.7 or 3.7). - # All PyPy macOS builds are 64-bit only. - - # Required in Linux to invoke `docker` ourselves - services: docker - - # Host distribution. This is the distribution from which we run the build - # and test containers, via docker. - dist: xenial - - # osx image that enables building Apple silicon libraries - osx_image: xcode12.2 - - matrix: - include: - - os: linux - env: MB_PYTHON_VERSION=2.7 - - os: linux - env: - - MB_PYTHON_VERSION=2.7 - - UNICODE_WIDTH=16 - - os: linux - env: - - MB_PYTHON_VERSION=2.7 - - PLAT=i686 - - os: linux - env: - - MB_PYTHON_VERSION=2.7 - - PLAT=i686 - - UNICODE_WIDTH=16 - - os: linux - env: - - MB_PYTHON_VERSION=3.5 - - os: linux - env: - - MB_PYTHON_VERSION=3.5 - - PLAT=i686 - - os: linux - env: - - MB_PYTHON_VERSION=3.6 - - os: linux - env: - - MB_PYTHON_VERSION=3.6 - - PLAT=i686 - - os: osx - env: - - MB_PYTHON_VERSION=2.7 - - MB_PYTHON_OSX_VER=10.6 - - os: osx - env: - - MB_PYTHON_VERSION=2.7 - - os: osx - env: - - MB_PYTHON_VERSION=3.5 - - os: osx - env: - - MB_PYTHON_VERSION=3.6 - - os: osx - env: - - MB_PYTHON_VERSION=3.7 - - MB_PYTHON_OSX_VER=10.6 - - os: osx - env: - - MB_PYTHON_VERSION=3.7 - - os: osx - env: - - MB_PYTHON_VERSION=3.8 - - os: osx - env: - - MB_PYTHON_VERSION=3.9 - - PLAT="universal2" - - os: osx - env: - - MB_PYTHON_VERSION=3.9 - - os: osx - language: generic - env: - - MB_PYTHON_VERSION=pypy-5.7 - - before_install: - - source multibuild/common_utils.sh - - source multibuild/travis_steps.sh - - before_install - - install: - # Maybe get and clean and patch source - - clean_code $REPO_DIR $BUILD_COMMIT - - build_wheel $REPO_DIR $PLAT - - script: - - install_run $PLAT - - after_success: - # Here you should put the code to upload your wheels - # See the Uploads section for more details. - - The example above is for a project building from a Git submodule. If you - aren't building from a submodule, but want to use ``pip`` to build from a - source archive on https://pypi.org or similar, replace the first few lines - of the ``.travis.yml`` file with something like:: - - env: - global: - # Instead of REPO_DIR, BUILD_COMMIT - - PROJECT_SPEC="tornado==4.1.1" - - then your ``install`` section could look something like this:: - - install: - - build_index_wheel $PROJECT_SPEC - - -* Next create a ``config.sh`` for your project, that fills in any steps you - need to do before building the wheel (such as building required libraries). - You also need this file to specify how to run your tests:: - - # Define custom utilities - # Test for macOS with [ -n "$IS_MACOS" ] - - function pre_build { - # Any stuff that you need to do before you start building the wheels - # Runs in the root directory of this repository. - : - } - - function run_tests { - # Runs tests on installed distribution from an empty directory - python --version - python -c 'import sys; import yourpackage; sys.exit(yourpackage.test())' - } - - Optionally you can specify a different location for ``config.sh`` file with - the ``$CONFIG_PATH`` environment variable. - -* Optionally, create an ``env_vars.sh`` file to override the defaults for any - environment variables used by - ``configure_build.sh``/``library_builders.sh``. In Linux, the environment - variables used for the build cannot be set in the ``.travis.yml`` file, - because the build processing runs in a Docker container, so the only - environment variables that reach the container are those passed in via the - ``docker run`` command, or those set in ``env_vars.sh``. - - As for the ``config.sh`` file, you can specify a different location for the - file by setting the ``$ENV_VARS_PATH`` environment variable. The path in - ``$ENV_VARS_PATH`` is relative to the repository root directory. For - example, if your repository had a subdirectory ``scripts`` with a file - ``my_env_vars.sh``, you should set ``ENV_VARS_PATH=scripts/my_env_vars.sh``. - -* Make sure your project is set up to build on Travis CI, and you should now - be ready (to begin the long slow debugging process, probably). - -* For the Windows wheels, create an ``appveyor.yml`` file, something like: - - - https://github.com/MacPython/astropy-wheels/blob/master/appveyor.yml - - https://github.com/MacPython/nipy-wheels/blob/master/appveyor.yml - - https://github.com/MacPython/pytables-wheels/blob/master/appveyor.yml - - Note the Windows test customizations etc are inside ``appveyor.yml``, - and that ``config.sh`` and ``env_vars.sh`` are only for the - Linux/Mac builds on Travis CI. - -* Make sure your project is set up to build on AppVeyor, and you should now - be ready (for what could be another round of slow debugging). - -* For Apple silicon support you can either create an ``arm64`` wheel or - a ``universal2`` wheel by supplying ``PLAT`` env variable. - ``universal2`` builds work on both ``arm64`` and ``x86_64`` platforms - and also make it possible for the wheel code to work when switching the - architecture on Apple silicon machines where ``x86_64`` can be run - using Rosetta2 emulation. - - There are two ways to build ``universal2`` builds. - - 1. Build with ``-arch x86_64 -arch arm64``. - These flags instruct the C/C++ compiler to compile twice and create a - fat object/executable/library. This is the easiest, but has several - drawbacks. If you are using C/C++ libraries that are built using - library_builders, it's highly likely that they don't build correctly - because most build systems and packages don't support building fat binaries. - We could possibly build them separately and fuse them, but the headers might - not be identical which is required when building the wheel as a ``universal2`` - wheel. If you are using Fortran, ``gfortran`` doesn't support fat binaries. - - 2. Build ``arm64`` and ``x86_64`` wheels separately and fuse them. - For this to work, we need to build the C/C++ libraries twice. Therefore, - the library building is once called with ``BUILD_PREFIX=${BUILD_PREFIX:-/usr/local}`` - for ``x86_64`` and then called again with ``BUILD_PREFIX=/opt/arm64-builds``. - Once the two wheels are created, these two are merged. Both the - ``arm64`` and ``universal2`` wheels are outputs for this build. - - In multibuild we are going with option 2. You can override this behaviour by - overriding the function ``wrap_wheel_builder``. - To build Apple silicon builds, you should use a CI service with Xcode 12 with - universal build support and make sure that xcode is the default. - -If your project depends on NumPy, you will want to build against the earliest -NumPy that your project supports - see `forward, backward NumPy compatibility -`_. -See the `astropy-wheels Travis file -`_ for an -example specifying NumPy build and test dependencies. - -Here are some simple example projects: - -* https://github.com/MacPython/astropy-wheels -* https://github.com/scikit-image/scikit-image-wheels -* https://github.com/MacPython/nipy-wheels -* https://github.com/MacPython/dipy-wheels - -Less simple projects where there are some serious build dependencies, and / or -macOS / Linux differences: - -* https://github.com/MacPython/matplotlib-wheels -* https://github.com/python-pillow/Pillow-wheels -* https://github.com/MacPython/h5py-wheels - -********************** -Multibuild development -********************** - -The main multibuild repository is always at -https://github.com/multi-build/multibuild - -We try to keep the ``master`` branch stable and do testing and development -in the ``devel`` branch. From time to time we merge ``devel`` into ``master``. - -In practice, you can check out the newest commit from ``devel`` that works -for you, then stay at it until you need newer features. diff --git a/multibuild.submodules/_gfortran_utils.sh b/multibuild.submodules/_gfortran_utils.sh deleted file mode 100644 index 941e4ff2..00000000 --- a/multibuild.submodules/_gfortran_utils.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Stripped-down version of bash utilities for use with gfortran - -function _mb_get_gf_lib_for_suf { - local suffix=$1 - local prefix=$2 - local plat=${3:-$PLAT} - local uname=${4:-$(uname)} - if [ -z "$prefix" ]; then echo Prefix not defined; exit 1; fi - local fname="$prefix-${uname}-${plat}${suffix}.tar.gz" - local out_fname="${ARCHIVE_SDIR}/$fname" - if [ ! -e "$out_fname" ]; then - curl -L "${GF_LIB_URL}/$fname" > $out_fname || (echo "Fetch failed"; exit 1) - fi - echo "$out_fname" -} - -function _mb_get_gf_lib { - # Get library with no suffix - _mb_get_gf_lib_for_suf "" $@ -} diff --git a/multibuild.submodules/common_utils.sh b/multibuild.submodules/common_utils.sh deleted file mode 100755 index 50049110..00000000 --- a/multibuild.submodules/common_utils.sh +++ /dev/null @@ -1,713 +0,0 @@ -#!/bin/bash -# Utilities for both OSX and Docker Linux -# python or python3 should be on the PATH - -# Only source common_utils once -if [ -n "$COMMON_UTILS_SOURCED" ]; then - return -fi -COMMON_UTILS_SOURCED=1 - -# Turn on exit-if-error -set -e - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -DOWNLOADS_SDIR=downloads -PYPY_URL=https://downloads.python.org/pypy -# For back-compatibility. We use the "ensurepip" module now -# instead of get-pip.py -GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py - -# Unicode width, default 32. Used here and in travis_linux_steps.sh -# In docker_build_wrap.sh it is passed in when calling "docker run" -# The docker test images also use it when choosing the python to run -# with, so it is passed in when calling "docker run" for tests. -UNICODE_WIDTH=${UNICODE_WIDTH:-32} - -if [ $(uname) == "Darwin" ]; then - IS_MACOS=1; IS_OSX=1; -else - # In the manylinux_2_24 image, based on Debian9, "python" is not installed - # so link in something for the various system calls before PYTHON_EXE is set - which python || export PATH=/opt/python/cp39-cp39/bin:$PATH -fi - -if [ "$MB_ML_LIBC" == "musllinux" ]; then - IS_ALPINE=1; - MB_ML_VER=${MB_ML_VER:-"_1_1"} -else - # Default Manylinux version - MB_ML_VER=${MB_ML_VER:-2014} -fi - -# Work round bug in travis xcode image described at -# https://github.com/direnv/direnv/issues/210 -shell_session_update() { :; } - -# Workaround for https://github.com/travis-ci/travis-ci/issues/8703 -# suggested by Thomas K at -# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274 -unset -f cd -unset -f pushd -unset -f popd - -function start_spinner { - if [ -n "$MB_SPINNER_PID" ]; then - return - fi - - >&2 echo "Building libraries..." - # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output - (while true; do - sleep 60 - >&2 echo "Still building..." - done) & - MB_SPINNER_PID=$! - disown -} - -function stop_spinner { - if [ ! -n "$MB_SPINNER_PID" ]; then - return - fi - - kill $MB_SPINNER_PID - unset MB_SPINNER_PID - - >&2 echo "Building libraries finished." -} - -function any_python { - for cmd in $PYTHON_EXE python3 python; do - if [ -n "$(type -t $cmd)" ]; then - echo $cmd - return - fi - done - echo "Could not find python or python3" - exit 1 -} - -function abspath { - # Can work with any Python; need not be our installed Python. - $(any_python) -c "import os.path; print(os.path.abspath('$1'))" -} - -function relpath { - # Path of first input relative to second (or $PWD if not specified) - # Can work with any Python; need not be our installed Python. - $(any_python) -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))" -} - -function realpath { - # Can work with any Python; need not be our installed Python. - $(any_python) -c "import os; print(os.path.realpath('$1'))" -} - -function lex_ver { - # Echoes dot-separated version string padded with zeros - # Thus: - # 3.2.1 -> 003002001 - # 3 -> 003000000 - echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' -} - -function unlex_ver { - # Reverses lex_ver to produce major.minor.micro - # Thus: - # 003002001 -> 3.2.1 - # 003000000 -> 3.0.0 - echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))" -} - -function strip_ver_suffix { - echo $(unlex_ver $(lex_ver $1)) -} - -function is_function { - # Echo "true" if input argument string is a function - # Allow errors during "set -e" blocks. - (set +e; $(declare -Ff "$1" > /dev/null) && echo true) -} - -function gh_clone { - git clone https://github.com/$1 -} - -# gh-clone was renamed to gh_clone, so we have this alias for -# backwards compatibility. -alias gh-clone=gh_clone - -function set_opts { - # Set options from input options string (in $- format). - local opts=$1 - local chars="exhmBH" - for (( i=0; i<${#chars}; i++ )); do - char=${chars:$i:1} - [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char - done -} - -function suppress { - # Run a command, show output only if return code not 0. - # Takes into account state of -e option. - # Compare - # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122 - # Set -e stuff agonized over in - # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell - local tmp=$(mktemp tmp.XXXXXXXXX) || return - local errexit_set - echo "Running $@" - if [[ $- = *e* ]]; then errexit_set=true; fi - set +e - ( if [[ -n $errexit_set ]]; then set -e; fi; "$@" > "$tmp" 2>&1 ) ; ret=$? - [ "$ret" -eq 0 ] || cat "$tmp" - rm -f "$tmp" - if [[ -n $errexit_set ]]; then set -e; fi - return "$ret" -} - -function expect_return { - # Run a command, succeeding (returning 0) only if the commend returns a specified code - # Parameters - # retcode expected return code (which may be zero) - # command the command called - # - # any further arguments are passed to the called command - # - # Returns 1 if called with less than 2 arguments - (( $# < 2 )) && echo "Must have at least 2 arguments" && return 1 - local retcode=$1 - local retval - ( "${@:2}" ) || retval=$? - [[ $retcode == ${retval:-0} ]] && return 0 - return ${retval:-1} -} - -function cmd_notexit { - # wraps a command, capturing its return code and preventing it - # from exiting the shell. Handles -e / +e modes. - # Parameters - # cmd - command - # any further parameters are passed to the wrapped command - # If called without an argument, it will exit the shell with an error - local cmd=$1 - if [ -z "$cmd" ];then echo "no command"; exit 1; fi - if [[ $- = *e* ]]; then errexit_set=true; fi - set +e - ("${@:1}") ; retval=$? - [[ -n $errexit_set ]] && set -e - return $retval -} - -function rm_mkdir { - # Remove directory if present, then make directory - local path=$1 - if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi - if [ -d "$path" ]; then rm -rf $path; fi - mkdir $path -} - -function untar { - local in_fname=$1 - if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi - local extension=${in_fname##*.} - case $extension in - tar) tar -xf $in_fname ;; - gz|tgz) tar -zxf $in_fname ;; - bz2) tar -jxf $in_fname ;; - zip) unzip -qq $in_fname ;; - xz) if [ -n "$IS_MACOS" ]; then - tar -xf $in_fname - else - if [[ ! $(type -P "unxz") ]]; then - echo xz must be installed to uncompress file; exit 1 - fi - unxz -c $in_fname | tar -xf - - fi ;; - *) echo Did not recognize extension $extension; exit 1 ;; - esac -} - -function install_rsync { - # install rsync via package manager - if [ -n "$IS_MACOS" ]; then - # macOS. The colon in the next line is the null command - : - elif [ -n "$IS_ALPINE" ]; then - [[ $(type -P rsync) ]] || apk add rsync - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - [[ $(type -P rsync) ]] || apt-get install -y rsync - else - # centos based distro - [[ $(type -P rsync) ]] || yum_install rsync - fi -} - -function fetch_unpack { - # Fetch input archive name from input URL - # Parameters - # url - URL from which to fetch archive - # archive_fname (optional) archive name - # - # Echos unpacked directory and file names. - # - # If `archive_fname` not specified then use basename from `url` - # If `archive_fname` already present at download location, use that instead. - local url=$1 - if [ -z "$url" ];then echo "url not defined"; exit 1; fi - local archive_fname=${2:-$(basename $url)} - local arch_sdir="${ARCHIVE_SDIR:-archives}" - if [ -z "$IS_MACOS" ]; then - local extension=${archive_fname##*.} - if [ "$extension" == "xz" ]; then - ensure_xz - fi - fi - # Make the archive directory in case it doesn't exist - mkdir -p $arch_sdir - local out_archive="${arch_sdir}/${archive_fname}" - # If the archive is not already in the archives directory, get it. - if [ ! -f "$out_archive" ]; then - # Source it from multibuild archives if available. - local our_archive="${MULTIBUILD_DIR}/archives/${archive_fname}" - if [ -f "$our_archive" ]; then - ln -s $our_archive $out_archive - else - # Otherwise download it. - curl -L $url > $out_archive - fi - fi - # Unpack archive, refreshing contents, echoing dir and file - # names. - rm_mkdir arch_tmp - install_rsync - (cd arch_tmp && \ - untar ../$out_archive && \ - ls -1d * && - rsync --delete -ah * ..) -} - -function clean_code { - local repo_dir=${1:-$REPO_DIR} - local build_commit=${2:-$BUILD_COMMIT} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 - # The package $repo_dir may be a submodule. git submodules do not - # have a .git directory. If $repo_dir is copied around, tools like - # Versioneer which require that it be a git repository are unable - # to determine the version. Give submodule proper git directory - fill_submodule "$repo_dir" - (cd $repo_dir \ - && git fetch origin --tags \ - && git checkout $build_commit \ - && git clean -fxd \ - && git reset --hard \ - && git submodule update --init --recursive) -} - -function build_wheel_cmd { - # Builds wheel with named command, puts into $WHEEL_SDIR - # - # Parameters: - # cmd (optional, default "pip_wheel_cmd" - # Name of command for building wheel - # repo_dir (optional, default $REPO_DIR) - # - # Depends on - # REPO_DIR (or via input argument) - # WHEEL_SDIR (optional, default "wheelhouse") - # BUILD_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local cmd=${1:-pip_wheel_cmd} - local repo_dir=${2:-$REPO_DIR} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - mkdir -p "$wheelhouse" - start_spinner - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - stop_spinner - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $BUILD_DEPENDS - fi - (cd $repo_dir && $cmd $wheelhouse) - repair_wheelhouse $wheelhouse -} - -function pip_wheel_cmd { - local abs_wheelhouse=$1 - pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . -} - -function bdist_wheel_cmd { - # Builds wheel with bdist_wheel, puts into wheelhouse - # - # It may sometimes be useful to use bdist_wheel for the wheel building - # process. For example, versioneer has problems with versions which are - # fixed with bdist_wheel: - # https://github.com/warner/python-versioneer/issues/121 - local abs_wheelhouse=$1 - check_python - $PYTHON_EXE setup.py bdist_wheel - cp dist/*.whl $abs_wheelhouse -} - -function wrap_wheel_builder { - # Wrapper for build commands, overwritten by macOS for universal2 or arm64 wheel building - $@ -} - -function build_pip_wheel { - # Standard wheel building command with pip wheel - wrap_wheel_builder build_wheel_cmd "pip_wheel_cmd" $@ -} - -function build_bdist_wheel { - # Wheel building with bdist_wheel. See bdist_wheel_cmd - wrap_wheel_builder build_wheel_cmd "bdist_wheel_cmd" $@ -} - -function build_wheel { - # Set default building method to pip - wrap_wheel_builder build_pip_wheel $@ -} - -function build_index_wheel_cmd { - # Builds wheel from some index, usually pypi - # - # Parameters: - # project_spec - # requirement to install, e.g. "tornado" or "tornado==4.4.1" - # *args - # Any other arguments to be passed to pip `install` and `wheel` - # commands. - # - # Depends on - # WHEEL_SDIR (optional, default "wheelhouse") - # BUILD_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - # - # You can also override `pip_opts` command to set indices other than pypi - local project_spec=$1 - [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1 - # Discard first argument to pass remainder to pip - shift - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - start_spinner - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - stop_spinner - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $@ $BUILD_DEPENDS - fi - pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec - repair_wheelhouse $wheelhouse -} - -function build_index_wheel { - wrap_wheel_builder build_index_wheel_cmd $@ -} - -function pip_opts { - [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL" -} - -function get_os { - # Report OS as given by uname - # Use any Python that comes to hand. - $(any_python) -c 'import platform; print(platform.uname()[0])' -} - -function get_platform { - # Report platform as given by uname - # Use any Python that comes to hand. - $(any_python) -c 'import platform; print(platform.uname()[4])' -} - -if [ "$(get_platform)" == x86_64 ] || \ - [ "$(get_platform)" == i686 ]; then IS_X86=1; fi - -function get_distutils_platform { - # Report platform as given by distutils get_platform. - # This is the platform tag that pip will use. - check_python - $PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())" -} - -function install_wheel { - # Install test dependencies and built wheel - # - # Pass any input flags to pip install steps - # - # Depends on: - # WHEEL_SDIR (optional, default "wheelhouse") - # TEST_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - check_pip - if [ -n "$TEST_DEPENDS" ]; then - while read TEST_DEPENDENCY; do - $PIP_CMD install $(pip_opts) $@ $TEST_DEPENDENCY - done <<< "$TEST_DEPENDS" - fi - - check_python - check_pip - - $PIP_CMD install packaging wheel - local supported_wheels=$($PYTHON_EXE $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl) - if [ -z "$supported_wheels" ]; then - echo "ERROR: no supported wheels found" - exit 1 - fi - # Install compatible wheel - $PIP_CMD install $(pip_opts) $@ $supported_wheels -} - -function install_run { - # Depends on function `run_tests` defined in `config.sh` - install_wheel - mkdir tmp_for_test - (cd tmp_for_test && run_tests) - rmdir tmp_for_test 2>/dev/null || echo "Cannot remove tmp_for_test" -} - -function fill_submodule { - # Restores .git directory to submodule, if necessary - # See: - # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule - local repo_dir="$1" - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local git_loc="$repo_dir/.git" - # For ordinary submodule, .git is a file. - [ -d "$git_loc" ] && return - # Need to recreate .git directory for submodule - local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url) - local repo_copy="$repo_dir-$RANDOM" - git clone --recursive "$repo_dir" "$repo_copy" - rm -rf "$repo_dir" - mv "${repo_copy}" "$repo_dir" - (cd "$repo_dir" && git remote set-url origin $origin_url) -} - -# The latest versions of PyPy. -LATEST_PP_5p0=5.0.1 -LATEST_PP_5p1=5.1.1 -LATEST_PP_5p3=5.3.1 -LATEST_PP_5p4=5.4.1 -LATEST_PP_5p6=5.6.0 -LATEST_PP_5p7=5.7.1 -LATEST_PP_5p8=5.8.0 -LATEST_PP_5p9=5.9.0 -LATEST_PP_5=$LATEST_PP_5p9 - -LATEST_PP_6p0=6.0.0 -LATEST_PP_6=$LATEST_PP_6p0 - -LATEST_PP_7p0=7.0.0 -LATEST_PP_7p1=7.1.1 -LATEST_PP_7p2=7.2.0 -LATEST_PP_7p3=7.3.12 -LATEST_PP_7=$LATEST_PP_7p3 - -function unroll_version { - # Convert major or major.minor format to major.minor.micro using the above - # values recursively - # Parameters: - # $prefix : one of LATEST_PP or LATEST_PP3 - # $version : major[.minor[.patch]] - # Hence: - # LATEST_PP 5 -> 5.7.0 - # LATEST 2.7 -> 2.7.11 - local prefix=$1 - local ver=$2 - local latest=${prefix}_${ver//./p} - if [ -n "${!latest}" ]; then - echo $(unroll_version ${prefix} ${!latest}) - else - echo $ver - fi -} - -function install_pypy { - # Installs pypy.org PyPy - # Parameter $version - # Version given in major or major.minor or major.minor.micro e.g - # "3" or "3.7" or "3.7.1". - # Uses $PLAT - # sets $PYTHON_EXE variable to python executable - - local version=$1 - # Need to convert pypy-7.2 to pypy2.7-v7.2.0 and pypy3.6-7.3 to pypy3.6-v7.3.0 - local prefix=$(get_pypy_build_prefix $version) - # since prefix is pypy3.6v7.2 or pypy2.7v7.2, grab the 4th (0-index) letter - local major=${prefix:4:1} - # get the pypy version 7.2.0 - local py_version=$(fill_pypy_ver $(echo $version | cut -f2 -d-)) - - case "$PLAT" in - "x86_64") if [ -n "$IS_MACOS" ]; then - if [ $(lex_ver $py_version) -ge $(lex_ver 7.3.10) ]; then - suffix="macos_x86_64"; - else - suffix="osx64"; - fi - else - suffix="linux64"; - fi;; - "i686") suffix="linux32";; - "ppc64le") suffix="ppc64le";; - "s390x") suffix="s390x";; - "aarch64") suffix="aarch64";; - *) echo unknown platform "$PLAT"; exit 1;; - esac - - local py_build=$prefix$py_version-$suffix - local py_zip=$py_build.tar.bz2 - local zip_path=$DOWNLOADS_SDIR/$py_zip - mkdir -p $DOWNLOADS_SDIR - wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR - untar $zip_path - # bug/feature: pypy package for pypy3 only has bin/pypy3 :( - if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pypy" ]; then - ln $py_build/bin/pypy3 $py_build/bin/pypy - fi - PYTHON_EXE=$(realpath $py_build/bin/pypy) - $PYTHON_EXE -mensurepip - $PYTHON_EXE -mpip install --upgrade pip setuptools wheel - if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pip" ]; then - ln $py_build/bin/pip3 $py_build/bin/pip - fi - PIP_CMD=pip -} - -function fill_pypy_ver { - # Convert major or major.minor format to major.minor.micro - # Parameters: - # $version : major[.minor[.patch]] - # Hence: - # 5 -> 5.7.0 - echo $(unroll_version LATEST_PP $1) -} - -function get_pypy_build_prefix { - # Return the file prefix of a PyPy file - # Parameters: - # $version : pypy version number, for example pypy-7.2 or pypy3.6-7.2 - local version=$1 - if [[ $version =~ pypy([0-9]+)\.([0-9]+)-([0-9]+)\.([0-9]+) ]]; then - local py_major=${BASH_REMATCH[1]} - local py_minor=${BASH_REMATCH[2]} - echo "pypy$py_major.$py_minor-v" - elif [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then - local major=${BASH_REMATCH[1]} - local minor=${BASH_REMATCH[2]} - if (( $major > 6 )); then - echo "pypy2.7-v" - elif (( $major > 5 || ($major == 5 && $minor >= 3) )); then - echo "pypy2-v" - else - echo "pypy-" - fi - else - echo "error: expected version like pypy-7.2 or pypy3.6-7.2, got $1" 1>&2 - exit 1 - fi -} - -retry () { - # Retry command (with arguments) up to 5 times - # https://gist.github.com/fungusakafungus/1026804 - local retry_max=5 - local count=$retry_max - while [ $count -gt 0 ]; do - "$@" && break - count=$(($count - 1)) - sleep 1 - done - - [ $count -eq 0 ] && { - echo "Retry failed [$retry_max]: $@" >&2 - return 1 - } - return 0 -} - -function install_pip { - # Generic install pip - echo "Deprecated - please see pip installs within the individual" - echo "install functions for each Python type." - echo "Multibuild itself no longer uses this function." - # Gets needed version from version implied by $PYTHON_EXE - # Installs pip into python given by $PYTHON_EXE - # Assumes pip will be installed into same directory as $PYTHON_EXE - check_python - mkdir -p $DOWNLOADS_SDIR - local py_mm=`get_py_mm` - local get_pip_path=$DOWNLOADS_SDIR/get-pip.py - curl $GET_PIP_URL > $get_pip_path - # Travis VMS now install pip for system python by default - force install - # even if installed already. - $PYTHON_EXE $get_pip_path --ignore-installed $pip_args - PIP_CMD=$(dirname $PYTHON_EXE)/pip$py_mm - if [ "$USER" != "root" ]; then - # inside a docker, there is no sudo but the user is already root - PIP_CMD="sudo $PIP_CMD" - fi - # Append pip_args if present (avoiding trailing space cf using variable - # above). - if [ -n "$pip_args" ]; then - PIP_CMD="$PIP_CMD $pip_args" - fi -} - -function check_python { - if [ -z "$PYTHON_EXE" ]; then - echo "PYTHON_EXE variable not defined" - exit 1 - fi -} - -function check_pip { - if [ -z "$PIP_CMD" ]; then - echo "PIP_CMD variable not defined" - exit 1 - fi -} - -function get_py_mm { - check_python - $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))" -} - -function cpython_path { - # Return path to cpython given - # * version (of form "2.7") - # * u_width ("16" or "32" default "32") - # - # For back-compatibility "u" as u_width also means "32" - local py_ver="${1:-2.7}" - local abi_suff=m - local u_width="${2:-${UNICODE_WIDTH}}" - local u_suff=u - # Python 3.8 and up no longer uses the PYMALLOC 'm' suffix - # https://github.com/pypa/wheel/pull/303 - if [ $(lex_ver $py_ver) -ge $(lex_ver 3.8) ]; then - abi_suff="" - fi - # Back-compatibility - if [ "$u_width" == "u" ]; then u_width=32; fi - # For Python >= 3.4, "u" suffix not meaningful - if [ $(lex_ver $py_ver) -ge $(lex_ver 3.4) ] || - [ "$u_width" == "16" ]; then - u_suff="" - elif [ "$u_width" == "" ]; then - u_width="32" - elif [ "$u_width" != "32" ]; then - echo "Incorrect u_width value $u_width" - exit 1 - fi - local no_dots=$(echo $py_ver | tr -d .) - echo "/opt/python/cp${no_dots}-cp${no_dots}$abi_suff${u_suff}" -} diff --git a/multibuild.submodules/configure_build.sh b/multibuild.submodules/configure_build.sh deleted file mode 100644 index f505b782..00000000 --- a/multibuild.submodules/configure_build.sh +++ /dev/null @@ -1,83 +0,0 @@ -# Find, load common utilities -# Defines IS_MACOS, fetch_unpack -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/common_utils.sh - -# Only source configure_build once -if [ -n "$CONFIGURE_BUILD_SOURCED" ]; then - return -fi -CONFIGURE_BUILD_SOURCED=1 - -BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" - -# IS_MACOS is defined in common_utils.sh -if [ -n "$IS_MACOS" ]; then - # Default compilation flags for OSX - source $MULTIBUILD_DIR/osx_utils.sh - PLAT=${PLAT:-$(macpython_arch_for_version $MB_PYTHON_VERSION)} - if [[ $PLAT == intel ]]; then - ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"} - elif [[ $PLAT == x86_64 ]]; then - ARCH_FLAGS=${ARCH_FLAGS:-"-arch x86_64"} - elif [[ $PLAT == arm64 ]]; then - ARCH_FLAGS=${ARCH_FLAGS:-"-arch arm64"} - elif [[ $PLAT == universal2 ]]; then - # Do nothing as we are going with fusing wheels - ARCH_FLAGS=${ARCH_FLAGS:-} - else - echo "Invalid platform = '$PLAT'. Supported values are 'intel', 'x86_64', 'arm64' or 'universal2'" - exit 1 - fi - # Only set CFLAGS, FFLAGS if they are not already defined. Build functions - # can override the arch flags by setting CFLAGS, FFLAGS - export CFLAGS="${CFLAGS:-$ARCH_FLAGS}" - export CXXFLAGS="${CXXFLAGS:-$ARCH_FLAGS}" - export FFLAGS="${FFLAGS:-$ARCH_FLAGS}" - - # Disable homebrew auto-update - export HOMEBREW_NO_AUTO_UPDATE=1 -else - # default compilation flags for linux - PLAT="${PLAT:-x86_64}" - # Strip all binaries after compilation. - STRIP_FLAGS=${STRIP_FLAGS:-"-Wl,-strip-all"} - - export CFLAGS="${CFLAGS:-$STRIP_FLAGS}" - export CXXFLAGS="${CXXFLAGS:-$STRIP_FLAGS}" - export FFLAGS="${FFLAGS:-$STRIP_FLAGS}" - if [[ $MB_ML_VER == "_2_24" ]]; then - # This is the first opportunity to distinguish between manylinuxes - apt update - if [ "${MB_PYTHON_VERSION:0:4}" == "pypy" ]; then - # debian:9 based distro - apt install -y wget - fi - elif [[ $MB_ML_VER == "1" ]]; then - # Need libtool, and for pypy need wget - # centos based distro - yum install -y libtool wget - elif [ "${MB_PYTHON_VERSION:0:4}" == "pypy" ]; then - if [ -n "$IS_ALPINE" ]; then - apk add wget - else - # centos based distro - yum install -y wget - fi - fi -fi - -export CPPFLAGS_BACKUP="$CPPFLAGS" -export LIBRARY_PATH_BACKUP="$LIBRARY_PATH" -export PKG_CONFIG_PATH_BACKUP="$PKG_CONFIG_PATH" - -function update_env_for_build_prefix { - # Promote BUILD_PREFIX on search path to any newly built libs - export CPPFLAGS="-I$BUILD_PREFIX/include $CPPFLAGS_BACKUP" - export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH_BACKUP" - export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH_BACKUP" - # Add binary path for configure utils etc - export PATH="$BUILD_PREFIX/bin:$PATH" -} - -update_env_for_build_prefix diff --git a/multibuild.submodules/docker_build_wrap.sh b/multibuild.submodules/docker_build_wrap.sh deleted file mode 100755 index 4628fda2..00000000 --- a/multibuild.submodules/docker_build_wrap.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Depends on: -# BUILD_COMMANDS -# PYTHON_VERSION -# CONFIG_PATH (can be empty) -# BUILD_COMMIT (may be used by config.sh) -# UNICODE_WIDTH (can be empty) -# BUILD_DEPENDS (may be used by config.sh, can be empty) -set -e - -# Change into root directory of repo -cd /io - -# Location of wheels, default "wheelhouse" -WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse} - -# Location of `config.sh` file, default "./config.sh" -CONFIG_PATH=${CONFIG_PATH:-config.sh} - -# Path is relative to repository from which we ran -ENV_VARS_PATH=${ENV_VARS_PATH:-env_vars.sh} - -# Always pull in common and library builder utils -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -# These routines also source common_utils.sh -source $MULTIBUILD_DIR/manylinux_utils.sh -if [ -r "$ENV_VARS_PATH" ]; then source "$ENV_VARS_PATH"; fi -source $MULTIBUILD_DIR/configure_build.sh -source $MULTIBUILD_DIR/library_builders.sh - -if [ "$USE_CCACHE" == "1" ]; then - activate_ccache -fi - -# The following also sets PYTHON_EXE and PIP_CMD -if [ "${PYTHON_VERSION:0:4}" == "pypy" ]; then - install_pypy $PYTHON_VERSION - export PATH=$(dirname $PYTHON_EXE):$PATH -else - # Set PATH for chosen Python, Unicode width - PYTHON_EXE=$(cpython_path $PYTHON_VERSION $UNICODE_WIDTH)/bin/python - ls $(dirname $PYTHON_EXE) - export PATH="$(dirname $PYTHON_EXE):$PATH" - # We can assume ensurepip is available and up to date. - $PYTHON_EXE -m ensurepip - PIP_CMD="$PYTHON_EXE -m pip" -fi - -# Configuration for this package, possibly overriding `build_wheel` defined in -# `common_utils.sh` via `manylinux_utils.sh`. -source "$CONFIG_PATH" - -$BUILD_COMMANDS diff --git a/multibuild.submodules/docker_test_wrap.sh b/multibuild.submodules/docker_test_wrap.sh deleted file mode 100755 index fa46de78..00000000 --- a/multibuild.submodules/docker_test_wrap.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Install and test steps on Linux -set -e - -# "python" and "pip" are already on the path as part of the docker -# startup code in choose_python.sh, but the following are required and not -# necessarily already set - -PYTHON_EXE=${PYTHON_EXE:-python} -PIP_CMD=${PIP_CMD:-pip} - -# Get needed utilities -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/common_utils.sh - -# Change into root directory of repo -cd /io - -# Configuration for this package in `config.sh`. -# This can overwrite `install_run` and `install_wheel` (called from -# `install_run`). These are otherwise defined in common_utils.sh. -# `config.sh` must define `run_tests` if using the default `install_run`. -CONFIG_PATH=${CONFIG_PATH:-config.sh} -source "$CONFIG_PATH" - -install_run diff --git a/multibuild.submodules/install_python.ps1 b/multibuild.submodules/install_python.ps1 deleted file mode 100644 index 6a87a0b0..00000000 --- a/multibuild.submodules/install_python.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -# Install specified Python version. -# Install only if: -# Our current matrix entry uses this Python version AND -# Python version is not already available. - -$py_exe = "${env:PYTHON}\Python.exe" -if ( [System.IO.File]::Exists($py_exe) ) { - echo "$py_exe exists" - exit 0 -} -$req_nodot = $env:PYTHON -replace '\D+Python(\d+(?:rc\d+)?)(-x64)?','$1' -$req_ver = $req_nodot -replace '(\d)(\d+)','$1.$2.0' -$req_dir = $req_nodot -replace '(\d)(\d+)(.*)','$1.$2.0' -$last_three = $env:PYTHON[-3 .. -1] -join '' - -if ($last_three -eq "x64") { - $exe_suffix="-amd64" -} else { - $exe_suffix="" -} - -$py_url = "https://www.python.org/ftp/python" -Write-Host "Installing Python ${req_dir}/${req_ver}$exe_suffix to $env:Python ..." -ForegroundColor Cyan -$exePath = "$env:TEMP\python-${req_ver}${exe_suffix}.exe" -$downloadFile = "$py_url/${req_dir}/python-${req_ver}${exe_suffix}.exe" - -Write-Host "Downloading $downloadFile..." -(New-Object Net.WebClient).DownloadFile($downloadFile, $exePath) -Write-Host "Installing..." -cmd /c start /wait $exePath /quiet TargetDir="$env:PYTHON" Shortcuts=0 Include_launcher=0 InstallLauncherAllUsers=0 -Write-Host "Python ${req_ver} installed to $env:PYTHON" - -echo "$(& $py_exe --version 2> $null)" diff --git a/multibuild.submodules/library_builders.sh b/multibuild.submodules/library_builders.sh deleted file mode 100644 index 1dbe4b60..00000000 --- a/multibuild.submodules/library_builders.sh +++ /dev/null @@ -1,550 +0,0 @@ -#Functions and environment variables to build various -#native libraries commonly used as dependencies - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/_gfortran_utils.sh -source $MULTIBUILD_DIR/configure_build.sh - -GF_LIB_URL="https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com" -# For OpenBLAS -OPENBLAS_LIB_URL="https://anaconda.org/multibuild-wheels-staging/openblas-libs" - -# Recipes for building some libraries -OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.3.10}" -# We use system zlib by default - see build_new_zlib -ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}" -LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.37}" -BZIP2_VERSION="${BZIP2_VERSION:-1.0.7}" -FREETYPE_VERSION="${FREETYPE_VERSION:-2.11.0}" -TIFF_VERSION="${TIFF_VERSION:-4.1.0}" -JPEG_VERSION="${JPEG_VERSION:-9b}" -JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-2.1.3}" -OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}" -LCMS2_VERSION="${LCMS2_VERSION:-2.9}" -GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}" -LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}" -XZ_VERSION="${XZ_VERSION:-5.2.2}" -LIBYAML_VERSION="${LIBYAML_VERSION:-0.2.2}" -SZIP_VERSION="${SZIP_VERSION:-2.1.1}" -HDF5_VERSION="${HDF5_VERSION:-1.10.5}" -LIBAEC_VERSION="${LIBAEC_VERSION:-1.0.4}" -LZO_VERSION=${LZO_VERSION:-2.10} -LZF_VERSION="${LZF_VERSION:-3.6}" -BLOSC_VERSION=${BLOSC_VERSION:-1.10.2} -SNAPPY_VERSION="${SNAPPY_VERSION:-1.1.3}" -CURL_VERSION=${CURL_VERSION:-7.49.1} -NETCDF_VERSION=${NETCDF_VERSION:-4.4.1.1} -SWIG_VERSION=${SWIG_VERSION:-4.0.1} -PCRE_VERSION=${PCRE_VERSION:-8.38} -SUITESPARSE_VERSION=${SUITESPARSE_VERSION:-4.5.6} -LIBTOOL_VERSION=${LIBTOOL_VERSION:-2.4.6} -RAGEL_VERSION=${RAGEL_VERSION:-6.10} -FLEX_VERSION=${FLEX_VERSION:-2.6.4} -BISON_VERSION=${BISON_VERSION:-3.0.4} -FFTW_VERSION=${FFTW_VERSION:-3.3.7} -CFITSIO_VERSION=${CFITSIO_VERSION:-3450} -OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1l} -# Hash from https://www.openssl.org/source/openssl-1.1.1?.tar.gz.sha256 -OPENSSL_HASH=${OPENSSL_HASH:-0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1} -OPENSSL_DOWNLOAD_URL=${OPENSSL_DOWNLOAD_URL:-https://www.openssl.org/source} - - -ARCHIVE_SDIR=${ARCHIVE_DIR:-archives} - - -function build_simple { - # Example: build_simple libpng $LIBPNG_VERSION \ - # https://download.sourceforge.net/libpng tar.gz \ - # --additional --configure --arguments - local name=$1 - local version=$2 - local url=$3 - local ext=${4:-tar.gz} - local configure_args=${@:5} - if [ -e "${name}-stamp" ]; then - return - fi - local name_version="${name}-${version}" - local archive=${name_version}.${ext} - fetch_unpack $url/$archive - (cd $name_version \ - && ./configure --prefix=$BUILD_PREFIX $configure_args \ - && make -j4 \ - && make install) - touch "${name}-stamp" -} - -function build_github { - # Example: build_github fredrik-johansson/arb 2.11.1 - local path=$1 - local tag_name=$2 - local configure_args=${@:3} - local name=`basename "$path"` - if [ -e "${name}-stamp" ]; then - return - fi - local out_dir=$(fetch_unpack "https://github.com/${path}/archive/${tag_name}.tar.gz") - (cd $out_dir \ - && ./configure --prefix=$BUILD_PREFIX $configure_args \ - && make -j4 \ - && make install) - touch "${name}-stamp" -} - -function openblas_get { - # Get an openblas compiled library from - # https://anaconda.org/multibuild-wheels-staging/openblas-libs - # The general form of the link is (under the URL above) - # URL/v0.3.10/downloads/openblas-v0.3.10-manylinux2010_x86_64.tar.gz - local plat=${1:-$} - # qual could be 64 to get a 64-bit version - local qual=$2 - local prefix=openblas${qual}-v$OPENBLAS_VERSION - local manylinux=manylinux${MB_ML_VER} - local fname="$prefix-${manylinux}_${plat}.tar.gz" - local out_fname="${ARCHIVE_SDIR}/$fname" - if [ ! -e "$out_fname" ]; then - local webname=${OPENBLAS_LIB_URL}/v${OPENBLAS_VERSION}/download/${fname} - curl -L "$webname" > $out_fname || exit 1 - # make sure it is not an HTML document of download failure - local ok=$(file $out_fname | grep "HTML document") - if [ -n "$ok" ]; then - echo Fetching "${OPENBLAS_LIB_URL}/$fname" failed; - exit 1; - fi - fi - echo "$out_fname" -} - -function build_openblas { - if [ -e openblas-stamp ]; then return; fi - if [ -n "$IS_MACOS" ]; then - brew install openblas - brew link --force openblas - else - mkdir -p $ARCHIVE_SDIR - local plat=${1:-${PLAT:-x86_64}} - local tar_path=$(abspath $(openblas_get $plat)) - (cd / && tar zxf $tar_path) - fi - touch openblas-stamp -} - -function build_zlib { - # Gives an old but safe version - if [ -n "$IS_MACOS" ]; then return; fi # OSX has zlib already - if [ -e zlib-stamp ]; then return; fi - if [ -n "$IS_ALPINE" ]; then - apk add zlib-dev - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - apt-get install -y zlib1g-dev - else - #centos based distro - yum_install zlib-devel - fi - touch zlib-stamp -} - -function build_new_zlib { - # Careful, this one may cause yum to segfault - # Fossils directory should also contain latest - build_simple zlib $ZLIB_VERSION https://zlib.net/fossils -} - -function build_jpeg { - if [ -e jpeg-stamp ]; then return; fi - fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz - (cd jpeg-${JPEG_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX \ - && make -j4 \ - && make install) - touch jpeg-stamp -} - -function build_libjpeg_turbo { - if [ -e jpeg-stamp ]; then return; fi - local cmake=$(get_modern_cmake) - fetch_unpack https://download.sourceforge.net/libjpeg-turbo/libjpeg-turbo-${JPEGTURBO_VERSION}.tar.gz - (cd libjpeg-turbo-${JPEGTURBO_VERSION} \ - && $cmake -G"Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib . \ - && make install) - - # Prevent build_jpeg - touch jpeg-stamp -} - -function build_libpng { - build_zlib - build_simple libpng $LIBPNG_VERSION https://download.sourceforge.net/libpng -} - -function build_bzip2 { - if [ -n "$IS_MACOS" ]; then return; fi # OSX has bzip2 libs already - if [ -e bzip2-stamp ]; then return; fi - fetch_unpack https://mirrors.kernel.org/sourceware/bzip2/bzip2-${BZIP2_VERSION}.tar.gz - (cd bzip2-${BZIP2_VERSION} \ - && make -f Makefile-libbz2_so \ - && make install PREFIX=$BUILD_PREFIX) - touch bzip2-stamp -} - -function build_tiff { - build_zlib - build_jpeg - ensure_xz - build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff -} - -function get_modern_cmake { - # Install cmake >= 2.8 - local cmake=cmake - if [ -n "$IS_MACOS" ]; then - brew install cmake > /dev/null - elif [ -n "$IS_ALPINE" ]; then - apk add cmake > /dev/null - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - apt-get install -y cmake - else - if [ "`yum search cmake | grep ^cmake28\.`" ]; then - cmake=cmake28 - fi - # centos based distro - yum_install $cmake > /dev/null - fi - echo $cmake -} - -function get_cmake { - >&2 echo "get_cmake has been deprecated. Please use get_modern_cmake instead." - get_modern_cmake -} - -function build_openjpeg { - if [ -e openjpeg-stamp ]; then return; fi - build_zlib - build_libpng - build_tiff - build_lcms2 - local cmake=$(get_modern_cmake) - local archive_prefix="v" - if [ $(lex_ver $OPENJPEG_VERSION) -lt $(lex_ver 2.1.1) ]; then - archive_prefix="version." - fi - local out_dir=$(fetch_unpack https://github.com/uclouvain/openjpeg/archive/${archive_prefix}${OPENJPEG_VERSION}.tar.gz) - (cd $out_dir \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ - && make install) - touch openjpeg-stamp -} - -function build_lcms2 { - build_tiff - build_simple lcms2 $LCMS2_VERSION https://downloads.sourceforge.net/project/lcms/lcms/$LCMS2_VERSION -} - -function build_giflib { - local name=giflib - local version=$GIFLIB_VERSION - local url=https://downloads.sourceforge.net/project/giflib - if [ $(lex_ver $GIFLIB_VERSION) -lt $(lex_ver 5.1.5) ]; then - build_simple $name $version $url - else - local ext=tar.gz - if [ -e "${name}-stamp" ]; then - return - fi - local name_version="${name}-${version}" - local archive=${name_version}.${ext} - fetch_unpack $url/$archive - (cd $name_version \ - && make -j4 \ - && make install) - touch "${name}-stamp" - fi -} - -function build_xz { - build_simple xz $XZ_VERSION https://tukaani.org/xz -} - -function ensure_xz { - if [[ ! $(type -P "xz") ]]; then - if [ -n "$IS_MACOS" ]; then - brew install xz - else - build_xz - fi - fi -} - -function build_libwebp { - build_libpng - build_tiff - build_giflib - build_simple libwebp $LIBWEBP_VERSION \ - https://storage.googleapis.com/downloads.webmproject.org/releases/webp tar.gz \ - --enable-libwebpmux --enable-libwebpdemux -} - -function build_freetype { - build_libpng - build_bzip2 - build_simple freetype $FREETYPE_VERSION https://download.savannah.gnu.org/releases/freetype -} - -function build_libyaml { - build_simple yaml $LIBYAML_VERSION https://pyyaml.org/download/libyaml -} - -function build_szip { - # Build szip without encoding (patent restrictions) - build_zlib - build_simple szip $SZIP_VERSION \ - https://support.hdfgroup.org/ftp/lib-external/szip/$SZIP_VERSION/src tar.gz \ - --enable-encoding=no -} - -function build_hdf5 { - if [ -e hdf5-stamp ]; then return; fi - build_zlib - # libaec is a drop-in replacement for szip - build_libaec - local hdf5_url=https://support.hdfgroup.org/ftp/HDF5/releases - local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}') - fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz - (cd hdf5-$HDF5_VERSION \ - && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib \ - && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \ - --enable-threadsafe --enable-unsupported --with-pthread=yes \ - && make -j4 \ - && make install) - touch hdf5-stamp -} - -function build_libaec { - if [ -e libaec-stamp ]; then return; fi - local root_name=libaec-1.0.4 - local tar_name=${root_name}.tar.gz - # Note URL will change for each version - fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/ea0b7d197a950b0c110da8dfdecbb71f/${tar_name} - (cd $root_name \ - && ./configure --prefix=$BUILD_PREFIX \ - && make \ - && make install) - touch libaec-stamp -} - -function build_blosc { - if [ -e blosc-stamp ]; then return; fi - local cmake=$(get_modern_cmake) - fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz - (cd c-blosc-${BLOSC_VERSION} \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ - && make install) - if [ -n "$IS_MACOS" ]; then - # Fix blosc library id bug - for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do - install_name_tool -id $lib $lib - done - fi - touch blosc-stamp -} - -function build_snappy { - build_simple snappy $SNAPPY_VERSION https://github.com/google/snappy/releases/download/$SNAPPY_VERSION -} - -function build_lzo { - if [ -e lzo-stamp ]; then return; fi - fetch_unpack https://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz - (cd lzo-${LZO_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX --enable-shared \ - && make \ - && make install) - touch lzo-stamp -} - -function build_lzf { - build_simple liblzf $LZF_VERSION http://dist.schmorp.de/liblzf -} - -function build_curl { - if [ -e curl-stamp ]; then return; fi - local flags="--prefix=$BUILD_PREFIX" - if [ -n "$IS_MACOS" ]; then - flags="$flags --with-darwinssl" - else # manylinux - flags="$flags --with-ssl" - build_openssl - fi - fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz - (cd curl-${CURL_VERSION} \ - && if [ -z "$IS_MACOS" ]; then \ - LIBS=-ldl ./configure $flags; else \ - ./configure $flags; fi\ - && make -j4 \ - && make install) - touch curl-stamp -} - -function check_sha256sum { - local fname=$1 - if [ -z "$fname" ]; then echo "Need path"; exit 1; fi - local sha256=$2 - if [ -z "$sha256" ]; then echo "Need SHA256 hash"; exit 1; fi - echo "${sha256} ${fname}" > ${fname}.sha256 - if [ -n "$IS_MACOS" ]; then - shasum -a 256 -c ${fname}.sha256 - else - sha256sum -c ${fname}.sha256 - fi - rm ${fname}.sha256 -} - -function build_openssl { - if [ -e openssl-stamp ]; then return; fi - fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz - check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} - (cd ${OPENSSL_ROOT} \ - && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \ - && make -j4 \ - && make install) - touch openssl-stamp -} - -function build_netcdf { - if [ -e netcdf-stamp ]; then return; fi - build_hdf5 - build_curl - fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz - (cd netcdf-c-${NETCDF_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX --enable-dap \ - && make -j4 \ - && make install) - touch netcdf-stamp -} - -function build_pcre { - build_simple pcre $PCRE_VERSION https://sourceforge.net/projects/pcre/files/pcre/${PCRE_VERSION} -} - -function build_swig { - if [ -e swig-stamp ]; then return; fi - if [ -n "$IS_MACOS" ]; then - brew install swig > /dev/null - else - build_pcre - build_simple swig $SWIG_VERSION https://prdownloads.sourceforge.net/swig - fi - touch swig-stamp -} - -function build_suitesparse { - if [ -e suitesparse-stamp ]; then return; fi - if [ -n "$IS_MACOS" ]; then - brew install suite-sparse > /dev/null - elif [ -n "$IS_ALPINE" ]; then - apk add suitesparse-dev - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - apt-get install -y libsuitesparse-dev > /dev/null - else - # centos based distro - yum_install suitesparse-devel > /dev/null - fi - touch suitesparse-stamp -} - -function build_libtool { - build_simple libtool $LIBTOOL_VERSION https://ftp.gnu.org/gnu/libtool -} - -function build_ragel { - local htprefix=https - if [ -n "$IS_MACOS" ]; then - # Invalid certificate, when using macOS curl. - # https://security.stackexchange.com/questions/232445/https-connection-to-specific-sites-fail-with-curl-on-macos - # Cert will likely be removed by Safari update in due course. - htprefix=http - fi - build_simple ragel $RAGEL_VERSION ${htprefix}://www.colm.net/files/ragel -} - -function build_bison { - build_simple bison $BISON_VERSION https://ftp.gnu.org/gnu/bison -} - -function build_flex { - # the flex repository's git tags have a 'v' prefix - build_simple flex $FLEX_VERSION \ - https://github.com/westes/flex/releases/download/v$FLEX_VERSION -} - -function build_fftw_case { - local configure_args=${@:0} - - build_simple fftw $FFTW_VERSION \ - http://www.fftw.org tar.gz \ - --with-pic --enable-shared --enable-threads --disable-fortran \ - $configure_args - # eval cd fftw-$FFTW_VERSION/tests && make check-local && cd - -} - -function build_fftw { - echo 'Building fftw' - - # Save off current CFLAGS - local old_cflags=$CFLAGS - - # Taken from: https://github.com/conda-forge/fftw-feedstock/blob/master/recipe/build.sh - export CFLAGS="-O3 -fomit-frame-pointer -fstrict-aliasing -ffast-math" - - # single - echo 'Building fftw: single' - build_fftw_case --enable-float --enable-sse --enable-sse2 --enable-avx - - # Clear stamp file which prevents subsequent builds - rm fftw-stamp - - # double - echo 'Building fftw: double' - build_fftw_case --enable-sse2 --enable-avx - - # Clear stamp file which prevents subsequent builds - rm fftw-stamp - - # long double (SSE2 and AVX not supported) - echo 'Building fftw: long double' - build_fftw_case --enable-long-double - - # Taken from: https://github.com/conda-forge/pyfftw-feedstock/blob/master/recipe/build.sh - export C_INCLUDE_PATH=$BUILD_PREFIX/include # required as fftw3.h installed here - - # define STATIC_FFTW_DIR so the patched setup.py will statically link FFTW - export STATIC_FFTW_DIR=$BUILD_PREFIX/lib - - # TODO: These can be made into asserts per: - # https://github.com/conda-forge/fftw-feedstock/blob/8eaa8a1c63e7fcb97c63c1ee8e33c62ef3afa9c7/recipe/meta.yaml#L29-L52 - ls -l $C_INCLUDE_PATH/fftw3* - ls -l $STATIC_FFTW_DIR/libfftw3* - - # restore CFLAGS - export CFLAGS=$old_cflags -} - -function build_cfitsio { - if [ -e cfitsio-stamp ]; then return; fi - if [ -n "$IS_MACOS" ]; then - brew install cfitsio - else - # cannot use build_simple because cfitsio has no dash between name and version - local cfitsio_name_ver=cfitsio${CFITSIO_VERSION} - fetch_unpack https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${cfitsio_name_ver}.tar.gz - (cd cfitsio \ - && ./configure --prefix=$BUILD_PREFIX \ - && make shared && make install) - fi - touch cfitsio-stamp -} diff --git a/multibuild.submodules/manylinux_utils.sh b/multibuild.submodules/manylinux_utils.sh deleted file mode 100644 index 8eecd5f9..00000000 --- a/multibuild.submodules/manylinux_utils.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# Useful utilities common across manylinux builds - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/common_utils.sh - -function get_platform { - # Report platform as given by uname - python -c 'import platform; print(platform.uname()[4])' -} - -function repair_wheelhouse { - # Runs 'auditwheel repair' over all wheels in a directory - # If the wheel is not renamed by the repair process, - # then the original wheel will be left unmodified - local in_dir=$1 - local out_dir=${2:-$in_dir} - for whl in $in_dir/*.whl; do - if [[ $whl == *none-any.whl ]]; then # Pure Python wheel - if [ "$in_dir" != "$out_dir" ]; then cp $whl $out_dir; fi - else - local tmpdir=$(mktemp -d -t) - - auditwheel repair $whl -w $tmpdir/ - - local built=$(find $tmpdir -name *.whl) - if [ $(basename $built) == $(basename $whl) ]; then - if [ "$in_dir" != "$out_dir" ]; then cp $whl $out_dir; fi - else - cp $built $out_dir - - # Remove unfixed if writing into same directory - if [ "$in_dir" == "$out_dir" ]; then rm $whl; fi - fi - rm -rf $tmpdir - fi - done - chmod -R a+rwX $out_dir -} - -function activate_ccache { - # Link up the correct location for ccache - mkdir -p /parent-home/.ccache - ln -s /parent-home/.ccache $HOME/.ccache - - # Now install ccache - local ccache_dir - if [ -n "$IS_ALPINE" ]; then - suppress apk add ccache - ccache_dir='/usr/lib/ccache/bin' - else - if [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - suppress apt-get install -y ccache - else - # centos based distro - suppress yum_install ccache - fi - - # Create fake compilers and prepend them to the PATH - # Note that yum is supposed to create these for us, - # but I had trouble finding them - ccache_dir='/usr/lib/ccache/compilers' - mkdir -p $ccache_dir - ln -s /usr/bin/ccache $ccache_dir/gcc - ln -s /usr/bin/ccache $ccache_dir/g++ - ln -s /usr/bin/ccache $ccache_dir/cc - ln -s /usr/bin/ccache $ccache_dir/c++ - fi - export PATH="$ccache_dir:$PATH" - - # Prove to the developer that ccache is activated - echo "Using C compiler: $(which gcc)" -} - -function yum_install { - # CentOS 5 yum doesn't fail in some cases, e.g. if package is not found - # https://serverfault.com/questions/694942/yum-should-error-when-a-package-is-not-available - yum install -y "$1" && rpm -q "$1" -} diff --git a/multibuild.submodules/osx_utils.sh b/multibuild.submodules/osx_utils.sh deleted file mode 100644 index 5749acd3..00000000 --- a/multibuild.submodules/osx_utils.sh +++ /dev/null @@ -1,560 +0,0 @@ -#!/bin/bash -# Use with ``source osx_utils.sh`` -set -e - -# Get our own location on this filesystem, load common utils -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -source $MULTIBUILD_DIR/common_utils.sh - -MACPYTHON_URL=https://www.python.org/ftp/python -MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions -WORKING_SDIR=working - -# As of 22 Aug 2024 - latest Python of each version with binary download -# available. -# See: https://www.python.org/downloads/macos/ -LATEST_2p7=2.7.18 -LATEST_3p5=3.5.4 -LATEST_3p6=3.6.8 -LATEST_3p7=3.7.9 -LATEST_3p8=3.8.10 -LATEST_3p9=3.9.13 -LATEST_3p10=3.10.11 -LATEST_3p11=3.11.9 -LATEST_3p12=3.12.5 -LATEST_3p13=3.13.0rc1 - - -function check_python { - if [ -z "$PYTHON_EXE" ]; then - echo "PYTHON_EXE variable not defined" - exit 1 - fi -} - -function check_pip { - if [ -z "$PIP_CMD" ]; then - echo "PIP_CMD variable not defined" - exit 1 - fi -} - -function check_var { - if [ -z "$1" ]; then - echo "required variable not defined" - exit 1 - fi -} - -function get_py_digit { - check_python - $PYTHON_EXE -c "import sys; print(sys.version_info[0])" -} - -function get_py_mm { - check_python - $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))" -} - -function get_py_mm_nodot { - check_python - $PYTHON_EXE -c "import sys; print('{0}{1}'.format(*sys.version_info[0:2]))" -} - -function get_py_prefix { - check_python - $PYTHON_EXE -c "import sys; print(sys.prefix)" -} - -function fill_pyver { - # Convert major or major.minor format to major.minor.micro - # - # Hence: - # 2 -> 2.7.11 (depending on LATEST_2p7 value) - # 2.7 -> 2.7.11 (depending on LATEST_2p7 value) - local ver=$1 - check_var $ver - if [[ $ver =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then - # Major.minor.micro format already - echo $ver - elif [ $ver == 2 ] || [ $ver == "2.7" ]; then - echo $LATEST_2p7 - elif [ $ver == 3 ] || [ $ver == "3.12" ]; then - echo $LATEST_3p12 - elif [ $ver == "3.13" ]; then - echo $LATEST_3p13 - elif [ $ver == "3.11" ]; then - echo $LATEST_3p11 - elif [ $ver == "3.10" ]; then - echo $LATEST_3p10 - elif [ $ver == "3.9" ]; then - echo $LATEST_3p9 - elif [ $ver == "3.8" ]; then - echo $LATEST_3p8 - elif [ $ver == "3.7" ]; then - echo $LATEST_3p7 - elif [ $ver == "3.6" ]; then - echo $LATEST_3p6 - elif [ $ver == "3.5" ]; then - echo $LATEST_3p5 - else - echo "Can't fill version $ver" 1>&2 - exit 1 - fi -} - -function macpython_sdk_list_for_version { - # return a list of SDK targets supported for a given CPython version - # Parameters - # $py_version (python version in major.minor.extra format) - # eg - # macpython_sdks_for_version 2.7.15 - # >> 10.6 10.9 - local _ver=$(fill_pyver $1) - local _major=${_ver%%.*} - local _return - - if [ "$(uname -m)" = "arm64" ]; then - _return="11.0" - elif [ "$_major" -eq "2" ]; then - [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6" - [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9" - elif [ "$_major" -eq "3" ]; then - [ $(lex_ver $_ver) -lt $(lex_ver 3.8) ] && _return="10.6" - [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5) ] && _return="$_return 10.9" - else - echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2 - exit 1 - fi - echo $_return -} - -function macpython_sdk_for_version { - # assumes the output of macpython_sdk_list_for_version is a list - # of SDK versions XX.Y in sorted order, eg "10.6 10.9" or "10.9" - echo $(macpython_sdk_list_for_version $1) | awk -F' ' '{print $NF}' -} - -function pyinst_ext_for_version { - # echo "pkg" or "dmg" depending on the passed Python version - # Parameters - # $py_version (python version in major.minor.extra format) - # - # Earlier Python installers are .dmg, later are .pkg. - local py_version=$1 - check_var $py_version - py_version=$(fill_pyver $py_version) - local py_0=${py_version:0:1} - if [ $py_0 -eq 2 ]; then - if [ "$(lex_ver $py_version)" -ge "$(lex_ver 2.7.9)" ]; then - echo "pkg" - else - echo "dmg" - fi - elif [ $py_0 -ge 3 ]; then - echo "pkg" - fi -} - -function pyinst_fname_for_version { - # echo filename for OSX installer file given Python and minimum - # macOS versions - # Parameters - # $py_version (Python version in major.minor.extra format) - # $py_osx_ver: {major.minor | not defined} - # if defined, the minimum macOS SDK version that Python is - # built for, eg: "10.6" or "10.9", if not defined, infers - # this from $py_version using macpython_sdk_for_version - local py_version=$1 - local inst_ext=$(pyinst_ext_for_version $py_version) - # Use the universal2 installer if we are on arm64 - # universal2 installer for python 3.8 needs macos 11.0 to run on - # and therefore x86_64 builds use the intel only installer. - # Note that intel only installer can create universal2 wheels, but - # creates intel only wheels by default. When PLAT=universal2 - # we set the env variable _PYTHON_HOST_PLATFORM to change this - # default. - if [ "$(uname -m)" == "arm64" ] || [ $(lex_ver $py_version) -ge $(lex_ver 3.10.0) ]; then - if [ "$py_version" == "3.9.1" ]; then - echo "python-${py_version}-macos11.0.${inst_ext}" - else - echo "python-${py_version}-macos11.${inst_ext}" - fi - else - local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)} - echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}" - fi -} - -function get_macpython_arch { - # echo arch (e.g. intel or x86_64), extracted from the distutils platform tag - # Parameters - # $distutils_plat PEP425 style platform tag, or if not provided, calls - # the function get_distutils_platform, provided by - # common_utils.sh. Fails if this is not a mac platform - # - # Note: MUST only be called after the version of Python used to build the - # target wheel has been installed and is on the path - local distutils_plat=${1:-$(get_distutils_platform)} - if [[ $distutils_plat =~ macosx-(1[0-9]\.[0-9]+)-(.*) ]]; then - echo ${BASH_REMATCH[2]} - else - echo "Error parsing macOS distutils platform '$distutils_plat'" - exit 1 - fi -} - -function get_macpython_osx_ver { - # echo minimum macOS version (e.g. 10.9) from the distutils platform tag - # Parameters - # $distutils_plat PEP425 style platform tag, or if not provided, calls - # the function get_distutils_platform, provided by - # common_utils.sh. Fails if this is not a mac platform - # - # Note: MUST only be called after the version of Python used to build the - # target wheel has been installed and is on the path - local distutils_plat=${1:-$(get_distutils_platform)} - if [[ $distutils_plat =~ macosx-(1[0-9]\.[0-9]+)-(.*) ]]; then - echo ${BASH_REMATCH[1]} - else - echo "Error parsing macOS distutils platform '$distutils_plat'" - exit 1 - fi -} - -function macpython_arch_for_version { - # echo arch (intel or x86_64) that a version of Python is expected - # to be built for - # Parameters - # $py_ver Python version, in the format (major.minor.patch) for - # CPython, or pypy-(major.minor) for PyPy - # $py_osx_ver minimum macOS version the target Python is built for - # (major.minor) - local py_ver=$1 - local py_osx_ver=${2:-$MB_PYTHON_OSX_VER} - check_var $1 - if [[ $(macpython_impl_for_version $py_ver) == "cp" ]]; then - if [[ "$py_osx_ver" == "10.6" ]]; then - echo "intel" - elif [[ "$py_osx_ver" == "10.9" ]]; then - echo "x86_64" - else - echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6 and 10.9" - exit 1 - fi - else - echo "x86_64" - fi -} - -function macpython_impl_for_version { - # echo Python implementation (cp for CPython, pp for PyPy) given a - # suitably formatted version string - # Parameters: - # $version : [implementation-]major[.minor[.patch]] - # Python implementation, e.g. "3.6" for CPython or - # "pypy-5.4" for PyPy - local version=$1 - check_var $1 - if [[ "$version" =~ ^pypy ]]; then - echo pp - elif [[ "$version" =~ ([0-9\.]+) ]]; then - echo cp - else - echo "config error: Issue parsing this implementation in install_python:" - echo " version=$version" - exit 1 - fi -} - -function strip_macpython_ver_prefix { - # strip any implementation prefix from a Python version string - # Parameters: - # $version : [implementation-]major[.minor[.patch]] - # Python implementation, e.g. "3.6" for CPython or - # "pypy-5.4" for PyPy - local version=$1 - check_var $1 - if [[ "$version" =~ (pypy-)?([0-9\.]+) ]]; then - echo ${BASH_REMATCH[2]} - fi -} - -function install_macpython { - # Install Python and set $PYTHON_EXE to the installed executable - # Parameters: - # $version : [implementation-]major[.minor[.patch]] - # The Python implementation to install, e.g. "3.6", "pypy-5.4" or "pypy3.6-7.2" - # $py_osx_ver: {major.minor | not defined} - # if defined, the macOS version that CPython is built for, e.g. - # "10.6" or "10.9". Ignored for PyPy - local version=$1 - local py_osx_ver=$2 - local impl=$(macpython_impl_for_version $version) - if [[ "$impl" == "pp" ]]; then - install_pypy $version - elif [[ "$impl" == "cp" ]]; then - local stripped_ver=$(strip_macpython_ver_prefix $version) - install_mac_cpython $stripped_ver $py_osx_ver - else - echo "Unexpected Python impl: ${impl}" - exit 1 - fi -} - -function install_mac_cpython { - # Installs Python.org Python - # Parameters - # $py_version - # Version given in major or major.minor or major.minor.micro e.g - # "3" or "3.7" or "3.7.1". - # $py_osx_ver - # {major.minor | not defined} - # if defined, the macOS version that Python is built for, e.g. - # "10.6" or "10.9" - # sets $PYTHON_EXE variable to Python executable - local py_version=$(fill_pyver $1) - local py_osx_ver=$2 - local py_stripped=$(strip_ver_suffix $py_version) - local py_inst=$(pyinst_fname_for_version $py_version $py_osx_ver) - local inst_path=$DOWNLOADS_SDIR/$py_inst - local retval="" - mkdir -p $DOWNLOADS_SDIR - # exit early on curl errors, but don't let it exit the shell - cmd_notexit curl -f $MACPYTHON_URL/$py_stripped/${py_inst} > $inst_path || retval=$? - if [ ${retval:-0} -ne 0 ]; then - echo "Python download failed! Check ${py_inst} exists on the server." - exit $retval - fi - - if [ "${py_inst: -3}" == "dmg" ]; then - hdiutil attach $inst_path -mountpoint /Volumes/Python - inst_path=/Volumes/Python/Python.mpkg - fi - sudo installer -pkg $inst_path -target / - local py_mm=${py_version%.*} - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$py_mm/bin/python$py_mm - # Install certificates for Python 3.6 - local inst_cmd="/Applications/Python ${py_mm}/Install Certificates.command" - if [ -e "$inst_cmd" ]; then - sh "$inst_cmd" - fi -} - -function install_virtualenv { - # Generic install of virtualenv - # Installs virtualenv into python given by $PYTHON_EXE - # Assumes virtualenv will be installed into same directory as $PYTHON_EXE - check_pip - # Travis VMS install virtualenv for system python by default - force - # install even if installed already - $PIP_CMD install virtualenv --ignore-installed - check_python - VIRTUALENV_CMD="$(dirname $PYTHON_EXE)/virtualenv" -} - -function make_workon_venv { - # Make a virtualenv in given directory ('venv' default) - # Set $PYTHON_EXE, $PIP_CMD to virtualenv versions - # Parameter $venv_dir - # directory for virtualenv - local venv_dir=$1 - if [ -z "$venv_dir" ]; then - venv_dir="venv" - fi - venv_dir=`abspath $venv_dir` - check_python - $PYTHON_EXE -m virtualenv $venv_dir - PYTHON_EXE=$venv_dir/bin/python - PIP_CMD=$venv_dir/bin/pip -} - -function remove_travis_ve_pip { - # Remove travis installs of virtualenv and pip - # FIXME: What if virtualenv is installed but pip is not? - if [ "$(sudo which virtualenv)" == /usr/local/bin/virtualenv ] && [ "$(sudo which pip)" == /usr/local/bin/pip ]; then - sudo pip uninstall -y virtualenv; - fi - if [ "$(sudo which pip)" == /usr/local/bin/pip ]; then - sudo pip uninstall -y pip; - fi -} - -function set_py_vars { - # Used by terryfy project; left here for back-compatibility - export PATH="`dirname $PYTHON_EXE`:$PATH" - export PYTHON_EXE PIP_CMD -} - -function get_macpython_environment { - # Set up MacPython environment - # Parameters: - # $version : [implementation-]major[.minor[.patch]] - # The Python implementation to install, e.g. "3.6" or "pypy-5.4" - # $venv_dir : {directory_name|not defined} - # If defined - make virtualenv in this directory, set python / pip - # commands accordingly - # $py_osx_ver: {major.minor | not defined} - # if defined, the macOS version that Python is built for, e.g. - # "10.6" or "10.9", if not defined, use the version from MB_PYTHON_OSX_VER - # - # Installs Python - # Sets $PYTHON_EXE to path to Python executable - # Sets $PIP_CMD to full command for pip (including sudo if necessary) - # If $venv_dir defined, Sets $VIRTUALENV_CMD to virtualenv executable - # Puts directory of $PYTHON_EXE on $PATH - local version=$1 - local venv_dir=$2 - local py_osx_ver=${3:-$MB_PYTHON_OSX_VER} - - if [ "$USE_CCACHE" == "1" ]; then - activate_ccache - fi - - remove_travis_ve_pip - install_macpython $version $py_osx_ver - PIP_CMD="$PYTHON_EXE -m pip" - # Python 3.5 no longer compatible with latest pip - if [ "$(get_py_mm)" == "3.5" ]; then - # https://stackoverflow.com/a/29751768/1939576 - curl -LO https://bootstrap.pypa.io/pip/3.5/get-pip.py - $PYTHON_EXE get-pip.py - rm get-pip.py - else - $PYTHON_EXE -m ensurepip - $PIP_CMD install --upgrade pip - fi - - if [ -n "$venv_dir" ]; then - install_virtualenv - make_workon_venv $venv_dir - source $venv_dir/bin/activate - else - export PATH="`dirname $PYTHON_EXE`:$PATH" - fi - export PYTHON_EXE PIP_CMD -} - -function install_delocate { - check_pip - $PIP_CMD install delocate -} - -function repair_wheelhouse { - local wheelhouse=$1 - install_delocate - delocate-wheel $wheelhouse/*.whl # copies library dependencies into wheel -} - -function install_pkg_config { - # Install pkg-config avoiding error from homebrew - # See : - # https://github.com/multi-build/multibuild/issues/24#issue-221951587 - command -v pkg-config > /dev/null 2>&1 || brew install pkg-config -} - -function activate_ccache { - - brew install ccache - export PATH=/usr/local/opt/ccache/libexec:$PATH - export CCACHE_CPP2=1 - - # Prove to the developer that ccache is activated - echo "Using C compiler: $(which clang)" -} - -function macos_intel_native_build_setup { - # Setup native build for single arch x86_64 wheels - export PLAT="x86_64" - export _PYTHON_HOST_PLATFORM="macosx-${MB_PYTHON_OSX_VER}-x86_64" - export CFLAGS+=" -arch x86_64" - export CXXFLAGS+=" -arch x86_64" - [[ $ARCHFLAGS =~ "-arch x86_64" ]] || export ARCHFLAGS+=" -arch x86_64" - export CPPFLAGS+=" -arch x86_64" - export LDFLAGS+=" -arch x86_64" -} - -function macos_intel_cross_build_setup { - echo "universal2 builds on arm64 are not supported yet." - exit 1 -} - -function macos_arm64_cross_build_setup { - # Setup cross build for single arch arm_64 wheels - export PLAT="arm64" - export BUILD_PREFIX=/opt/arm64-builds - sudo mkdir -p $BUILD_PREFIX/lib $BUILD_PREFIX/include - sudo chown -R $USER $BUILD_PREFIX - update_env_for_build_prefix - export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" - export CFLAGS+=" -arch arm64" - export CXXFLAGS+=" -arch arm64" - export CPPFLAGS+=" -arch arm64" - [[ $ARCHFLAGS =~ "-arch arm64" ]] || export ARCHFLAGS+=" -arch arm64" - export FCFLAGS+=" -arch arm64" - export FC=$FC_ARM64 - export F90=${F90_ARM64:-${FC}} - export F77=${F77_ARM64:-${FC}} - export MACOSX_DEPLOYMENT_TARGET="11.0" - export CROSS_COMPILING=1 - export LDFLAGS+=" -arch arm64 -L$BUILD_PREFIX/lib -Wl,-rpath,$BUILD_PREFIX/lib ${FC_ARM64_LDFLAGS:-}" - # This would automatically let autoconf know that we are cross compiling for arm64 darwin - export host_alias="aarch64-apple-darwin20.0.0" -} - -function macos_arm64_native_build_setup { - # Setup native build for single arch arm_64 wheels - export PLAT="arm64" - # We don't want universal2 builds and only want an arm64 build - export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" - export ARCHFLAGS+=" -arch arm64" - $@ -} - -function fuse_macos_intel_arm64 { - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - local py_osx_ver=$(echo ${MACOSX_DEPLOYMENT_TARGET} | sed "s/\./_/g") - mkdir -p tmp_fused_wheelhouse - for whl in $wheelhouse/*.whl; do - if [[ "$whl" == *macosx_${py_osx_ver}_x86_64.whl ]]; then - whl_base=$(sed "s/macosx_${py_osx_ver}_x86_64.whl//" <<< $whl) - if [[ -f "${whl_base}macosx_11_0_arm64.whl" ]]; then - delocate-fuse $whl "${whl_base}macosx_11_0_arm64.whl" -w tmp_fused_wheelhouse - mv tmp_fused_wheelhouse/$(basename $whl) $wheelhouse/$(basename ${whl_base})macosx_${py_osx_ver}_universal2.whl - # Since we want one wheel that's installable for testing we are deleting the *_x86_64 wheel. - # We are not deleting arm64 wheel because the size is lower and homebrew/conda-forge python - # will use them by default - rm $whl - fi - fi - done -} - -function wrap_wheel_builder { - if [[ "${PLAT:-}" == "universal2" ]]; then - if [[ "$(uname -m)" == "arm64" ]]; then - (macos_intel_cross_build_setup && $@) - rm -rf *-stamp - (macos_arm64_native_build_setup && $@) - else - (macos_intel_native_build_setup && $@) - rm -rf *-stamp - (macos_arm64_cross_build_setup && $@) - fi - fuse_macos_intel_arm64 - elif [[ "${PLAT:-}" == "arm64" ]]; then - if [[ "$(uname -m)" == "arm64" ]]; then - (macos_arm64_native_build_setup && $@) - else - (macos_arm64_cross_build_setup && $@) - fi - elif [[ "${PLAT:-}" == "x86_64" ]]; then - if [[ "$(uname -m)" == "x86_64" ]]; then - (macos_intel_native_build_setup && $@) - else - (macos_intel_cross_build_setup && $@) - fi - else - $@ - fi -} diff --git a/multibuild.submodules/supported_wheels.py b/multibuild.submodules/supported_wheels.py deleted file mode 100755 index cdec034a..00000000 --- a/multibuild.submodules/supported_wheels.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -""" Filter out wheel filenames not supported on this platform -""" -from __future__ import print_function - -import sys -from os.path import basename - -from packaging.tags import sys_tags - -try: - from wheel.install import WHEEL_INFO_RE as wheel_matcher -except ImportError: # As of Wheel 0.32.0 - from wheel.wheelfile import WHEEL_INFO_RE - wheel_matcher = WHEEL_INFO_RE.match - - -def tags_for(fname): - # Copied from WheelFile code - parsed_filename = wheel_matcher(basename(fname)) - tags = parsed_filename.groupdict() - for pyver in tags['pyver'].split('.'): - for abi in tags['abi'].split('.'): - for plat in tags['plat'].split('.'): - yield (pyver, abi, plat) - - -def main(): - supported = { - (tag.interpreter, tag.abi, tag.platform) for tag in sys_tags() - } - for fname in sys.argv[1:]: - tags = set(tags_for(fname)) - if supported.intersection(tags): - print(fname) - - -if __name__ == '__main__': - main() diff --git a/multibuild.submodules/tests/config.sh b/multibuild.submodules/tests/config.sh deleted file mode 100644 index 9d4866a7..00000000 --- a/multibuild.submodules/tests/config.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -ex -export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' - -function run_tests { - $PYTHON_EXE -c "import simplejson" -} - diff --git a/multibuild.submodules/tests/env_vars.sh b/multibuild.submodules/tests/env_vars.sh deleted file mode 100644 index 4995f411..00000000 --- a/multibuild.submodules/tests/env_vars.sh +++ /dev/null @@ -1,2 +0,0 @@ -export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' -set -x \ No newline at end of file diff --git a/multibuild.submodules/tests/test_common_utils.sh b/multibuild.submodules/tests/test_common_utils.sh deleted file mode 100644 index 57351deb..00000000 --- a/multibuild.submodules/tests/test_common_utils.sh +++ /dev/null @@ -1,113 +0,0 @@ -# Test common_utils - -[ "$(abspath foo)" == "$PWD/foo" ] || ingest "abspath foo" -[ "$(abspath foo/bar)" == "$PWD/foo/bar" ] || ingest "abspath foo/bar" -[ "$(abspath /foo)" == "/foo" ] || ingest "abspath /foo" -[ "$(relpath $PWD/foo)" == "foo" ] || ingest "relpath foo" -[ "$(relpath foo/bar foo)" == "bar" ] || ingest "relpath foo/bar" -[ "$(realpath /foo)" == "/foo" ] || ingest "realpath /foo" - -[ "$(lex_ver 2)" == "002000000" ] || ingest "lex_ver 2" -[ "$(lex_ver 2.1)" == "002001000" ] || ingest "lex_ver 2.1" -[ "$(lex_ver 2.1.4)" == "002001004" ] || ingest "lex_ver 2.1.4" -[ "$(lex_ver 2.1.4rc1)" == "002001004" ] || ingest "lex_ver 2.1.4" - -[ "$(unlex_ver 002000000)" == "2.0.0" ] || ingest "unlex_ver 002000000" -[ "$(unlex_ver 003002012)" == "3.2.12" ] || ingest "unlex_ver 003002012" -# Not octal -[ "$(unlex_ver 003044099)" == "3.44.99" ] || ingest "unlex_ver 003044099" -[ "$(unlex_ver 003543012)" == "3.543.12" ] || ingest "unlex_ver 003543012" -[ "$(unlex_ver 003543012abc)" == "3.543.12" ] || ingest "unlex_ver 003543012abc" - -[ "$(strip_ver_suffix 3.4.0rc1)" == "3.4.0" ] || ingest "unlex_ver strip suff 1" -[ "$(strip_ver_suffix 3.24.12a4)" == "3.24.12" ] || ingest "unlex_ver strip suff 2" - -[ "$(is_function abspath)" == "true" ] || ingest "is_function abspath" -[ "$(is_function foo)" == "" ] || ingest "is_function foo" -bar=baz -[ "$(is_function bar)" == "" ] || ingest "is_function bar" - -# Check function is not run in is_function. Thanks to Andrew Murray. -function rmfile { - rm testfile -} - -touch testfile -[ "$(is_function rmfile)" == "true" ] || ingest "is_function rmfile" -[ -f testfile ] || ingest "testfile removed during isfunction check" -rm testfile - -rm_mkdir tmp_dir -[ -d tmp_dir ] || ingest "tmp_dir does not exist" -touch tmp_dir/tmp_file -rm_mkdir tmp_dir -[ -e tmp_dir/tmp_file ] && ingest "tmp_dir/tmp_file should have been deleted" -rmdir tmp_dir - -# Test suppress command -function bad_cmd { - echo bad - return 1 -} - -function bad_mid_cmd { - # Command returns 0, but errors in the middle - echo ok for now - false - echo should be bad now - return 0 -} - -function good_cmd { - echo good - return 0 -} - -# Store state of options including -e, -x -# https://stackoverflow.com/questions/14564746/in-bash-how-to-get-the-current-status-of-set-x -ORIG_OPTS=$- -set +ex -[ "$(suppress bad_cmd)" == "$(printf "Running bad_cmd\nbad")" ] \ - || ingest "suppress bad_cmd" -suppress bash -c '! false' &>/dev/null \ - || ingest "suppress cmd with space" -[ "$(suppress good_cmd)" == "Running good_cmd" ] \ - || ingest "suppress good_cmd" -[ "$(suppress bad_mid_cmd)" == "Running bad_mid_cmd" ] \ - || ingest "suppress bad_mid_cmd" -# Can't use pipes here, because of the effect on set -e behavior. -expected="$(printf "Running bad_cmd\nbad")" -actual="$(set -e; suppress bad_cmd)" -[ "$actual" == "$expected" ] || ingest "suppress bad_cmd set -e" -expected="$(printf "Running good_cmd")" -actual="$(set -e; suppress good_cmd)" -[ "$actual" == "$expected" ] || ingest "suppress good_cmd set -e" -expected="$(printf "Running bad_mid_cmd\nok for now")" -actual="$(set -e; suppress bad_mid_cmd)" -[ "$actual" == "$expected" ] || ingest "suppress bad_mid_cmd set -e" -# Reset options -set_opts $ORIG_OPTS - -! expect_return 1 || ingest "Too few arguments" -! expect_return 1 good_cmd || ingest "unexpected success" -! expect_return 0 bad_cmd || ingest "unexpected failure" -expect_return 1 bad_cmd || ingest "fail with expected error 1" -! expect_return 2 bad_cmd || ingest "fail with unexpected error" -expect_return 0 good_cmd || ingest "succeed as expected" - -cmd_notexit good_cmd || ingest -! cmd_notexit bad_cmd || ingest -! cmd_notexit exit 1 || ingest - -# On Linux docker containers in travis, can be x86_64, i686, s390x, ppc64le, or -# aarch64 -[ "$(get_platform)" == x86_64 ] || \ - [ "$(get_platform)" == i686 ] || \ - [ "$(get_platform)" == aarch64 ] || \ - [ "$(get_platform)" == ppc64le ] || \ - [ "$(get_platform)" == s390x ] || \ - exit 1 - -# Crudest possible check for get_distutils_platform -expected=$(python -c "import distutils.util as du; print(du.get_platform())") -[ "$(get_distutils_platform)" == "$expected" ] || ingest "bad distutils platform" diff --git a/multibuild.submodules/tests/test_fill_pypy_ver.sh b/multibuild.submodules/tests/test_fill_pypy_ver.sh deleted file mode 100644 index 0d5e05ab..00000000 --- a/multibuild.submodules/tests/test_fill_pypy_ver.sh +++ /dev/null @@ -1,17 +0,0 @@ -# Test Python version fill utility, for pypy -[ "$(fill_pypy_ver 5)" == $LATEST_PP_5 ] || ingest "lpp5" -[ "$(fill_pypy_ver 6)" == $LATEST_PP_6 ] || ingest "lpp6" -[ "$(fill_pypy_ver 7)" == $LATEST_PP_7 ] || ingest "lpp7" -[ "$(fill_pypy_ver 5.0)" == $LATEST_PP_5p0 ] || ingest -[ "$(fill_pypy_ver 5.1)" == $LATEST_PP_5p1 ] || ingest -[ "$(fill_pypy_ver 5.3)" == $LATEST_PP_5p3 ] || ingest -[ "$(fill_pypy_ver 5.4)" == $LATEST_PP_5p4 ] || ingest -[ "$(fill_pypy_ver 5.6)" == $LATEST_PP_5p6 ] || ingest -[ "$(fill_pypy_ver 5.7)" == $LATEST_PP_5p7 ] || ingest -[ "$(fill_pypy_ver 5.8)" == $LATEST_PP_5p8 ] || ingest -[ "$(fill_pypy_ver 5.9)" == $LATEST_PP_5p9 ] || ingest -[ "$(fill_pypy_ver 6.0)" == $LATEST_PP_6p0 ] || ingest -[ "$(fill_pypy_ver 7.0)" == $LATEST_PP_7p0 ] || ingest -[ "$(fill_pypy_ver 7.1)" == $LATEST_PP_7p1 ] || ingest -[ "$(fill_pypy_ver 7.2)" == $LATEST_PP_7p2 ] || ingest -[ "$(fill_pypy_ver 7.3)" == $LATEST_PP_7p3 ] || ingest diff --git a/multibuild.submodules/tests/test_fill_pyver.sh b/multibuild.submodules/tests/test_fill_pyver.sh deleted file mode 100644 index 627e1b1e..00000000 --- a/multibuild.submodules/tests/test_fill_pyver.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Test python version fill utility -[ "$(fill_pyver 2)" == $LATEST_2p7 ] || ingest -[ "$(fill_pyver 2.7)" == $LATEST_2p7 ] || ingest -[ "$(fill_pyver 2.7.8)" == "2.7.8" ] || ingest -[ "$(fill_pyver 3)" == $LATEST_3p12 ] || ingest -[ "$(fill_pyver 3.13)" == $LATEST_3p13 ] || ingest -[ "$(fill_pyver 3.13.0)" == "3.13.0" ] || ingest -[ "$(fill_pyver 3.12)" == $LATEST_3p12 ] || ingest -[ "$(fill_pyver 3.12.0)" == "3.12.0" ] || ingest -[ "$(fill_pyver 3.11)" == $LATEST_3p11 ] || ingest -[ "$(fill_pyver 3.11.0)" == "3.11.0" ] || ingest -[ "$(fill_pyver 3.10)" == $LATEST_3p10 ] || ingest -[ "$(fill_pyver 3.10.0)" == "3.10.0" ] || ingest -[ "$(fill_pyver 3.9)" == $LATEST_3p9 ] || ingest -[ "$(fill_pyver 3.9.0)" == "3.9.0" ] || ingest -[ "$(fill_pyver 3.8)" == $LATEST_3p8 ] || ingest -[ "$(fill_pyver 3.8.0)" == "3.8.0" ] || ingest -[ "$(fill_pyver 3.7)" == $LATEST_3p7 ] || ingest -[ "$(fill_pyver 3.7.0)" == "3.7.0" ] || ingest -[ "$(fill_pyver 3.6)" == $LATEST_3p6 ] || ingest -[ "$(fill_pyver 3.6.0)" == "3.6.0" ] || ingest -[ "$(fill_pyver 3.5)" == $LATEST_3p5 ] || ingest -[ "$(fill_pyver 3.5.0)" == "3.5.0" ] || ingest diff --git a/multibuild.submodules/tests/test_fill_submodule.sh b/multibuild.submodules/tests/test_fill_submodule.sh deleted file mode 100644 index 740c5e61..00000000 --- a/multibuild.submodules/tests/test_fill_submodule.sh +++ /dev/null @@ -1,45 +0,0 @@ -# Test fill_submodule function -current_wd=$PWD - -rm_mkdir tmp_repos -cd tmp_repos -mkdir project -(cd project && git init && - echo "Interesting!" > README.txt && - git add README.txt && - local_author && - git commit -m "first project" && - git tag first-commit) -mkdir superproject -cd superproject -git init -git submodule add ../project -local_author -git commit -m "first superproject" -# Check the submodule is working correctly before intervention -cd project -remote_url=$(git config --get remote.origin.url) -[ "$(git log --format="%s")" == "first project" ] || ingest "bad submodule" -[ -f .git ] || ingest "expecting .git to be a file" -cd .. -# Intervene -fill_submodule project -cd project -[ "$(git log --format="%s")" == "first project" ] || ingest "bad after filling" -[ -d .git ] || ingest "expecting .git to be a directory" -[ "$(git config --get remote.origin.url)" == "$remote_url" ] || ingest "bad remote" -# Check we can do a checkout of a branch -git checkout master -# Checkout a tag -git checkout first-commit -cd .. -# Intervene again (has .git directory now) -fill_submodule project -cd project -[ "$(git log --format="%s")" == "first project" ] || ingest "bad after refilling" -[ -d .git ] || ingest "expecting .git to be a directory" -[ "$(git config --get remote.origin.url)" == "$remote_url" ] || ingest "bad remote" -cd .. - -cd "$current_wd" -rm -rf tmp_repos diff --git a/multibuild.submodules/tests/test_library_builders.sh b/multibuild.submodules/tests/test_library_builders.sh deleted file mode 100644 index 65218ce1..00000000 --- a/multibuild.submodules/tests/test_library_builders.sh +++ /dev/null @@ -1,74 +0,0 @@ -# Test some library builders - -# The environment -uname -a - -if [ -n "$IS_MACOS" ]; then - # Building on macOS - export BUILD_PREFIX="${PWD}/builds" - rm_mkdir $BUILD_PREFIX - source configure_build.sh - source library_builders.sh -else - # Building on Linux - # Glibc version - ldd --version - # configure_build.sh, library_builders.sh sourced in - # docker_build_wrap.sh -fi - -source tests/utils.sh - -start_spinner - -fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/2.7.4/harfbuzz-2.7.4.tar.xz -[ -d harfbuzz-2.7.4 ] || ingest ".tar.xz should have been unpacked" - -suppress build_bzip2 -suppress build_openssl -suppress build_libpng -suppress build_szip -suppress build_swig -# We need to find a failable test for build_github -# It needs a standalone C library with ./configure script. -# E.g. arb (below) requires a couple of other libraries. -# Run here just for the output, even though they fail. -(set +e ; - build_github fredrik-johansson/arb 2.21.1 ; - build_github glennrp/libpng v1.6.37 ; - build_github wbhart/mpir mpir-3.0.0 - ) -suppress build_flex -if [[ $MB_ML_VER != "_2_24" ]]; then - suppress build_openblas -fi -suppress ensure_xz -suppress build_tiff -suppress build_libwebp -suppress build_lcms2 -suppress build_freetype -suppress build_libyaml -if [ -z "$IS_MACOS" ]; then - # Gives compiler conformance error on macOS Sierra: - # https://gist.github.com/5e20e137ea51fa8ca9fc443191f9d463 - # https://gist.github.com/ad86c474f3c0b7ec74290bb13f9414af - suppress build_lzo -fi -suppress build_ragel -if [ -z "$IS_MACOS" ]; then - # already installed in the macOS image, so `brew install cfitsio` fails - suppress build_cfitsio -fi -suppress build_new_zlib -suppress build_hdf5 -rm jpeg-stamp -suppress build_libjpeg_turbo -suppress get_modern_cmake - -[ ${MB_PYTHON_VERSION+x} ] || ingest "\$MB_PYTHON_VERSION is not set" -[ "$MB_PYTHON_VERSION" == "$PYTHON_VERSION" ] || ingest "\$MB_PYTHON_VERSION must be equal to \$PYTHON_VERSION" - -stop_spinner - -# Exit 1 if any test errors -barf diff --git a/multibuild.submodules/tests/test_manylinux_utils.sh b/multibuild.submodules/tests/test_manylinux_utils.sh deleted file mode 100644 index cec6b93d..00000000 --- a/multibuild.submodules/tests/test_manylinux_utils.sh +++ /dev/null @@ -1,15 +0,0 @@ -# Tests for manylinux utils that can run outside docker - -# CPython path calculator -[ "$(cpython_path 2.7)" == "/opt/python/cp27-cp27mu" ] || ingest "cp 2.7" -[ "$(cpython_path 2.7 32)" == "/opt/python/cp27-cp27mu" ] || ingest "cp 2.7 32" -[ "$(cpython_path 2.7 16)" == "/opt/python/cp27-cp27m" ] || ingest "cp 2.7 16" -[ "$(cpython_path 3.5)" == "/opt/python/cp35-cp35m" ] || ingest "cp 3.5" -[ "$(cpython_path 3.5 32)" == "/opt/python/cp35-cp35m" ] || ingest "cp 3.5 32" -[ "$(cpython_path 3.5 16)" == "/opt/python/cp35-cp35m" ] || ingest "cp 3.5 16" -[ "$(cpython_path 3.7)" == "/opt/python/cp37-cp37m" ] || ingest "cp 3.7" -[ "$(cpython_path 3.7 32)" == "/opt/python/cp37-cp37m" ] || ingest "cp 3.7 32" -[ "$(cpython_path 3.7 16)" == "/opt/python/cp37-cp37m" ] || ingest "cp 3.7 16" -[ "$(cpython_path 3.8)" == "/opt/python/cp38-cp38" ] || ingest "cp 3.8" -[ "$(cpython_path 3.8 32)" == "/opt/python/cp38-cp38" ] || ingest "cp 3.8 32" -[ "$(cpython_path 3.8 16)" == "/opt/python/cp38-cp38" ] || ingest "cp 3.8 16" \ No newline at end of file diff --git a/multibuild.submodules/tests/test_manylinux_utils_docker.sh b/multibuild.submodules/tests/test_manylinux_utils_docker.sh deleted file mode 100644 index 2259d431..00000000 --- a/multibuild.submodules/tests/test_manylinux_utils_docker.sh +++ /dev/null @@ -1,11 +0,0 @@ -# Tests for manylinux utils that must run inside docker - -source manylinux_utils.sh -source tests/utils.sh - -if [[ $MB_ML_VER != "_2_24" ]]; then - suppress yum_install rsync && suppress yum erase -y rsync \ - || ingest "yum_install valid" - suppress bash -c '! yum_install nonexistent' || ingest "yum_install nonexistent" - barf -fi diff --git a/multibuild.submodules/tests/test_multibuild.sh b/multibuild.submodules/tests/test_multibuild.sh deleted file mode 100644 index f87c158d..00000000 --- a/multibuild.submodules/tests/test_multibuild.sh +++ /dev/null @@ -1,67 +0,0 @@ -# Test multibuild utilities -export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' -set -x -source common_utils.sh -# This is normally set -# - on linux via docker_build_wrap.sh, docker_test_wrap.sh -# - on macOS via make_workon_env or install_macpython, -# which is called by get_macpython_environment which is called by before_install -# We call it here so that these pre-install tests succeed -PYTHON_EXE=python - -source tests/utils.sh -source tests/test_common_utils.sh -source tests/test_fill_submodule.sh - -# make sure it is set via one of the mechanisms above -unset PYTHON_EXE - -if [ -n "$IS_MACOS" ]; then - source osx_utils.sh - MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$(macpython_sdk_for_version $MB_PYTHON_VERSION)} - PLAT=${PLAT:-$(macpython_arch_for_version $MB_PYTHON_VERSION)} - - # exit early if this cmd is expected to fail (and does) - if [[ -n $OSX_ENV_EXPECT_FAIL ]]; then - expect_return 22 get_macpython_environment $MB_PYTHON_VERSION ${VENV:-""} $MB_PYTHON_OSX_VER - exit 0 - fi - get_macpython_environment $MB_PYTHON_VERSION ${VENV:-""} $MB_PYTHON_OSX_VER - - source tests/test_python_install.sh - source tests/test_fill_pyver.sh - source tests/test_fill_pypy_ver.sh - source tests/test_osx_utils.sh -else - source manylinux_utils.sh - source tests/test_manylinux_utils.sh -fi -if [ -n "$TEST_BUILDS" ]; then - MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-3.7} - if [ -n "$IS_MACOS" ]; then - # This checked in test_library_builders. - # Will be set automatically by docker call in build_multilinux below. - PYTHON_VERSION=${MB_PYTHON_VERSION} - source tests/test_library_builders.sh - source tests/config.sh - elif [ ! -x "$(command -v docker)" ]; then - echo "Skipping build tests; no docker available" - else - touch config.sh - source travis_linux_steps.sh - my_plat=${PLAT:-x86_64} - build_multilinux $my_plat "source tests/test_manylinux_utils_docker.sh" - build_multilinux $my_plat "source tests/test_library_builders.sh" - build_multilinux $my_plat "pip install simplejson" - CONFIG_PATH=tests/config.sh - fi - build_index_wheel simplejson - install_run $PLAT -fi - -source tests/test_supported_wheels.sh - -# Exit 1 if any test errors -barf -# Don't need Travis' machinery trace -set +x diff --git a/multibuild.submodules/tests/test_osx_utils.sh b/multibuild.submodules/tests/test_osx_utils.sh deleted file mode 100644 index 3e36bf4f..00000000 --- a/multibuild.submodules/tests/test_osx_utils.sh +++ /dev/null @@ -1,77 +0,0 @@ -# Tests for OSX utils - -# Test extension for downloaded Python.org installer -[ "$(pyinst_ext_for_version 2.7.8)" == dmg ] || ingest -[ "$(pyinst_ext_for_version 2.7.9)" == pkg ] || ingest -[ "$(pyinst_ext_for_version 2.7)" == pkg ] || ingest -[ "$(pyinst_ext_for_version 2)" == pkg ] || ingest -[ "$(pyinst_ext_for_version 3.5.0)" == pkg ] || ingest -[ "$(pyinst_ext_for_version 3.5)" == pkg ] || ingest -[ "$(pyinst_ext_for_version 3)" == pkg ] || ingest - -[ "$(pyinst_fname_for_version 2.7.14)" == "python-2.7.14-macosx10.6.pkg" ] || ingest -[ "$(pyinst_fname_for_version 2.7.15)" == "python-2.7.15-macosx10.9.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.6.8)" == "python-3.6.8-macosx10.9.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.9.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx10.9.pkg" ] || ingest - -[ "$(pyinst_fname_for_version 2.7.14 10.6)" == "python-2.7.14-macosx10.6.pkg" ] || ingest -[ "$(pyinst_fname_for_version 2.7.15 10.6)" == "python-2.7.15-macosx10.6.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.6.8 10.6)" == "python-3.6.8-macosx10.6.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.7.1 10.6)" == "python-3.7.1-macosx10.6.pkg" ] || ingest - -[ "$(pyinst_fname_for_version 2.7.15 10.11)" == "python-2.7.15-macosx10.11.pkg" ] || ingest -[ "$(pyinst_fname_for_version 3.7.1 10.12)" == "python-3.7.1-macosx10.12.pkg" ] || ingest - -# Test utilities for getting Python version versions -[ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest -[ "$(get_py_mm)" == "${cpython_version:0:3}" ] || ingest -[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || ingest - -# test lookup of arch from Python macOS target build -[ "$(macpython_arch_for_version 2.7 10.6)" == "intel" ] || ingest -[ "$(macpython_arch_for_version 2.7 10.9)" == "x86_64" ] || ingest -[ "$(macpython_arch_for_version pypy-2.7)" == "x86_64" ] || ingest - -# test lookup of arch / min macOS versions from installed Python distutils tag -[ "$(get_macpython_arch macosx-10.6-intel)" == "intel" ] || ingest -[ "$(get_macpython_arch macosx-10.6-x86_64)" == "x86_64" ] || ingest -[ "$(get_macpython_osx_ver macosx-10.6-intel)" == "10.6" ] || ingest - -# test utilities for extracting version and impl from Python version string -[ "$(strip_macpython_ver_prefix 3.7.2)" == "3.7.2" ] || ingest -[ "$(strip_macpython_ver_prefix pypy-5.4)" == "5.4" ] || ingest -[ "$(macpython_impl_for_version 3.7.2)" == "cp" ] || ingest -[ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest - -# Test lookup of available macOS SDK build targets from python version -[ "$(macpython_sdk_list_for_version 3.8)" == "10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 3.7.5)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 3.7)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 3.6.5)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 3.6)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 3.5)" == "10.6" ] || ingest -[ "$(macpython_sdk_list_for_version 2.7)" == "10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 2.7.14)" == "10.6" ] || ingest -[ "$(macpython_sdk_list_for_version 2.7.15)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 2.7.17)" == "10.6 10.9" ] || ingest -[ "$(macpython_sdk_list_for_version 2.7.18)" == "10.9" ] || ingest - -(PLAT="arm64"; [ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest) -(PLAT="universal2"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -(PLAT="x86_64"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -[ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest -[ "$(macpython_sdk_for_version 3.8)" == "10.9" ] || ingest -[ "$(macpython_sdk_for_version 3.5)" == "10.6" ] || ingest -[ "$(macpython_sdk_for_version 2.7)" == "10.9" ] || ingest -[ "$(macpython_sdk_for_version 2.7.14)" == "10.6" ] || ingest - -# Test pkg-config install -install_pkg_config - -function echo_host_platform { - echo $_PYTHON_HOST_PLATFORM -} - -# Make sure `_PYTHON_HOST_PLATFORM` is set when building x86_64 thin wheel -(PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ]) diff --git a/multibuild.submodules/tests/test_python_install.sh b/multibuild.submodules/tests/test_python_install.sh deleted file mode 100644 index 58f8f761..00000000 --- a/multibuild.submodules/tests/test_python_install.sh +++ /dev/null @@ -1,82 +0,0 @@ -# Some debug echoes -echo "Python on path: `which python`" -echo "Python cmd: $PYTHON_EXE" -echo "pip on path: $(which pip)" -echo "pip cmd: $PIP_CMD" -echo "virtualenv on path: $(which virtualenv)" -echo "virtualenv cmd: $VIRTUALENV_CMD" - -# Check that a pip install puts scripts on path -# (Need setuptools >= 25.0.1 for delocate install). -$PIP_CMD install "setuptools>=25" -install_delocate -delocate-listdeps --version || ingest "Delocate not installed right" - -# Python version from Python to compare against required -if [[ $($PYTHON_EXE --version 2>&1 | awk '{print $2}') =~ ([0-9.]*).?([0-9.]*) ]] -then - # CPython version, 2.7.x on both CPython 2.7 and PyPy 5.4 - cpython_version=${BASH_REMATCH[1]} - # CPython/PyPy version - implementer_version=${BASH_REMATCH[2]:-$cpython_version} -fi -python_mm="${cpython_version:0:1}.${cpython_version:2:1}" - -# extract implementation prefix and version -if [[ "$MB_PYTHON_VERSION" =~ (pypy[0-9\.]*-)?([0-9\.]+) ]]; then - _impl=${BASH_REMATCH[1]:-"cp"} - requested_impl=${_impl:0:2} - requested_version=${BASH_REMATCH[2]} -else - ingest "Error parsing MB_PYTHON_VERSION=$MB_PYTHON_VERSION" -fi - -# simple regex match, a 2.7 pattern will match 2.7.11, but not 2 -if ! [[ "$implementer_version" =~ $requested_version ]]; then - ingest "Wrong python version: ${implementer_version}!=${requested_version}" -fi - -if [ -n "$VENV" ]; then # in virtualenv - # Correct pip and Python versions should be on PATH - if [ "$($PYTHON_EXE --version 2>&1)" != "$(python --version 2>&1)" ]; then - ingest "Python versions do not match" - fi - if [ "$($PIP_CMD --version)" != "$(pip --version)" ]; then - ingest "Pip versions do not match" - fi - # Versions in environment variables have full path - if [ "$PYTHON_EXE" != "$PWD/venv/bin/python" ]; then - ingest "Wrong virtualenv python '$PYTHON_EXE'" - fi - if [ "$PIP_CMD" != "${PWD}/venv/bin/pip${expected_pip_args}" ]; then - ingest "Wrong virtualenv pip '$PIP_CMD'" - fi -else # not virtualenv - if [[ $requested_impl == 'cp' ]]; then - macpie_bin="$MACPYTHON_PY_PREFIX/$python_mm/bin" - bin_name="python$python_mm" - else # pypy - macpie_bin="$PWD/pypy$python_mm-v$implementer_version-osx64/bin" - if [ "$(lex_ver $implementer_version)" -ge "$(lex_ver 7.3.2)" ]; then - bin_name="pypy3" - else - bin_name="pypy" - fi - fi - if [ "$PYTHON_EXE" != "$macpie_bin/$bin_name" ]; then - ingest "Wrong macpython python cmd '$PYTHON_EXE'" - fi - if [ "$PIP_CMD" != "$PYTHON_EXE -m pip" ]; then - ingest "Wrong macpython or pypy pip '$PIP_CMD'" - fi -fi - -# check macOS version and arch are as expected -distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())") -expected_arch=$(macpython_arch_for_version $MB_PYTHON_VERSION) -if [[ $requested_impl == 'cp' ]]; then - expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch" -else - expected_tag="macosx-10.[0-9]+-$expected_arch" -fi -[[ $distutils_plat =~ $expected_tag ]] || ingest diff --git a/multibuild.submodules/tests/test_supported_wheels.sh b/multibuild.submodules/tests/test_supported_wheels.sh deleted file mode 100644 index 6b59a7a7..00000000 --- a/multibuild.submodules/tests/test_supported_wheels.sh +++ /dev/null @@ -1,59 +0,0 @@ -# Test supported wheels script -PYTHON_EXE=${PYTHON_EXE:-python} -if [ -z "$PIP_CMD" ]; then - pip_install="$PYTHON_EXE -m pip install" -else - pip_install="$PIP_CMD install" -fi -# Needed for supported_wheels script -$pip_install packaging -# Current wheel versions not available for older Pythons. -lpv=$(lex_ver $MB_PYTHON_VERSION) -# Check no errors. -if [ $lpv -ge $(lex_ver 3.5) ] || [ $lpv -lt $(lex_ver 3) ]; then - for whl in wheel==0.31.1 wheel==0.32.0 wheel; do - $pip_install -U $whl - $PYTHON_EXE supported_wheels.py \ - tornado-5.1-cp27-cp27m-macosx_10_6_intel.whl \ - tornado-5.1-cp27-cp27m-macosx_10_9_intel.whl \ - tornado-5.1-cp27-cp27m-macosx_10_9_x86_64.whl \ - tornado-5.1-cp27-cp27m-macosx_10_13_x86_64.whl \ - tornado-5.1-cp36-cp36m-macosx_10_6_intel.whl \ - tornado-5.1-cp36-cp36m-macosx_10_9_intel.whl \ - tornado-5.1-cp36-cp36m-macosx_10_9_x86_64.whl \ - tornado-5.1-cp36-cp36m-macosx_10_13_x86_64.whl \ - texext-0.6.1-cp36-none-any.whl - done -fi - -# Test that wheels for versions other than our own, not supported. -py_impl=$($PYTHON_EXE -c 'import platform; print(platform.python_implementation())') -if [ "$py_impl" == 'CPython' ] && [ $(uname) == 'Darwin' ]; then - our_ver=$($PYTHON_EXE -c 'import sys; print("{}{}".format(*sys.version_info[:2]))') - other_ver=$([ "$our_ver" == "37" ] && echo "36" || echo "37") - # Python <= 3.7 needs m for API tag. - api_m=$([ $our_ver -le 37 ] && echo "m") || : - whl_suff="cp${our_ver}-cp${our_ver}${api_m}-macosx_10_9_x86_64.whl" - good_whl="tornado-5.1-${whl_suff}" - bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-macosx_10_9_x86_64.whl" - if [ "$($PYTHON_EXE supported_wheels.py $bad_whl)" != "" ]; then - echo "$bad_whl not supported, but supported wheels says it is." - RET=1 - fi - if [ "$($PYTHON_EXE supported_wheels.py $good_whl)" != "$good_whl" ]; then - echo "$good_whl supported, but supported wheels says it is not." - RET=1 - fi - good_whl2="mypkg-0.3-${whl_suff}" - both="$good_whl -$good_whl2" - if [ "$($PYTHON_EXE supported_wheels.py $good_whl $good_whl2)" != "$both" ]; then - echo "$good_whl, $good_whl2 supported, supported_wheels does not return both." - RET=1 - fi - if [ "$($PYTHON_EXE supported_wheels.py $good_whl $bad_whl $good_whl2)" != "$both" ]; then - echo "$good_whl, $good_whl2 supported, $bad_whl not; supported_wheels disagrees." - RET=1 - fi -fi - diff --git a/multibuild.submodules/tests/utils.sh b/multibuild.submodules/tests/utils.sh deleted file mode 100644 index 2908a9b5..00000000 --- a/multibuild.submodules/tests/utils.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Test utilities -RET=${RET:-0} - -function ingest { - local msg="${1:-"no message"}" - echo "Test failed: $msg" - RET=1 -} - -function barf { - [ "$RET" == 0 ] || exit 1 -} - -function local_author { - # Run in git repository to set commit author - git config user.email "my@noble.self" - git config user.name "Noble Self" -} diff --git a/multibuild.submodules/travis_linux_steps.sh b/multibuild.submodules/travis_linux_steps.sh deleted file mode 100644 index f53600fb..00000000 --- a/multibuild.submodules/travis_linux_steps.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -# Wheel build, install, run test steps on Linux -# -# In fact the main work is to wrap up the real functions in docker commands. -# The real work is in the BUILD_SCRIPT (which builds the wheel) and -# `docker_install_run.sh`, which can be configured with `config.sh`. -# -# Must define -# before_install -# build_wheel -# install_run -set -e - -# Get our own location on this filesystem -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") - -# Allow travis Python version as proxy for multibuild Python version -MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-$TRAVIS_PYTHON_VERSION} - -function before_install { - # Install a virtualenv to work in. - # Virtualenv package may not be installed on host Python. - python3.12 -m pip install virtualenv - python3.12 -m venv venv - source venv/bin/activate - python --version # just to check - # Tomli for pyproject.toml parsing, to get dependencies. - pip install --upgrade pip wheel tomli -} - -function build_wheel { - # Builds wheel, puts into $WHEEL_SDIR - # - # In fact wraps the actual work which happens in the container. - # - # Depends on - # REPO_DIR (or via input argument) - # PLAT (can be passed in as argument) - # MB_PYTHON_VERSION - # BUILD_COMMIT - # UNICODE_WIDTH (optional) - # BUILD_DEPENDS (optional) - # MANYLINUX_URL (optional) - # WHEEL_SDIR (optional) - local repo_dir=${1:-$REPO_DIR} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local plat=${2:-${PLAT:-x86_64}} - build_multilinux $plat "build_wheel $repo_dir" -} - -function build_index_wheel { - # Builds wheel from an index (e.g pypi), puts into $WHEEL_SDIR - # - # In fact wraps the actual work which happens in the container. - # - # Depends on - # PLAT (can be passed in as argument) - # MB_PYTHON_VERSION - # UNICODE_WIDTH (optional) - # BUILD_DEPENDS (optional) - # MANYLINUX_URL (optional) - # WHEEL_SDIR (optional) - local project_spec=$1 - [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1 - local plat=${2:-${PLAT:-x86_64}} - build_multilinux $plat "build_index_wheel $project_spec" -} - -function build_multilinux { - # Runs passed build commands in manylinux container - # - # Depends on - # MB_PYTHON_VERSION - # MB_ML_VER - # MB_ML_LIBC (optional) - # UNICODE_WIDTH (optional) - # BUILD_DEPENDS (optional) - # DOCKER_IMAGE (optional) - # MANYLINUX_URL (optional) - # WHEEL_SDIR (optional) - local plat=$1 - [ -z "$plat" ] && echo "plat not defined" && exit 1 - local build_cmds="$2" - local libc=${MB_ML_LIBC:-manylinux} - local docker_image=${DOCKER_IMAGE:-quay.io/pypa/${libc}${MB_ML_VER}_\$plat} - docker_image=$(eval echo "$docker_image") - retry docker pull $docker_image - docker run --rm \ - -e BUILD_COMMANDS="$build_cmds" \ - -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e MB_PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e UNICODE_WIDTH="$UNICODE_WIDTH" \ - -e BUILD_COMMIT="$BUILD_COMMIT" \ - -e CONFIG_PATH="$CONFIG_PATH" \ - -e ENV_VARS_PATH="$ENV_VARS_PATH" \ - -e WHEEL_SDIR="$WHEEL_SDIR" \ - -e MANYLINUX_URL="$MANYLINUX_URL" \ - -e BUILD_DEPENDS="$BUILD_DEPENDS" \ - -e USE_CCACHE="$USE_CCACHE" \ - -e REPO_DIR="$REPO_DIR" \ - -e PLAT="$PLAT" \ - -e MB_ML_VER="$MB_ML_VER" \ - -e MB_ML_LIBC="$libc" \ - -v $PWD:/io \ - -v $HOME:/parent-home \ - $docker_image /io/$MULTIBUILD_DIR/docker_build_wrap.sh -} - -function install_run { - # Install wheel, run tests - # - # In fact wraps the actual work which happens in the container. - # - # Depends on - # PLAT (can be passed in as argument) - # MB_PYTHON_VERSION - # MB_ML_LIBC (optional) - # UNICODE_WIDTH (optional) - # WHEEL_SDIR (optional) - # MANYLINUX_URL (optional) - # TEST_DEPENDS (optional) - # MB_TEST_VER (optional) - local plat=${1:-${PLAT:-x86_64}} - if [ -z "$DOCKER_TEST_IMAGE" ]; then - if [ "$MB_ML_LIBC" == "musllinux" ]; then - # PLAT is the same as $plat, - # unless $plat is "aarch64", in which case it becomes "arm64v8" - local docker_image="multibuild/alpine3.18_{PLAT}" - elif [ "$plat" == i686 ]; then - local docker_image="matthewbrett/trusty:32" - else - # PLAT is the same as $plat, - # unless $plat is "aarch64", in which case it becomes "arm64v8" - local docker_image="multibuild/focal_{PLAT}" - fi - else - local docker_image="$DOCKER_TEST_IMAGE" - fi - # aarch64 is called arm64v8 in Ubuntu - local plat_subst=$([ "$plat" == aarch64 ] && echo arm64v8 || echo $plat) - docker_image="${docker_image/\{PLAT\}/$plat_subst}" - docker pull $docker_image - docker run --rm \ - -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e MB_PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e UNICODE_WIDTH="$UNICODE_WIDTH" \ - -e CONFIG_PATH="$CONFIG_PATH" \ - -e WHEEL_SDIR="$WHEEL_SDIR" \ - -e MANYLINUX_URL="$MANYLINUX_URL" \ - -e MB_ML_LIBC="$MB_ML_LIBC" \ - -e TEST_DEPENDS="$TEST_DEPENDS" \ - -v $PWD:/io \ - $docker_image /io/$MULTIBUILD_DIR/docker_test_wrap.sh -} diff --git a/multibuild.submodules/travis_osx_steps.sh b/multibuild.submodules/travis_osx_steps.sh deleted file mode 100644 index 19e4fb44..00000000 --- a/multibuild.submodules/travis_osx_steps.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Wheel build, install, run test steps on OSX -set -e - -if [ "$PLAT" == "arm64" ] || [ "$PLAT" == "universal2" ]; then - if [[ "$(xcrun --sdk macosx --show-sdk-version | cut -d '.' -f 1)" -lt 11 ]]; then - latestXcode=$(ls /Applications | grep Xcode[_0-9\.]*\.app | sort -V | tail -n 1) - if ([ "$GITHUB_WORKFLOW" != "" ] || [ "$PIPELINE_WORKSPACE" != "" ]) && [ $latestXcode ]; then - sudo xcode-select -switch /Applications/$latestXcode - fi - if [[ "$(xcrun --sdk macosx --show-sdk-version | cut -d '.' -f 1)" -lt 11 ]]; then - echo "Need SDK>=11 for arm64 builds. Please run xcode-select to select a newer SDK" - exit 1 - fi - fi - export SDKROOT=${SDKROOT:-$(xcrun --sdk macosx --show-sdk-path)} -fi - -# Get needed utilities -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -MB_PYTHON_VERSION=${MB_PYTHON_VERSION:-$TRAVIS_PYTHON_VERSION} - -ENV_VARS_PATH=${ENV_VARS_PATH:-env_vars.sh} - -# These load common_utils.sh -source $MULTIBUILD_DIR/osx_utils.sh -MB_PYTHON_OSX_VER=${MB_PYTHON_OSX_VER:-$(macpython_sdk_for_version $MB_PYTHON_VERSION)} - -if [ -r "$ENV_VARS_PATH" ]; then source "$ENV_VARS_PATH"; fi -source $MULTIBUILD_DIR/configure_build.sh -source $MULTIBUILD_DIR/library_builders.sh - -# NB - config.sh sourced at end of this function. -# config.sh can override any function defined here. - -function before_install { - export CC=clang - export CXX=clang++ - - get_macpython_environment $MB_PYTHON_VERSION venv - source venv/bin/activate - # Tomli for pyproject.toml parsing, to get dependencies. - pip install --upgrade pip wheel tomli -} - -# build_wheel function defined in common_utils (via osx_utils) -# install_run function defined in common_utils - -# Local configuration may define custom pre-build, source patching. -# It can also overwrite the functions above. -CONFIG_PATH=${CONFIG_PATH:-config.sh} -source "$CONFIG_PATH" diff --git a/multibuild.submodules/travis_steps.sh b/multibuild.submodules/travis_steps.sh deleted file mode 100644 index 773183fd..00000000 --- a/multibuild.submodules/travis_steps.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Despite the name, this file is not specific to Travis-CI. -# It sets up the local environment for wheel building and testing. -# For Mac, configure xcode, and set up before_install and other -# functions to wrap builds. -# For linux, set up before_install to work in virtualenv, and set up wrapping -# to run build and tests in docker containers. - -WHEEL_SDIR=${WHEEL_SDIR:-wheelhouse} - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") - -if [ ! -d "$PWD/$WHEEL_SDIR" ]; then mkdir $PWD/$WHEEL_SDIR; fi -if [[ "$(uname)" == "Darwin" ]]; then - source $MULTIBUILD_DIR/travis_osx_steps.sh -else - source $MULTIBUILD_DIR/travis_linux_steps.sh -fi diff --git a/multibuild/.appveyor.yml b/multibuild/.appveyor.yml deleted file mode 100644 index 6db3a67a..00000000 --- a/multibuild/.appveyor.yml +++ /dev/null @@ -1,91 +0,0 @@ -# https://www.appveyor.com/docs/windows-images-software - -image: Visual Studio 2022 - -environment: - global: - REPO_DIR: python-appveyor-demo - PACKAGE_NAME: python_appveyor_demo - BUILD_COMMIT: master - BUILD_DEPENDS: "cython" - TEST_DEPENDS: "nose" - - matrix: - - PYTHON: "C:\\Miniconda37" - PYTHON_VERSION: "3.7" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Miniconda37-x64" - PYTHON_VERSION: "3.7" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Miniconda38-x64" - PYTHON_VERSION: "3.8" - PYTHON_ARCH: "64" - -# We always use a 64-bit machine, but can build x86 distributions -# with the TARGET_ARCH variable. -platform: - - x64 - -matrix: - fast_finish: false - -install: - # Install miniconda and fix headers - - where python - - where py # On Windows, py might be a better way to find and run CPython - - py --list # py -3.10-32, -3.10-64 with same syntax for 3.{9-3}, 2.{7,6} - - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PYTHON%\Library\bin;%PATH% - - echo %PATH% - - where conda - - conda info - - # Check that we have the expected version and architecture for Python - - python --version - - python -c "import struct; print(struct.calcsize('P') * 8)" - - py --version - - py -c "import struct; print(struct.calcsize('P') * 8)" - - # clone a origsel/python-appveyor-demo - # this repo includes a simple package to test appveyor - - git clone https://github.com/ogrisel/python-appveyor-demo.git - -build_script: - # Install build requirements - - conda install --yes %BUILD_DEPENDS% - - # build wheel: - - cd %REPO_DIR% - - git checkout %BUILD_COMMIT% - - python setup.py bdist_wheel - - ls dist/* - -test_script: - # create test env - - conda create --yes -n test_env python=%PYTHON_VERSION% %TEST_DEPENDS% - - activate test_env - - # install from wheel - - pip install --no-index --find-links dist/ %PACKAGE_NAME% - - # run tests from install wheel - - cd .. - - python -m pyappveyordemo.tests.test_extension - - # Smoke test of install_python script - # Use C:\PythonXY, C:\PythonXY-x64, C:\PythonXYrcZ, or C:\PythonXYrcZ-x64 - - set PYTHON=C:\Python37 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python37-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python38-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python38 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python39-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python39 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python310-x64 - - ps: .\install_python.ps1 - - set PYTHON=C:\Python310 - - ps: .\install_python.ps1 diff --git a/multibuild/.github/workflows/lint_python.yml b/multibuild/.github/workflows/lint_python.yml index fe56fb21..0ca9e8da 100644 --- a/multibuild/.github/workflows/lint_python.yml +++ b/multibuild/.github/workflows/lint_python.yml @@ -4,22 +4,16 @@ jobs: lint_python: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - run: pip install --upgrade pip wheel setuptools - - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear - flake8-comprehensions isort mypy pytest pyupgrade safety - - run: bandit --recursive --skip B101 . # B101 is assert statements - - run: black --check . || true - - run: codespell --ignore-words-list="commend" # --skip="*.css,*.js,*.lock" - - run: flake8 . --count --max-complexity=10 --max-line-length=88 - --show-source --statistics - - run: isort --check-only --profile black . + - run: pipx run codespell --ignore-words-list="commend" + - run: pip install mypy pytest ruff + - run: ruff format --check --config "format.quote-style = 'single'" + - run: ruff check --output-format=github --select=ALL --ignore=D203,D212,Q000 - run: pip install -r requirements.txt || pip install --editable . || true - run: mkdir --parents --verbose .mypy_cache - run: mypy --ignore-missing-imports --install-types --non-interactive . - run: pytest . || pytest --doctest-modules . || true - - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true - - run: safety check diff --git a/multibuild/.github/workflows/test.yml b/multibuild/.github/workflows/test.yml index 0ff11f16..a8d13822 100644 --- a/multibuild/.github/workflows/test.yml +++ b/multibuild/.github/workflows/test.yml @@ -20,28 +20,37 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] # [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.12"] # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"] + os: [ubuntu-latest, macos-15-intel, macos-latest ] #, windows-latest] + python-version: ["3.13"] # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3"] + + include: + # By default, specify an OSX_VER of 11.0, triggering ARM64 build. + - python-osx-ver: "11.0" + + # macos-15-intel is the last remaining x86-64 runner; + # set OSX_VER to 10.9 trigger x86_64 builds by default. + - os: macos-15-intel + python-osx-ver: "10.9" # The type of runner that the job will run on runs-on: ${{ matrix.os }} + env: + MB_PYTHON_VERSION: 3.13 + TEST_BUILDS: 1 + MB_PYTHON_OSX_VER: ${{ matrix.python-osx-ver }} + ENV_VARS_PATH: "test/env_vars.sh" # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Runs a single command using the runner's Python - - name: Run a one-line script - shell: python - run: print("Hello, world!") - - # Runs a set of commands using the runner's shell - - name: Run a multi-line script + - name: Test run: | - echo Add other actions to build, - echo test, and deploy your project. + pip install setuptools # for distutils + source tests/test_multibuild.sh diff --git a/multibuild/.gitignore b/multibuild/.gitignore index 37d7cb51..93d5c923 100644 --- a/multibuild/.gitignore +++ b/multibuild/.gitignore @@ -2,15 +2,19 @@ downloads/ archives/ *.orig *.swp +venv*/ +.venv/ # lib_check downloads these *-stamp +arch_tmp/ arb*/ bzip2*/ cfitsio/ flex*/ freetype*/ giflib*/ +harfbuzz*/ hdf5*/ jpeg*/ lcms2*/ diff --git a/multibuild/.travis.yml b/multibuild/.travis.yml index e6836e3d..b7a128cf 100644 --- a/multibuild/.travis.yml +++ b/multibuild/.travis.yml @@ -4,12 +4,12 @@ language: generic cache: directories: - - $HOME/.ccache + - $HOME/.ccache env: - global: - # Always set Python version - - MB_PYTHON_VERSION=3.11 + global: + # Always set Python version + - MB_PYTHON_VERSION=3.9 matrix: include: diff --git a/multibuild/README.rst b/multibuild/README.rst index 464f8271..63ff8bad 100644 --- a/multibuild/README.rst +++ b/multibuild/README.rst @@ -1,151 +1,33 @@ -################################################ -Utilities for building on Travis CI and AppVeyor -################################################ +############################################################ +Utilities for building wheels that use third-party libraries +############################################################ ************************************ Update: Uploads, Rackspace, Anaconda ************************************ The original Multibuild default was to upload wheels to a Rackspace container, -where Rackspace kindly donated the hosting to the Scikit-learn team. We had -a URL pointing to the Rackspace container: `http://wheels.scipy.org`. +where Rackspace kindly donated the hosting to the Scikit-learn team. Rackspace +finally stopped subsidizing this container. Some projects using Multibuild have moved +to using https://anaconda.org/scientific-python-nightly-wheels, see SPEC04_ for more +info. -Rackspace finally stopped subsidizing this container, and the Rackspace of -`http://wheels.scipy.org` is no more. Some projects using Multibuild have moved -to using https://anaconda.org/scipy-wheels-nightly/ for weekly uploads and -https://anaconda.org/multibuild-wheels-staging for staging wheels to PyPI. - -******************* -Uploads to Anaconda -******************* - -If you want to upload to Anaconda, and you don't need the extra storage space for nightly builds that Anaconda kindly donates to NumPy, SciPy etc, then you can do this with your own Anaconda organization. - -See https://github.com/MacPython/nipy-wheels for a simple example. - -* Make an account at Anaconda.org. -* Make an organization - for example we have used ``nipy``. -* Navigate to ``https://anaconda.org/nipy`` (but use your organization). -* Go to your account menu towards the top left. This should be labeled with your - organization name. -* Select "Settings", then "Access". -* Create an access token. Give it permission: "Allow write access to the API - site", and (not sure if this is necessary) "Allow uploads to PyPI - repositories". -* "View" your token. It may be of form - ``ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6``. -* Encrypt this to your repository, maybe using the ``travis`` command line tool - (``gem install travis``). Your command will be something like:: - - travis encrypt -r MacPython/nipy-wheels ANACONDA_SECRET=ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6 - - Note that ``MacPython/nipy-wheels`` is your Github organization/repository. The encryption only applies to Travis CI running against this repository. - -* Go to your `.travis.yml` file and add the output ``secure`` key. This will - look something like:: - - env: - global: - # Following generated with - # travis encrypt -r MacPython/nipy-wheels ANACONDA_SECRET= - # where has API write access to the anaconda.org - # organization named in $ANACONDA_ORG - - secure: "IqN7LjXWVBaijggUoB+ohjzFzH6nU0OyxznXEMgWoNxQJRiYXXKAt/Z5c4ldp9LUynefJO306M8foN4Gm8M8PNDlhjElzdOtIkGYtDKUXx7aXtrg8rPk1mzuM1F27er4Dbi7WFtpPClr8z8JKNNV50yeM1o2cXu4HgrPrRKgKk/2D8EQaPQlcOqul0O63D9AjVoW3EIG0aWEnZQQGfuGAPgyr0OS92LX2h1pcD2lNZHhqYmXmm5U0IwZmWL3Y0N7PO3VXcOCeIbiHAlJzhk4C4+86TT7DN+VhmfGyY/s61fOz47K+lEZLVqqeQki+HV75fti0XwYG7rjcSvDanNx+w2J/ogSLQpiNxZ0FZ+W8psXEaFUgFf7oXzRkW9gQ4KAsItEWHifq061ngr5AWLPLh+01LGP1Xg8wT5WEVUzBfD2uJPsy20DLcP9WGYa6cBNwtpqmUkdVgM3ZCPWlro7+v1kqxsKp91uh8SRKVlkD4mwbf0FnWxbNZ9v4Z9gs0pZoRclzL+/YcIcSTYAwiQRqaX7T0tpxaUZ0VYTMwCgpsufUX1idV1HV5+WKr9FUocoq+1RRW/JeXkisX9FRvem8cSGmnxB/hynlxoqzttCVMwtrKWPwxH4dHD+lavouho68Q7iBql1ZBZEhQy0O9NC1wr4Rg2CeDPZuzqVjmSPuXQ=" - - When Travis CI runs, this causes the ``ANACONDA_SECRET`` environment variable - to contain the API key above. - - Also add this to your global environment variables:: - - - ANACONDA_ORG="nipy" - -* To upload from Travis CI, add a clause like this to the end of your - ``.travis.yml``:: - - after_success: - # Upload wheels to Anaconda.org - - pip install git+https://github.com/Anaconda-Platform/anaconda-project - - pip install git+https://github.com/Anaconda-Platform/anaconda-client - - anaconda -t ${ANACONDA_SECRET} upload --force -u ${ANACONDA_ORG} ${TRAVIS_BUILD_DIR}/wheelhouse/*.whl - - ``wheelhouse/*.whl`` defines the files you want to upload. - -* You might also want to build and upload from AppVeyor. To encrypt the API - key above, go to https://ci.appveyor.com/account/matthew-brett/settings - (where ``matthew-brett`` is the account from which your AppVeyor job runs. - Click on "Encrypt YaML" on the left. Type in your API key value (e.g. - ``ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6``) as the value to encrypt. Click "Encrypt" and note the text it suggests. Now, at the top of your ``appveyor.yml`` file, add something like:: - - environment: - global: - ANACONDA_ORG: "nipy" - ANACONDA_SECRET: - secure: Ds0PkQD0b/QOfoNoiPuFJb01zg0Mq0dkAxIG2jRXocCAereSXdWw6XYaDrutHWme - - where ``secure:`` is the text suggested from "Encrypt" above, and ``nipy`` is your Anaconda organization. - - Finally, add a clause like the following to the end of your ``appveyor.yml`` file:: - - on_success: - # Upload the generated wheel package to Anaconda.org - - pip install git+https://github.com/Anaconda-Platform/anaconda-project - - pip install git+https://github.com/Anaconda-Platform/anaconda-client - - anaconda -t %ANACONDA_SECRET% upload --force -u %ANACONDA_ORG% nipy\dist\*.whl - - where ``nipy\dist\*.whl`` finds the files you want to upload. - -There's a simple example of these steps applied at -https://github.com/MacPython/nipy-wheels. - -Here is the NumPy code (for running on Travis CI) to upload to Anaconda: -https://github.com/MacPython/numpy-wheels/blob/master/.travis.yml#L99 - -For projects housed under the MacPython GitHub organization, you have access to -Anaconda upload tokens via the "Organization Secrets" -https://github.com/MacPython/numexpr-wheels/settings/secrets . You can use -these to move to GitHub Actions (they provide x86 machines for Windows, Linux -and Mac). Otherwise we (please raise an issue here) will need to negotiate -getting you tokens, or you can make your own, as above. - -********************** -Use Github for uploads -********************** - -Another option is to use GitHub for staging --- as do Cython `for Travis CI -`_ and -`for AppVeyor -`_. +.. _SPEC04: https://scientific-python.org/specs/spec-0004/ ************ Introduction ************ -A set of scripts to automate builds of macOS and Manylinux1 wheels on the -`Travis CI `_ infrastructure, and also Windows -wheels on the `AppVeyor `_ infrastructure. - -The Travis CI scripts are designed to build *and test*: - -* 64-bit macOS wheels built for macOS 10.9+ -* 64/32-bit macOS wheels built for macOS 10.6+ -* 64-bit ``manylinuxX_x86_64`` wheels, both narrow and wide Unicode builds, - where `X` is any valid Manylinux version: `1`, `2010`, `2014`, `_2_24` or `_2_28`. -* 32-bit ``manylinuxX_i686`` wheels, both narrow and wide Unicode builds - -You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10 -and 3.11. - -The small innovation here is that you can test against Linux 32-bit builds, both -wide and narrow Unicode Python 2 builds, which was not easy on the default -Travis CI configurations. +A set of scripts to automate builds of macOS and manylinux wheels. It will +work with windows, but that is not the prime target. -The AppVeyor setup is designed to build *and test*: +The CI scripts are designed to build *and test*: -* 64-bit Windows ``win_amd64`` wheels -* 32-bit Windows ``win32`` wheels +* 64-bit macOS x86_64 wheels built for macOS 10.9+ +* 64-bit macOS arm64 wheels built for macOS 10.6+ +* ``manylinux`` wheels in all the variations -You can currently build and test against Pythons 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10 -and 3.11. +You can build and test against all released versions of CPython and PyPy. ***************** How does it work? @@ -174,37 +56,33 @@ The following bash scripts are sourced in this order:: See ``multibuild/travis_osx_steps.sh`` -The macOS build / test phases run on the macOS VM started by Travis CI. -Therefore any environment variable defined in ``.travis.yml`` or the bash +The macOS build / test phases run on the VM started by CI. +Therefore any environment variable defined in the CI script or the bash shell scripts listed above are available for your build and test. Build options are controlled mainly by the following environment variables: -* ``MB_PYTHON_VER`` sets the Python version targeted: ``major.minor.patch`` +* ``MB_PYTHON_VERSION`` sets the Python version targeted: ``major.minor.patch`` for CPython, or ``pypy-major.minor`` for PyPy. -* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C - extensions. For CPython targets it may be set to 10.6 or 10.9, provided a +* ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C extensions. + For CPython targets it may be set to 10.6, 10.9 or 11.0, provided a corresponding Python build is available at `python.org `_. It defaults to the highest version available. It's ignored for PyPy targets. * ``PLAT`` sets the architectures built for any C extensions: ``x86_64`` or - ``intel`` for 64-bit or 64/32-bit respectively. It defaults to the same - arches as the target Python version: 64-bit for CPython macOS 10.9 or PyPy, - and 64/32-bit for CPython 10.6. + ``arm64``. It defaults to the same arches as the target Python version: arm64 + for macOS 11.0; x86_64 for CPython macOS 10.9 or PyPy; and 64/32-bit for + CPython 10.6. In most cases it's best to rely on the defaults for ``MB_PYTHON_OSX_VER`` and ``PLAT``, rather than setting them explicitly. Examples of exceptions to this guideline include: -* setting ``MB_PYTHON_OSX_VER=10.6`` to build a 10.6 64/32-bit CPython wheel +* setting ``MB_PYTHON_OSX_VER=10.6`` to build a 10.6 CPython wheel for Python 2.7 (default for 2.7 is 10.9 64-bit) * setting ``MB_PYTHON_OSX_VER=10.6 and PLAT=x86_64`` to build a 10.6 64-bit - only wheel (10.6 would normally be 64/32-bit). Such a wheel would still have - a platform tag of ``macosx_10_6_intel`` , advertising support for both 64 and - 32-bit, but wouldn't work in 32-bit mode. This may be OK given how unlikely it - is that there is still anyone actually running Python on macOS in 32-bit - mode. + wheel. The ``build_wheel`` function builds the wheel, and ``install_run`` function installs and tests it. Look in ``multibuild/common_utils.sh`` for @@ -275,20 +153,20 @@ variable. The default version is dependent on ``MB_ML_LIBC`` and ``PLAT``. When ``MB_ML_LIBC`` is ``musllinux``: -* ``multibuild/alpine3.18_x86_64``, when ``PLAT`` is ``x86_64`` -* ``multibuild/alpine3.18_arm64v8``, when ``PLAT`` is ``aarch64`` +* ``multibuild/alpine3.22_x86_64``, when ``PLAT`` is ``x86_64`` +* ``multibuild/alpine3.22_arm64v8``, when ``PLAT`` is ``aarch64`` Otherwise: -* ``multibuild/focal_x86_64``, when ``PLAT`` is ``x86_64`` +* ``multibuild/noble_x86_64``, when ``PLAT`` is ``x86_64`` * ``matthewbrett/trusty:32`` when ``PLAT`` is ``i686`` (Yes, an older image for 32-bit) -* ``multibuild/focal_arm64v8`` when ``PLAT`` is ``aarch64`` -* ``multibuild/focal_ppc64le`` when ``PLAT`` is ``ppc64le`` -* ``multibuild/focal_s390x`` when ``PLAT`` is ``s390x`` +* ``multibuild/noble_arm64v8`` when ``PLAT`` is ``aarch64`` +* ``multibuild/noble_ppc64le`` when ``PLAT`` is ``ppc64le`` +* ``multibuild/noble_s390x`` when ``PLAT`` is ``s390x`` -Other valid values are any in https://hub.docker.com/orgs/multibuild/repositories, +Other valid values are any in https://quay.io/organization/pypa, using the correct platform code. Alternatively, you can use the substitution -pattern ``multibuild/focal_{PLAT}`` in the ``.travis.yml`` file. +pattern ``multibuild/noble_{PLAT}`` in the ``.travis.yml`` file. See ``multibuild/docker_test_wrap.sh``. @@ -342,7 +220,7 @@ of the default implementations. To use these scripts ******************** -* Make a repository for building wheels on Travis CI - e.g. +* Make a repository for building wheels - e.g. https://github.com/MacPython/astropy-wheels - or in your case maybe ``https://github.com/your-org/your-project-wheels``; @@ -354,7 +232,7 @@ To use these scripts git submodule add https://github.com/your-org/your-project.git -* Create a ``.travis.yml`` file, something like this:: +* For Travis CI, create a ``.travis.yml`` file, something like this:: env: global: @@ -532,22 +410,9 @@ To use these scripts example, if your repository had a subdirectory ``scripts`` with a file ``my_env_vars.sh``, you should set ``ENV_VARS_PATH=scripts/my_env_vars.sh``. -* Make sure your project is set up to build on Travis CI, and you should now +* Make sure your project is set up to build, and you should now be ready (to begin the long slow debugging process, probably). -* For the Windows wheels, create an ``appveyor.yml`` file, something like: - - - https://github.com/MacPython/astropy-wheels/blob/master/appveyor.yml - - https://github.com/MacPython/nipy-wheels/blob/master/appveyor.yml - - https://github.com/MacPython/pytables-wheels/blob/master/appveyor.yml - - Note the Windows test customizations etc are inside ``appveyor.yml``, - and that ``config.sh`` and ``env_vars.sh`` are only for the - Linux/Mac builds on Travis CI. - -* Make sure your project is set up to build on AppVeyor, and you should now - be ready (for what could be another round of slow debugging). - * For Apple silicon support you can either create an ``arm64`` wheel or a ``universal2`` wheel by supplying ``PLAT`` env variable. ``universal2`` builds work on both ``arm64`` and ``x86_64`` platforms @@ -576,7 +441,7 @@ To use these scripts In multibuild we are going with option 2. You can override this behaviour by overriding the function ``wrap_wheel_builder``. - To build Apple silicon builds, you should use a CI service with Xcode 12 with + To build Apple silicon builds, you should use a CI service with Xcode>=12 with universal build support and make sure that xcode is the default. If your project depends on NumPy, you will want to build against the earliest diff --git a/multibuild/common_utils.sh b/multibuild/common_utils.sh deleted file mode 100755 index 5254ede0..00000000 --- a/multibuild/common_utils.sh +++ /dev/null @@ -1,712 +0,0 @@ -#!/bin/bash -# Utilities for both OSX and Docker Linux -# Python should be on the PATH - -# Only source common_utils once -if [ -n "$COMMON_UTILS_SOURCED" ]; then - return -fi -COMMON_UTILS_SOURCED=1 - -# Turn on exit-if-error -set -e - -MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") -DOWNLOADS_SDIR=downloads -PYPY_URL=https://downloads.python.org/pypy -# For back-compatibility. We use the "ensurepip" module now -# instead of get-pip.py -GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py - -# Unicode width, default 32. Used here and in travis_linux_steps.sh -# In docker_build_wrap.sh it is passed in when calling "docker run" -# The docker test images also use it when choosing the python to run -# with, so it is passed in when calling "docker run" for tests. -UNICODE_WIDTH=${UNICODE_WIDTH:-32} - -if [ $(uname) == "Darwin" ]; then - IS_MACOS=1; IS_OSX=1; -else - # In the manylinux_2_24 image, based on Debian9, "python" is not installed - # so link in something for the various system calls before PYTHON_EXE is set - which python || export PATH=/opt/python/cp39-cp39/bin:$PATH -fi - -if [ "$MB_ML_LIBC" == "musllinux" ]; then - IS_ALPINE=1; - MB_ML_VER=${MB_ML_VER:-"_1_1"} -else - # Default Manylinux version - MB_ML_VER=${MB_ML_VER:_2_24} -fi - -# Work round bug in travis xcode image described at -# https://github.com/direnv/direnv/issues/210 -shell_session_update() { :; } - -# Workaround for https://github.com/travis-ci/travis-ci/issues/8703 -# suggested by Thomas K at -# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274 -unset -f cd -unset -f pushd -unset -f popd - -function start_spinner { - if [ -n "$MB_SPINNER_PID" ]; then - return - fi - - >&2 echo "Building libraries..." - # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output - (while true; do - sleep 60 - >&2 echo "Still building..." - done) & - MB_SPINNER_PID=$! - disown -} - -function stop_spinner { - if [ ! -n "$MB_SPINNER_PID" ]; then - return - fi - - kill $MB_SPINNER_PID - unset MB_SPINNER_PID - - >&2 echo "Building libraries finished." -} - -function abspath { - # Can work with any Python; need not be our installed Python. - python3 -c "import os.path; print(os.path.abspath('$1'))" -} - -function relpath { - # Path of first input relative to second (or $PWD if not specified) - # Can work with any Python; need not be our installed Python. - python3 -c "import os.path; print(os.path.relpath('$1','${2:-$PWD}'))" -} - -function realpath { - # Can work with any Python; need not be our installed Python. - python3 -c "import os; print(os.path.realpath('$1'))" -} - -function lex_ver { - # Echoes dot-separated version string padded with zeros - # Thus: - # 3.2.1 -> 003002001 - # 3 -> 003000000 - echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}' -} - -function unlex_ver { - # Reverses lex_ver to produce major.minor.micro - # Thus: - # 003002001 -> 3.2.1 - # 003000000 -> 3.0.0 - echo "$((10#${1:0:3}+0)).$((10#${1:3:3}+0)).$((10#${1:6:3}+0))" -} - -function strip_ver_suffix { - echo $(unlex_ver $(lex_ver $1)) -} - -function is_function { - # Echo "true" if input argument string is a function - # Allow errors during "set -e" blocks. - (set +e; $(declare -Ff "$1" > /dev/null) && echo true) -} - -function gh_clone { - git clone https://github.com/$1 -} - -# gh-clone was renamed to gh_clone, so we have this alias for -# backwards compatibility. -alias gh-clone=gh_clone - -function set_opts { - # Set options from input options string (in $- format). - local opts=$1 - local chars="exhmBH" - for (( i=0; i<${#chars}; i++ )); do - char=${chars:$i:1} - [ -n "${opts//[^${char}]/}" ] && set -$char || set +$char - done -} - -function suppress { - # Run a command, show output only if return code not 0. - # Takes into account state of -e option. - # Compare - # https://unix.stackexchange.com/questions/256120/how-can-i-suppress-output-only-if-the-command-succeeds#256122 - # Set -e stuff agonized over in - # https://unix.stackexchange.com/questions/296526/set-e-in-a-subshell - local tmp=$(mktemp tmp.XXXXXXXXX) || return - local errexit_set - echo "Running $@" - if [[ $- = *e* ]]; then errexit_set=true; fi - set +e - ( if [[ -n $errexit_set ]]; then set -e; fi; "$@" > "$tmp" 2>&1 ) ; ret=$? - [ "$ret" -eq 0 ] || cat "$tmp" - rm -f "$tmp" - if [[ -n $errexit_set ]]; then set -e; fi - return "$ret" -} - -function expect_return { - # Run a command, succeeding (returning 0) only if the commend returns a specified code - # Parameters - # retcode expected return code (which may be zero) - # command the command called - # - # any further arguments are passed to the called command - # - # Returns 1 if called with less than 2 arguments - (( $# < 2 )) && echo "Must have at least 2 arguments" && return 1 - local retcode=$1 - local retval - ( "${@:2}" ) || retval=$? - [[ $retcode == ${retval:-0} ]] && return 0 - return ${retval:-1} -} - -function cmd_notexit { - # wraps a command, capturing its return code and preventing it - # from exiting the shell. Handles -e / +e modes. - # Parameters - # cmd - command - # any further parameters are passed to the wrapped command - # If called without an argument, it will exit the shell with an error - local cmd=$1 - if [ -z "$cmd" ];then echo "no command"; exit 1; fi - if [[ $- = *e* ]]; then errexit_set=true; fi - set +e - ("${@:1}") ; retval=$? - [[ -n $errexit_set ]] && set -e - return $retval -} - -function rm_mkdir { - # Remove directory if present, then make directory - local path=$1 - if [ -z "$path" ]; then echo "Need not-empty path"; exit 1; fi - if [ -d "$path" ]; then rm -rf $path; fi - mkdir $path -} - -function untar { - local in_fname=$1 - if [ -z "$in_fname" ];then echo "in_fname not defined"; exit 1; fi - local extension=${in_fname##*.} - case $extension in - tar) tar -xf $in_fname ;; - gz|tgz) tar -zxf $in_fname ;; - bz2) tar -jxf $in_fname ;; - zip) unzip -qq $in_fname ;; - xz) if [ -n "$IS_MACOS" ]; then - tar -xf $in_fname - else - if [[ ! $(type -P "unxz") ]]; then - echo xz must be installed to uncompress file; exit 1 - fi - unxz -c $in_fname | tar -xf - - fi ;; - *) echo Did not recognize extension $extension; exit 1 ;; - esac -} - -function install_rsync { - # install rsync via package manager - if [ -n "$IS_MACOS" ]; then - # macOS. The colon in the next line is the null command - : - elif [ -n "$IS_ALPINE" ]; then - [[ $(type -P rsync) ]] || apk add rsync - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - [[ $(type -P rsync) ]] || apt-get install -y rsync - else - # centos based distro - [[ $(type -P rsync) ]] || yum_install rsync - fi -} - -function fetch_unpack { - # Fetch input archive name from input URL - # Parameters - # url - URL from which to fetch archive - # archive_fname (optional) archive name - # - # Echos unpacked directory and file names. - # - # If `archive_fname` not specified then use basename from `url` - # If `archive_fname` already present at download location, use that instead. - local url=$1 - if [ -z "$url" ];then echo "url not defined"; exit 1; fi - local archive_fname=${2:-$(basename $url)} - local arch_sdir="${ARCHIVE_SDIR:-archives}" - if [ -z "$IS_MACOS" ]; then - local extension=${archive_fname##*.} - if [ "$extension" == "xz" ]; then - ensure_xz - fi - fi - # Make the archive directory in case it doesn't exist - mkdir -p $arch_sdir - local out_archive="${arch_sdir}/${archive_fname}" - # If the archive is not already in the archives directory, get it. - if [ ! -f "$out_archive" ]; then - # Source it from multibuild archives if available. - local our_archive="${MULTIBUILD_DIR}/archives/${archive_fname}" - if [ -f "$our_archive" ]; then - ln -s $our_archive $out_archive - else - # Otherwise download it. - curl -L $url > $out_archive - fi - fi - # Unpack archive, refreshing contents, echoing dir and file - # names. - rm_mkdir arch_tmp - install_rsync - (cd arch_tmp && \ - untar ../$out_archive && \ - ls -1d * && - rsync --delete -ah * ..) -} - -function clean_code { - local repo_dir=${1:-$REPO_DIR} - local build_commit=${2:-$BUILD_COMMIT} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 - # The package $repo_dir may be a submodule. git submodules do not - # have a .git directory. If $repo_dir is copied around, tools like - # Versioneer which require that it be a git repository are unable - # to determine the version. Give submodule proper git directory - fill_submodule "$repo_dir" - (cd $repo_dir \ - && git fetch origin --tags \ - && git checkout $build_commit \ - && git clean -fxd \ - && git reset --hard \ - && git submodule update --init --recursive) -} - -function build_wheel_cmd { - # Builds wheel with named command, puts into $WHEEL_SDIR - # - # Parameters: - # cmd (optional, default "pip_wheel_cmd" - # Name of command for building wheel - # repo_dir (optional, default $REPO_DIR) - # - # Depends on - # REPO_DIR (or via input argument) - # WHEEL_SDIR (optional, default "wheelhouse") - # BUILD_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local cmd=${1:-pip_wheel_cmd} - local repo_dir=${2:-$REPO_DIR} - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - mkdir -p "$wheelhouse" - #start_spinner - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - #stop_spinner - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $BUILD_DEPENDS - fi - (cd $repo_dir && $cmd $wheelhouse) - repair_wheelhouse $wheelhouse -} - -function pip_wheel_cmd { - local abs_wheelhouse=$1 - if [[ "$REPO_DIR" == "gdal" ]]; then - python -vv -m build -n -w . -o $abs_wheelhouse - fi - if [[ "$REPO_DIR" == "pyogrio" ]]; then - python -vv -m build -w . -o $abs_wheelhouse - fi - if [[ "$REPO_DIR" == "geopandas" ]]; then - python -vv -m build -w . -o $abs_wheelhouse - fi - pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . -} - -function bdist_wheel_cmd { - # Builds wheel with bdist_wheel, puts into wheelhouse - # - # It may sometimes be useful to use bdist_wheel for the wheel building - # process. For example, versioneer has problems with versions which are - # fixed with bdist_wheel: - # https://github.com/warner/python-versioneer/issues/121 - local abs_wheelhouse=$1 - check_python - $PYTHON_EXE setup.py bdist_wheel - cp dist/*.whl $abs_wheelhouse -} - -function wrap_wheel_builder { - # Wrapper for build commands, overwritten by macOS for universal2 or arm64 wheel building - $@ -} - -function build_pip_wheel { - # Standard wheel building command with pip wheel - wrap_wheel_builder build_wheel_cmd "pip_wheel_cmd" $@ -} - -function build_bdist_wheel { - # Wheel building with bdist_wheel. See bdist_wheel_cmd - wrap_wheel_builder build_wheel_cmd "bdist_wheel_cmd" $@ -} - -function build_wheel { - # Set default building method to pip - wrap_wheel_builder build_pip_wheel $@ -} - -function build_index_wheel_cmd { - # Builds wheel from some index, usually pypi - # - # Parameters: - # project_spec - # requirement to install, e.g. "tornado" or "tornado==4.4.1" - # *args - # Any other arguments to be passed to pip `install` and `wheel` - # commands. - # - # Depends on - # WHEEL_SDIR (optional, default "wheelhouse") - # BUILD_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - # - # You can also override `pip_opts` command to set indices other than pypi - local project_spec=$1 - [ -z "$project_spec" ] && echo "project_spec not defined" && exit 1 - # Discard first argument to pass remainder to pip - shift - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - #start_spinner - if [ -n "$(is_function "pre_build")" ]; then pre_build; fi - #stop_spinner - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $@ $BUILD_DEPENDS - fi - pip wheel $(pip_opts) $@ -w $wheelhouse --no-deps $project_spec - repair_wheelhouse $wheelhouse -} - -function build_index_wheel { - wrap_wheel_builder build_index_wheel_cmd $@ -} - -function pip_opts { - [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL" -} - -function get_os { - # Report OS as given by uname - # Use any Python that comes to hand. - python3 -c 'import platform; print(platform.uname()[0])' -} - -function get_platform { - # Report platform as given by uname - # Use any Python that comes to hand. - #python3 -c 'import platform; print(platform.uname()[4])' - python -c 'import platform; print(platform.uname()[4])' -} - -if [ "$(get_platform)" == x86_64 ] || \ - [ "$(get_platform)" == i686 ]; then IS_X86=1; fi - -function get_distutils_platform { - # Report platform as given by distutils get_platform. - # This is the platform tag that pip will use. - check_python - $PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())" -} - -function install_wheel { - # Install test dependencies and built wheel - # - # Pass any input flags to pip install steps - # - # Depends on: - # WHEEL_SDIR (optional, default "wheelhouse") - # TEST_DEPENDS (optional, default "") - # MANYLINUX_URL (optional, default "") (via pip_opts function) - local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) - check_pip - if [ -n "$TEST_DEPENDS" ]; then - while read TEST_DEPENDENCY; do - $PIP_CMD install $(pip_opts) $@ $TEST_DEPENDENCY - done <<< "$TEST_DEPENDS" - fi - - check_python - check_pip - - $PIP_CMD install packaging wheel - local supported_wheels=$($PYTHON_EXE $MULTIBUILD_DIR/supported_wheels.py $wheelhouse/*.whl) - if [ -z "$supported_wheels" ]; then - echo "ERROR: no supported wheels found" - exit 1 - fi - # Install compatible wheel - $PIP_CMD install $(pip_opts) $@ $supported_wheels -} - -function install_run { - # Depends on function `run_tests` defined in `config.sh` - install_wheel - mkdir tmp_for_test - (cd tmp_for_test && run_tests) - rmdir tmp_for_test 2>/dev/null || echo "Cannot remove tmp_for_test" -} - -function fill_submodule { - # Restores .git directory to submodule, if necessary - # See: - # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule - local repo_dir="$1" - [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - local git_loc="$repo_dir/.git" - # For ordinary submodule, .git is a file. - [ -d "$git_loc" ] && return - # Need to recreate .git directory for submodule - local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url) - local repo_copy="$repo_dir-$RANDOM" - git clone --recursive "$repo_dir" "$repo_copy" - rm -rf "$repo_dir" - mv "${repo_copy}" "$repo_dir" - (cd "$repo_dir" && git remote set-url origin $origin_url) -} - -# The latest versions of PyPy. -LATEST_PP_5p0=5.0.1 -LATEST_PP_5p1=5.1.1 -LATEST_PP_5p3=5.3.1 -LATEST_PP_5p4=5.4.1 -LATEST_PP_5p6=5.6.0 -LATEST_PP_5p7=5.7.1 -LATEST_PP_5p8=5.8.0 -LATEST_PP_5p9=5.9.0 -LATEST_PP_5=$LATEST_PP_5p9 - -LATEST_PP_6p0=6.0.0 -LATEST_PP_6=$LATEST_PP_6p0 - -LATEST_PP_7p0=7.0.0 -LATEST_PP_7p1=7.1.1 -LATEST_PP_7p2=7.2.0 -LATEST_PP_7p3=7.3.12 -LATEST_PP_7=$LATEST_PP_7p3 - -function unroll_version { - # Convert major or major.minor format to major.minor.micro using the above - # values recursively - # Parameters: - # $prefix : one of LATEST_PP or LATEST_PP3 - # $version : major[.minor[.patch]] - # Hence: - # LATEST_PP 5 -> 5.7.0 - # LATEST 2.7 -> 2.7.11 - local prefix=$1 - local ver=$2 - local latest=${prefix}_${ver//./p} - if [ -n "${!latest}" ]; then - echo $(unroll_version ${prefix} ${!latest}) - else - echo $ver - fi -} - -function install_pypy { - # Installs pypy.org PyPy - # Parameter $version - # Version given in major or major.minor or major.minor.micro e.g - # "3" or "3.7" or "3.7.1". - # Uses $PLAT - # sets $PYTHON_EXE variable to python executable - - local version=$1 - # Need to convert pypy-7.2 to pypy2.7-v7.2.0 and pypy3.6-7.3 to pypy3.6-v7.3.0 - local prefix=$(get_pypy_build_prefix $version) - # since prefix is pypy3.6v7.2 or pypy2.7v7.2, grab the 4th (0-index) letter - local major=${prefix:4:1} - # get the pypy version 7.2.0 - local py_version=$(fill_pypy_ver $(echo $version | cut -f2 -d-)) - - case "$PLAT" in - "x86_64") if [ -n "$IS_MACOS" ]; then - if [ $(lex_ver $py_version) -ge $(lex_ver 7.3.10) ]; then - suffix="macos_x86_64"; - else - suffix="osx64"; - fi - else - suffix="linux64"; - fi;; - "i686") suffix="linux32";; - "ppc64le") suffix="ppc64le";; - "s390x") suffix="s390x";; - "aarch64") suffix="aarch64";; - *) echo unknown platform "$PLAT"; exit 1;; - esac - - local py_build=$prefix$py_version-$suffix - local py_zip=$py_build.tar.bz2 - local zip_path=$DOWNLOADS_SDIR/$py_zip - mkdir -p $DOWNLOADS_SDIR - wget -nv $PYPY_URL/${py_zip} -P $DOWNLOADS_SDIR - untar $zip_path - # bug/feature: pypy package for pypy3 only has bin/pypy3 :( - if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pypy" ]; then - ln $py_build/bin/pypy3 $py_build/bin/pypy - fi - PYTHON_EXE=$(realpath $py_build/bin/pypy) - $PYTHON_EXE -mensurepip - $PYTHON_EXE -mpip install --upgrade pip setuptools wheel - if [ "$major" == "3" ] && [ ! -x "$py_build/bin/pip" ]; then - ln $py_build/bin/pip3 $py_build/bin/pip - fi - PIP_CMD=pip -} - -function fill_pypy_ver { - # Convert major or major.minor format to major.minor.micro - # Parameters: - # $version : major[.minor[.patch]] - # Hence: - # 5 -> 5.7.0 - echo $(unroll_version LATEST_PP $1) -} - -function get_pypy_build_prefix { - # Return the file prefix of a PyPy file - # Parameters: - # $version : pypy version number, for example pypy-7.2 or pypy3.6-7.2 - local version=$1 - if [[ $version =~ pypy([0-9]+)\.([0-9]+)-([0-9]+)\.([0-9]+) ]]; then - local py_major=${BASH_REMATCH[1]} - local py_minor=${BASH_REMATCH[2]} - echo "pypy$py_major.$py_minor-v" - elif [[ $version =~ ([0-9]+)\.([0-9]+) ]]; then - local major=${BASH_REMATCH[1]} - local minor=${BASH_REMATCH[2]} - if (( $major > 6 )); then - echo "pypy2.7-v" - elif (( $major > 5 || ($major == 5 && $minor >= 3) )); then - echo "pypy2-v" - else - echo "pypy-" - fi - else - echo "error: expected version like pypy-7.2 or pypy3.6-7.2, got $1" 1>&2 - exit 1 - fi -} - -retry () { - # Retry command (with arguments) up to 5 times - # https://gist.github.com/fungusakafungus/1026804 - local retry_max=5 - local count=$retry_max - while [ $count -gt 0 ]; do - "$@" && break - count=$(($count - 1)) - sleep 1 - done - - [ $count -eq 0 ] && { - echo "Retry failed [$retry_max]: $@" >&2 - return 1 - } - return 0 -} - -function install_pip { - # Generic install pip - echo "Deprecated - please see pip installs within the individual" - echo "install functions for each Python type." - echo "Multibuild itself no longer uses this function." - # Gets needed version from version implied by $PYTHON_EXE - # Installs pip into python given by $PYTHON_EXE - # Assumes pip will be installed into same directory as $PYTHON_EXE - check_python - mkdir -p $DOWNLOADS_SDIR - local py_mm=`get_py_mm` - local get_pip_path=$DOWNLOADS_SDIR/get-pip.py - curl $GET_PIP_URL > $get_pip_path - # Travis VMS now install pip for system python by default - force install - # even if installed already. - $PYTHON_EXE $get_pip_path --ignore-installed $pip_args - PIP_CMD=$(dirname $PYTHON_EXE)/pip$py_mm - if [ "$USER" != "root" ]; then - # inside a docker, there is no sudo but the user is already root - PIP_CMD="sudo $PIP_CMD" - fi - # Append pip_args if present (avoiding trailing space cf using variable - # above). - if [ -n "$pip_args" ]; then - PIP_CMD="$PIP_CMD $pip_args" - fi -} - -function check_python { - if [ -z "$PYTHON_EXE" ]; then - echo "PYTHON_EXE variable not defined" - exit 1 - fi -} - -function check_pip { - if [ -z "$PIP_CMD" ]; then - echo "PIP_CMD variable not defined" - exit 1 - fi -} - -function get_py_mm { - check_python - $PYTHON_EXE -c "import sys; print('{0}.{1}'.format(*sys.version_info[0:2]))" -} - -function cpython_path { - # Return path to cpython given - # * version (of form "2.7") - # * u_width ("16" or "32" default "32") - # - # For back-compatibility "u" as u_width also means "32" - local py_ver="${1:-2.7}" - local abi_suff=m - local u_width="${2:-${UNICODE_WIDTH}}" - local u_suff=u - # Python 3.8 and up no longer uses the PYMALLOC 'm' suffix - # https://github.com/pypa/wheel/pull/303 - if [ $(lex_ver $py_ver) -ge $(lex_ver 3.8) ]; then - abi_suff="" - fi - # Back-compatibility - if [ "$u_width" == "u" ]; then u_width=32; fi - # For Python >= 3.4, "u" suffix not meaningful - if [ $(lex_ver $py_ver) -ge $(lex_ver 3.4) ] || - [ "$u_width" == "16" ]; then - u_suff="" - elif [ "$u_width" == "" ]; then - u_width="32" - elif [ "$u_width" != "32" ]; then - echo "Incorrect u_width value $u_width" - exit 1 - fi - local no_dots=$(echo $py_ver | tr -d .) - echo "/opt/python/cp${no_dots}-cp${no_dots}$abi_suff${u_suff}" -} diff --git a/multibuild/library_builders.sh b/multibuild/library_builders.sh index a22f438f..28bcbc55 100644 --- a/multibuild/library_builders.sh +++ b/multibuild/library_builders.sh @@ -12,21 +12,21 @@ OPENBLAS_LIB_URL="https://anaconda.org/multibuild-wheels-staging/openblas-libs" # Recipes for building some libraries OPENBLAS_VERSION="${OPENBLAS_VERSION:-0.3.10}" # We use system zlib by default - see build_new_zlib -ZLIB_VERSION="${ZLIB_VERSION:-1.2.10}" -LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.37}" +ZLIB_VERSION="${ZLIB_VERSION:-1.3.1}" +LIBPNG_VERSION="${LIBPNG_VERSION:-1.6.50}" BZIP2_VERSION="${BZIP2_VERSION:-1.0.7}" FREETYPE_VERSION="${FREETYPE_VERSION:-2.11.0}" TIFF_VERSION="${TIFF_VERSION:-4.1.0}" JPEG_VERSION="${JPEG_VERSION:-9b}" -JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-2.1.3}" +JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-3.1.0}" OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}" LCMS2_VERSION="${LCMS2_VERSION:-2.9}" -GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}" -LIBWEBP_VERSION="${LIBWEBP_VERSION:-0.5.0}" -XZ_VERSION="${XZ_VERSION:-5.2.2}" +GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.4}" +LIBWEBP_VERSION="${LIBWEBP_VERSION:-1.4.0}" +XZ_VERSION="${XZ_VERSION:-5.8.1}" LIBYAML_VERSION="${LIBYAML_VERSION:-0.2.2}" SZIP_VERSION="${SZIP_VERSION:-2.1.1}" -HDF5_VERSION="${HDF5_VERSION:-1.10.5}" +HDF5_VERSION="${HDF5_VERSION:-1.14.5}" LIBAEC_VERSION="${LIBAEC_VERSION:-1.0.4}" LZO_VERSION=${LZO_VERSION:-2.10} LZF_VERSION="${LZF_VERSION:-3.6}" @@ -43,9 +43,9 @@ FLEX_VERSION=${FLEX_VERSION:-2.6.4} BISON_VERSION=${BISON_VERSION:-3.0.4} FFTW_VERSION=${FFTW_VERSION:-3.3.7} CFITSIO_VERSION=${CFITSIO_VERSION:-3450} -OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1l} +OPENSSL_ROOT=${OPENSSL_ROOT:-openssl-1.1.1w} # Hash from https://www.openssl.org/source/openssl-1.1.1?.tar.gz.sha256 -OPENSSL_HASH=${OPENSSL_HASH:-0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1} +OPENSSL_HASH=${OPENSSL_HASH:-cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8} OPENSSL_DOWNLOAD_URL=${OPENSSL_DOWNLOAD_URL:-https://www.openssl.org/source} @@ -67,10 +67,10 @@ function build_simple { local name_version="${name}-${version}" local archive=${name_version}.${ext} fetch_unpack $url/$archive - (cd $name_version \ - && ./configure --prefix=$BUILD_PREFIX $configure_args \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) + (cd $name_version \ + && ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS $configure_args \ + && make -j4 \ + && make install) touch "${name}-stamp" } @@ -85,7 +85,7 @@ function build_github { fi local out_dir=$(fetch_unpack "https://github.com/${path}/archive/${tag_name}.tar.gz") (cd $out_dir \ - && ./configure --prefix=$BUILD_PREFIX $configure_args \ + && ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS $configure_args \ && make -j4 \ && make install) touch "${name}-stamp" @@ -154,20 +154,23 @@ function build_new_zlib { function build_jpeg { if [ -e jpeg-stamp ]; then return; fi - fetch_unpack https://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz - (cd jpeg-${JPEG_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX \ - && make -j4 \ - && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) + fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz + (cd jpeg-${JPEG_VERSION} \ + && ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS \ + && make -j4 \ + && make install) touch jpeg-stamp } function build_libjpeg_turbo { if [ -e jpeg-stamp ]; then return; fi local cmake=$(get_modern_cmake) - fetch_unpack https://download.sourceforge.net/libjpeg-turbo/libjpeg-turbo-${JPEGTURBO_VERSION}.tar.gz + fetch_unpack https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/${JPEGTURBO_VERSION}/libjpeg-turbo-${JPEGTURBO_VERSION}.tar.gz (cd libjpeg-turbo-${JPEGTURBO_VERSION} \ - && $cmake -G"Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib . \ + && $cmake -G"Unix Makefiles" -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ + -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib \ + $HOST_CMAKE_FLAGS . \ + && make -j4 \ && make install) # Prevent build_jpeg @@ -197,21 +200,23 @@ function build_tiff { } function get_modern_cmake { - # Install cmake >= 2.8 + # Install cmake >= 2.8 if it isn't installed local cmake=cmake - if [ -n "$IS_MACOS" ]; then - brew install cmake > /dev/null - elif [ -n "$IS_ALPINE" ]; then - apk add cmake > /dev/null - elif [[ $MB_ML_VER == "_2_24" ]]; then - # debian:9 based distro - apt-get install -y cmake - else - if [ "`yum search cmake | grep ^cmake28\.`" ]; then - cmake=cmake28 + if ! which $cmake > /dev/null; then + if [ -n "$IS_MACOS" ]; then + brew install cmake > /dev/null + elif [ -n "$IS_ALPINE" ]; then + apk add cmake > /dev/null + elif [[ $MB_ML_VER == "_2_24" ]]; then + # debian:9 based distro + apt-get install -y cmake + else + if [ "`yum search cmake | grep ^cmake28\.`" ]; then + cmake=cmake28 + fi + # centos based distro + yum_install $cmake > /dev/null fi - # centos based distro - yum_install $cmake > /dev/null fi echo $cmake } @@ -234,7 +239,8 @@ function build_openjpeg { fi local out_dir=$(fetch_unpack https://github.com/uclouvain/openjpeg/archive/${archive_prefix}${OPENJPEG_VERSION}.tar.gz) (cd $out_dir \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib $HOST_CMAKE_FLAGS . \ + && make -j4 \ && make install) touch openjpeg-stamp } @@ -247,7 +253,7 @@ function build_lcms2 { function build_giflib { local name=giflib local version=$GIFLIB_VERSION - local url=https://downloads.sourceforge.net/project/giflib + local url=https://sourceforge.net/projects/giflib/files if [ $(lex_ver $GIFLIB_VERSION) -lt $(lex_ver 5.1.5) ]; then build_simple $name $version $url else @@ -260,13 +266,13 @@ function build_giflib { fetch_unpack $url/$archive (cd $name_version \ && make -j4 \ - && make install) + && make install PREFIX=$BUILD_PREFIX) touch "${name}-stamp" fi } function build_xz { - build_simple xz $XZ_VERSION https://tukaani.org/xz + build_simple xz $XZ_VERSION https://github.com/tukaani-project/xz/releases/download/v$XZ_VERSION } function ensure_xz { @@ -311,13 +317,14 @@ function build_hdf5 { build_zlib # libaec is a drop-in replacement for szip build_libaec - local hdf5_url=https://support.hdfgroup.org/ftp/HDF5/releases - local short=$(echo $HDF5_VERSION | awk -F "." '{printf "%d.%d", $1, $2}') - fetch_unpack $hdf5_url/hdf5-$short/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz + local hdf5_url=https://support.hdfgroup.org/releases/hdf5 + local short=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d", $1, $2}') + local long=$(echo $HDF5_VERSION | awk -F "." '{printf "v%d_%d_%d", $1, $2, $3}') + fetch_unpack $hdf5_url/$short/$long/downloads/hdf5-$HDF5_VERSION.tar.gz (cd hdf5-$HDF5_VERSION \ && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PREFIX/lib \ && ./configure --with-szlib=$BUILD_PREFIX --prefix=$BUILD_PREFIX \ - --enable-threadsafe --enable-unsupported --with-pthread=yes \ + --enable-threadsafe --enable-unsupported --with-pthread=yes $HOST_CONFIGURE_FLAGS \ && make -j4 \ && make install) touch hdf5-stamp @@ -330,8 +337,8 @@ function build_libaec { # Note URL will change for each version fetch_unpack https://gitlab.dkrz.de/k202009/libaec/uploads/ea0b7d197a950b0c110da8dfdecbb71f/${tar_name} (cd $root_name \ - && ./configure --prefix=$BUILD_PREFIX \ - && make \ + && ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS \ + && make -j4 \ && make install) touch libaec-stamp } @@ -341,14 +348,9 @@ function build_blosc { local cmake=$(get_modern_cmake) fetch_unpack https://github.com/Blosc/c-blosc/archive/v${BLOSC_VERSION}.tar.gz (cd c-blosc-${BLOSC_VERSION} \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX . \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib $HOST_CMAKE_FLAGS . \ + && make -j4 \ && make install) - if [ -n "$IS_MACOS" ]; then - # Fix blosc library id bug - for lib in $(ls ${BUILD_PREFIX}/lib/libblosc*.dylib); do - install_name_tool -id $lib $lib - done - fi touch blosc-stamp } @@ -360,8 +362,8 @@ function build_lzo { if [ -e lzo-stamp ]; then return; fi fetch_unpack https://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz (cd lzo-${LZO_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX --enable-shared \ - && make \ + && ./configure --prefix=$BUILD_PREFIX --enable-shared $HOST_CONFIGURE_FLAGS \ + && make -j4 \ && make install) touch lzo-stamp } @@ -420,7 +422,7 @@ function build_netcdf { build_curl fetch_unpack https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz (cd netcdf-c-${NETCDF_VERSION} \ - && ./configure --prefix=$BUILD_PREFIX --enable-dap \ + && ./configure --prefix=$BUILD_PREFIX --enable-dap $HOST_CONFIGURE_FLAGS \ && make -j4 \ && make install) touch netcdf-stamp @@ -543,7 +545,7 @@ function build_cfitsio { local cfitsio_name_ver=cfitsio${CFITSIO_VERSION} fetch_unpack https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${cfitsio_name_ver}.tar.gz (cd cfitsio \ - && ./configure --prefix=$BUILD_PREFIX \ + && ./configure --prefix=$BUILD_PREFIX $HOST_CONFIGURE_FLAGS \ && make shared && make install) fi touch cfitsio-stamp diff --git a/multibuild/osx_utils.sh b/multibuild/osx_utils.sh index 754289e7..cf6da2f7 100644 --- a/multibuild/osx_utils.sh +++ b/multibuild/osx_utils.sh @@ -10,7 +10,7 @@ MACPYTHON_URL=https://www.python.org/ftp/python MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions WORKING_SDIR=working -# As of 22 Aug 2024 - latest Python of each version with binary download +# As of 15 Oct 2025 - latest Python of each version with binary download # available. # See: https://www.python.org/downloads/macos/ LATEST_2p7=2.7.18 @@ -21,8 +21,9 @@ LATEST_3p8=3.8.10 LATEST_3p9=3.9.13 LATEST_3p10=3.10.11 LATEST_3p11=3.11.9 -LATEST_3p12=3.12.5 -LATEST_3p13=3.13.0rc1 +LATEST_3p12=3.12.10 +LATEST_3p13=3.13.9 +LATEST_3p14=3.14.0 function check_python { @@ -79,10 +80,12 @@ function fill_pyver { echo $ver elif [ $ver == 2 ] || [ $ver == "2.7" ]; then echo $LATEST_2p7 - elif [ $ver == 3 ] || [ $ver == "3.12" ]; then - echo $LATEST_3p12 + elif [ $ver == 3 ] || [ $ver == "3.14" ]; then + echo $LATEST_3p14 elif [ $ver == "3.13" ]; then echo $LATEST_3p13 + elif [ $ver == "3.12" ]; then + echo $LATEST_3p12 elif [ $ver == "3.11" ]; then echo $LATEST_3p11 elif [ $ver == "3.10" ]; then @@ -114,14 +117,14 @@ function macpython_sdk_list_for_version { local _major=${_ver%%.*} local _return - if [ "$(uname -m)" = "arm64" ]; then - _return="13.0" - elif [ "$_major" -eq "2" ]; then + if [ "$_major" -eq "2" ]; then [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6" [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9" elif [ "$_major" -eq "3" ]; then [ $(lex_ver $_ver) -lt $(lex_ver 3.8) ] && _return="10.6" [ $(lex_ver $_ver) -ge $(lex_ver 3.6.5) ] && _return="$_return 10.9" + [ $(lex_ver $_ver) -ge $(lex_ver 3.8.10) ] && [ "$_ver" != "3.9.0" ] && _return="$_return 11.0" + [ $(lex_ver $_ver) -ge $(lex_ver 3.10) ] && _return="11.0" else echo "Error version=${_ver}, expecting 2.x or 3.x" 1>&2 exit 1 @@ -166,23 +169,39 @@ function pyinst_fname_for_version { # built for, eg: "10.6" or "10.9", if not defined, infers # this from $py_version using macpython_sdk_for_version local py_version=$1 + local py_osx_ver local inst_ext=$(pyinst_ext_for_version $py_version) - # Use the universal2 installer if we are on arm64 + # macOS 3.8.10 and 3.9.1 introduced a second universal2 installer release. + # (3.9.0 did *not* have a universal2 installer) # universal2 installer for python 3.8 needs macos 11.0 to run on # and therefore x86_64 builds use the intel only installer. # Note that intel only installer can create universal2 wheels, but # creates intel only wheels by default. When PLAT=universal2 # we set the env variable _PYTHON_HOST_PLATFORM to change this # default. - if [ "$(uname -m)" == "arm64" ] || [ $(lex_ver $py_version) -ge $(lex_ver 3.10.0) ]; then - if [ "$py_version" == "3.9.1" ]; then - echo "python-${py_version}-macos11.0.${inst_ext}" - else - echo "python-${py_version}-macos11.${inst_ext}" - fi + if [ -z "$2" ]; then + if [ $(lex_ver $py_version) -ge $(lex_ver 3.8.10) ] \ + && [ $(lex_ver $py_version) -lt $(lex_ver 3.10.0) ]; then + if [ $(uname -m) == "x86_64" ]; then + py_osx_ver="10.9" + else + py_osx_ver="11.0" + fi + else + py_osx_ver=$(macpython_sdk_for_version $py_version) + fi + else + py_osx_ver=$2 + fi + + if [ "$py_osx_ver" == "11.0" ]; then + if [ "$py_version" == "3.9.1" ]; then + echo "python-${py_version}-macos11.0.${inst_ext}" + else + echo "python-${py_version}-macos11.${inst_ext}" + fi else - local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)} - echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}" + echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}" fi } @@ -223,7 +242,7 @@ function get_macpython_osx_ver { } function macpython_arch_for_version { - # echo arch (intel or x86_64) that a version of Python is expected + # echo arch (intel, x86_64 or arm64) that a version of Python is expected # to be built for # Parameters # $py_ver Python version, in the format (major.minor.patch) for @@ -238,8 +257,10 @@ function macpython_arch_for_version { echo "intel" elif [[ "$py_osx_ver" == "10.9" ]]; then echo "x86_64" + elif [[ "$py_osx_ver" == "11.0" ]]; then + echo "arm64" else - echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6 and 10.9" + echo "Unexpected CPython macOS version: ${py_osx_ver}, supported values: 10.6, 10.9, 11.0" exit 1 fi else @@ -394,9 +415,6 @@ function get_macpython_environment { # $venv_dir : {directory_name|not defined} # If defined - make virtualenv in this directory, set python / pip # commands accordingly - # $py_osx_ver: {major.minor | not defined} - # if defined, the macOS version that Python is built for, e.g. - # "10.6" or "10.9", if not defined, use the version from MB_PYTHON_OSX_VER # # Installs Python # Sets $PYTHON_EXE to path to Python executable @@ -405,14 +423,13 @@ function get_macpython_environment { # Puts directory of $PYTHON_EXE on $PATH local version=$1 local venv_dir=$2 - local py_osx_ver=${3:-$MB_PYTHON_OSX_VER} if [ "$USE_CCACHE" == "1" ]; then activate_ccache fi remove_travis_ve_pip - install_macpython $version $py_osx_ver + install_macpython $version PIP_CMD="$PYTHON_EXE -m pip" # Python 3.5 no longer compatible with latest pip if [ "$(get_py_mm)" == "3.5" ]; then @@ -486,7 +503,9 @@ function macos_arm64_cross_build_setup { sudo mkdir -p $BUILD_PREFIX/lib $BUILD_PREFIX/include sudo chown -R $USER $BUILD_PREFIX update_env_for_build_prefix - export _PYTHON_HOST_PLATFORM="macosx-13.0-arm64" + echo "BUILD_PREFIX is : $USER $BUILD_PREFIX" + sleep 120 + export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" export CFLAGS+=" -arch arm64" export CXXFLAGS+=" -arch arm64" export CPPFLAGS+=" -arch arm64" @@ -495,7 +514,7 @@ function macos_arm64_cross_build_setup { export FC=$FC_ARM64 export F90=${F90_ARM64:-${FC}} export F77=${F77_ARM64:-${FC}} - export MACOSX_DEPLOYMENT_TARGET="13.0" + export MACOSX_DEPLOYMENT_TARGET="11.0" export CROSS_COMPILING=1 export LDFLAGS+=" -arch arm64 -L$BUILD_PREFIX/lib -Wl,-rpath,$BUILD_PREFIX/lib ${FC_ARM64_LDFLAGS:-}" # This would automatically let autoconf know that we are cross compiling for arm64 darwin @@ -506,7 +525,7 @@ function macos_arm64_native_build_setup { # Setup native build for single arch arm_64 wheels export PLAT="arm64" # We don't want universal2 builds and only want an arm64 build - export _PYTHON_HOST_PLATFORM="macosx-13.0-arm64" + export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" export ARCHFLAGS+=" -arch arm64" $@ } diff --git a/multibuild/supported_wheels.py b/multibuild/supported_wheels.py index cdec034a..8d2469f3 100755 --- a/multibuild/supported_wheels.py +++ b/multibuild/supported_wheels.py @@ -1,23 +1,28 @@ #!/usr/bin/env python -""" Filter out wheel filenames not supported on this platform -""" -from __future__ import print_function +"""Filter out wheel filenames not supported on this platform.""" + +from __future__ import annotations import sys from os.path import basename +from typing import TYPE_CHECKING from packaging.tags import sys_tags +if TYPE_CHECKING: + from collections.abc import Iterator + try: - from wheel.install import WHEEL_INFO_RE as wheel_matcher + from wheel.install import WHEEL_INFO_RE as wheel_matcher # noqa: N811 except ImportError: # As of Wheel 0.32.0 from wheel.wheelfile import WHEEL_INFO_RE + wheel_matcher = WHEEL_INFO_RE.match -def tags_for(fname): - # Copied from WheelFile code - parsed_filename = wheel_matcher(basename(fname)) +def tags_for(fname: str) -> Iterator[tuple[str, str, str]]: + """Copied from WheelFile code.""" # noqa: D401 + parsed_filename = wheel_matcher(basename(fname)) # noqa: PTH119 tags = parsed_filename.groupdict() for pyver in tags['pyver'].split('.'): for abi in tags['abi'].split('.'): @@ -25,14 +30,13 @@ def tags_for(fname): yield (pyver, abi, plat) -def main(): - supported = { - (tag.interpreter, tag.abi, tag.platform) for tag in sys_tags() - } +def main() -> None: + """Print filenames of all supported wheels.""" + supported = {(tag.interpreter, tag.abi, tag.platform) for tag in sys_tags()} for fname in sys.argv[1:]: tags = set(tags_for(fname)) if supported.intersection(tags): - print(fname) + print(fname) # noqa: T201 if __name__ == '__main__': diff --git a/multibuild/tests/patches/harfbuzz-2.7.4.tar.xz.patch b/multibuild/tests/patches/harfbuzz-2.7.4.tar.xz.patch new file mode 100644 index 00000000..f4999883 --- /dev/null +++ b/multibuild/tests/patches/harfbuzz-2.7.4.tar.xz.patch @@ -0,0 +1,13 @@ +# This is a completely cosmetic patch to validate that source patching works as expected. +# +diff -ur harfbuzz-2.7.4-orig/src/gen-def.py harfbuzz-2.7.4/src/gen-def.py | sed 's/^--- harfbuzz-2.7.4-orig\//--- /; s/^+++ harfbuzz-2.7.4\//+++ /' +--- src/gen-def.py 2020-12-27 11:01:18 ++++ src/gen-def.py 2026-05-17 15:58:28 +@@ -18,6 +18,7 @@ + symbols = sorted (re.findall (r"^hb_\w+(?= \()", "\n".join (headers_content), re.M)) + if '--experimental-api' not in sys.argv: + # Move these to harfbuzz-sections.txt when got stable ++ # Harfbuzz has been patched by multibuild + experimental_symbols = \ + """hb_font_draw_glyph + hb_draw_funcs_t \ No newline at end of file diff --git a/multibuild/tests/test_common_utils.sh b/multibuild/tests/test_common_utils.sh index 57351deb..8fc97c5c 100644 --- a/multibuild/tests/test_common_utils.sh +++ b/multibuild/tests/test_common_utils.sh @@ -99,13 +99,12 @@ cmd_notexit good_cmd || ingest ! cmd_notexit bad_cmd || ingest ! cmd_notexit exit 1 || ingest -# On Linux docker containers in travis, can be x86_64, i686, s390x, ppc64le, or -# aarch64 [ "$(get_platform)" == x86_64 ] || \ [ "$(get_platform)" == i686 ] || \ [ "$(get_platform)" == aarch64 ] || \ [ "$(get_platform)" == ppc64le ] || \ [ "$(get_platform)" == s390x ] || \ + [ "$(get_platform)" == arm64 ] || \ exit 1 # Crudest possible check for get_distutils_platform diff --git a/multibuild/tests/test_fill_pyver.sh b/multibuild/tests/test_fill_pyver.sh index 627e1b1e..64b0a8e0 100644 --- a/multibuild/tests/test_fill_pyver.sh +++ b/multibuild/tests/test_fill_pyver.sh @@ -2,7 +2,9 @@ [ "$(fill_pyver 2)" == $LATEST_2p7 ] || ingest [ "$(fill_pyver 2.7)" == $LATEST_2p7 ] || ingest [ "$(fill_pyver 2.7.8)" == "2.7.8" ] || ingest -[ "$(fill_pyver 3)" == $LATEST_3p12 ] || ingest +[ "$(fill_pyver 3)" == $LATEST_3p14 ] || ingest +[ "$(fill_pyver 3.14)" == $LATEST_3p14 ] || ingest +[ "$(fill_pyver 3.14.0)" == "3.14.0" ] || ingest [ "$(fill_pyver 3.13)" == $LATEST_3p13 ] || ingest [ "$(fill_pyver 3.13.0)" == "3.13.0" ] || ingest [ "$(fill_pyver 3.12)" == $LATEST_3p12 ] || ingest diff --git a/multibuild/tests/test_fill_submodule.sh b/multibuild/tests/test_fill_submodule.sh index 740c5e61..304cede7 100644 --- a/multibuild/tests/test_fill_submodule.sh +++ b/multibuild/tests/test_fill_submodule.sh @@ -13,7 +13,7 @@ mkdir project mkdir superproject cd superproject git init -git submodule add ../project +git -c protocol.file.allow=always submodule add ../project local_author git commit -m "first superproject" # Check the submodule is working correctly before intervention diff --git a/multibuild/tests/test_library_builders.sh b/multibuild/tests/test_library_builders.sh index 65218ce1..53e30342 100644 --- a/multibuild/tests/test_library_builders.sh +++ b/multibuild/tests/test_library_builders.sh @@ -21,8 +21,10 @@ source tests/utils.sh start_spinner +PATCH_DIR=$(pwd)/tests/patches fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/2.7.4/harfbuzz-2.7.4.tar.xz [ -d harfbuzz-2.7.4 ] || ingest ".tar.xz should have been unpacked" +[ -n "$(grep 'Harfbuzz has been patched by multibuild' harfbuzz-2.7.4/src/gen-def.py)" ] || ingest "Harfbuzz should have been patched" suppress build_bzip2 suppress build_openssl @@ -34,13 +36,15 @@ suppress build_swig # E.g. arb (below) requires a couple of other libraries. # Run here just for the output, even though they fail. (set +e ; - build_github fredrik-johansson/arb 2.21.1 ; + build_github flintlib/arb 2.21.1 ; build_github glennrp/libpng v1.6.37 ; build_github wbhart/mpir mpir-3.0.0 ) suppress build_flex if [[ $MB_ML_VER != "_2_24" ]]; then + (set +e ; suppress build_openblas + ) fi suppress ensure_xz suppress build_tiff diff --git a/multibuild/tests/test_multibuild.sh b/multibuild/tests/test_multibuild.sh index f87c158d..e9854fb3 100644 --- a/multibuild/tests/test_multibuild.sh +++ b/multibuild/tests/test_multibuild.sh @@ -63,5 +63,3 @@ source tests/test_supported_wheels.sh # Exit 1 if any test errors barf -# Don't need Travis' machinery trace -set +x diff --git a/multibuild/tests/test_osx_utils.sh b/multibuild/tests/test_osx_utils.sh index 3e36bf4f..6ea44b88 100644 --- a/multibuild/tests/test_osx_utils.sh +++ b/multibuild/tests/test_osx_utils.sh @@ -15,6 +15,17 @@ [ "$(pyinst_fname_for_version 3.7.1)" == "python-3.7.1-macosx10.9.pkg" ] || ingest [ "$(pyinst_fname_for_version 3.8.0)" == "python-3.8.0-macosx10.9.pkg" ] || ingest +if [ "$(uname -m)" = "arm64" ]; then + [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macos11.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macos11.0.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macos11.pkg" ] || ingest +else + [ "$(pyinst_fname_for_version 3.8.10)" == "python-3.8.10-macosx10.9.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.1)" == "python-3.9.1-macosx10.9.pkg" ] || ingest + [ "$(pyinst_fname_for_version 3.9.2)" == "python-3.9.2-macosx10.9.pkg" ] || ingest +fi +[ "$(pyinst_fname_for_version 3.11.3)" == "python-3.11.3-macos11.pkg" ] || ingest + [ "$(pyinst_fname_for_version 2.7.14 10.6)" == "python-2.7.14-macosx10.6.pkg" ] || ingest [ "$(pyinst_fname_for_version 2.7.15 10.6)" == "python-2.7.15-macosx10.6.pkg" ] || ingest [ "$(pyinst_fname_for_version 3.6.8 10.6)" == "python-3.6.8-macosx10.6.pkg" ] || ingest @@ -25,13 +36,14 @@ # Test utilities for getting Python version versions [ "$(get_py_digit)" == "${cpython_version:0:1}" ] || ingest -[ "$(get_py_mm)" == "${cpython_version:0:3}" ] || ingest -[ "$(get_py_mm_nodot)" == $(echo "${cpython_version:0:3}" | tr -d .) ] || ingest +[ "$(get_py_mm)" == "$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}')" ] || ingest +[ "$(get_py_mm_nodot)" == $(echo "$(echo $cpython_version | awk -F "." '{printf "%d%d", $1, $2}')" | tr -d .) ] || ingest # test lookup of arch from Python macOS target build [ "$(macpython_arch_for_version 2.7 10.6)" == "intel" ] || ingest [ "$(macpython_arch_for_version 2.7 10.9)" == "x86_64" ] || ingest [ "$(macpython_arch_for_version pypy-2.7)" == "x86_64" ] || ingest +[ "$(macpython_arch_for_version 3.10 11.0)" == "arm64" ] || ingest # test lookup of arch / min macOS versions from installed Python distutils tag [ "$(get_macpython_arch macosx-10.6-intel)" == "intel" ] || ingest @@ -45,7 +57,14 @@ [ "$(macpython_impl_for_version pypy-5.4)" == "pp" ] || ingest # Test lookup of available macOS SDK build targets from python version -[ "$(macpython_sdk_list_for_version 3.8)" == "10.9" ] || ingest +[ "$(macpython_sdk_list_for_version 3.11)" == "11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.11.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9.3)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.9.0)" == "10.9" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8.10)" == "10.9 11.0" ] || ingest +[ "$(macpython_sdk_list_for_version 3.8.3)" == "10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.7.5)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.7)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 3.6.5)" == "10.6 10.9" ] || ingest @@ -57,11 +76,14 @@ [ "$(macpython_sdk_list_for_version 2.7.17)" == "10.6 10.9" ] || ingest [ "$(macpython_sdk_list_for_version 2.7.18)" == "10.9" ] || ingest -(PLAT="arm64"; [ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest) -(PLAT="universal2"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -(PLAT="x86_64"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest) -[ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest -[ "$(macpython_sdk_for_version 3.8)" == "10.9" ] || ingest +[ "$(macpython_sdk_for_version 3.11)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.11.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9.3)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.9.0)" == "10.9" ] || ingest +[ "$(macpython_sdk_for_version 3.8)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.8.10)" == "11.0" ] || ingest +[ "$(macpython_sdk_for_version 3.8.3)" == "10.9" ] || ingest [ "$(macpython_sdk_for_version 3.5)" == "10.6" ] || ingest [ "$(macpython_sdk_for_version 2.7)" == "10.9" ] || ingest [ "$(macpython_sdk_for_version 2.7.14)" == "10.6" ] || ingest @@ -74,4 +96,6 @@ function echo_host_platform { } # Make sure `_PYTHON_HOST_PLATFORM` is set when building x86_64 thin wheel -(PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ]) +if [ "$(uname -m)" != "arm64" ]; then + (PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ]) +fi diff --git a/multibuild/tests/test_python_install.sh b/multibuild/tests/test_python_install.sh index 58f8f761..ef3da366 100644 --- a/multibuild/tests/test_python_install.sh +++ b/multibuild/tests/test_python_install.sh @@ -20,7 +20,7 @@ then # CPython/PyPy version implementer_version=${BASH_REMATCH[2]:-$cpython_version} fi -python_mm="${cpython_version:0:1}.${cpython_version:2:1}" +python_mm=$(echo $cpython_version | awk -F "." '{printf "%d.%d", $1, $2}') # extract implementation prefix and version if [[ "$MB_PYTHON_VERSION" =~ (pypy[0-9\.]*-)?([0-9\.]+) ]]; then @@ -70,13 +70,3 @@ else # not virtualenv ingest "Wrong macpython or pypy pip '$PIP_CMD'" fi fi - -# check macOS version and arch are as expected -distutils_plat=$($PYTHON_EXE -c "import distutils.util; print(distutils.util.get_platform())") -expected_arch=$(macpython_arch_for_version $MB_PYTHON_VERSION) -if [[ $requested_impl == 'cp' ]]; then - expected_tag="macosx-$MB_PYTHON_OSX_VER-$expected_arch" -else - expected_tag="macosx-10.[0-9]+-$expected_arch" -fi -[[ $distutils_plat =~ $expected_tag ]] || ingest diff --git a/multibuild/tests/test_supported_wheels.sh b/multibuild/tests/test_supported_wheels.sh index 6b59a7a7..d79e680e 100644 --- a/multibuild/tests/test_supported_wheels.sh +++ b/multibuild/tests/test_supported_wheels.sh @@ -30,12 +30,13 @@ fi py_impl=$($PYTHON_EXE -c 'import platform; print(platform.python_implementation())') if [ "$py_impl" == 'CPython' ] && [ $(uname) == 'Darwin' ]; then our_ver=$($PYTHON_EXE -c 'import sys; print("{}{}".format(*sys.version_info[:2]))') + our_tag=$($PYTHON_EXE -c 'import sysconfig; print(sysconfig.get_platform().replace("-","_").replace(".","_"))') other_ver=$([ "$our_ver" == "37" ] && echo "36" || echo "37") # Python <= 3.7 needs m for API tag. api_m=$([ $our_ver -le 37 ] && echo "m") || : - whl_suff="cp${our_ver}-cp${our_ver}${api_m}-macosx_10_9_x86_64.whl" + whl_suff="cp${our_ver}-cp${our_ver}${api_m}-${our_tag}.whl" good_whl="tornado-5.1-${whl_suff}" - bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-macosx_10_9_x86_64.whl" + bad_whl="tornado-5.1-cp${other_ver}-cp${other_ver}m-${our_tag}.whl" if [ "$($PYTHON_EXE supported_wheels.py $bad_whl)" != "" ]; then echo "$bad_whl not supported, but supported wheels says it is." RET=1 @@ -56,4 +57,3 @@ $good_whl2" RET=1 fi fi - diff --git a/multibuild/travis_linux_steps.sh b/multibuild/travis_linux_steps.sh index 9f2e1e6d..cfc30e3e 100644 --- a/multibuild/travis_linux_steps.sh +++ b/multibuild/travis_linux_steps.sh @@ -125,13 +125,13 @@ function install_run { if [ "$MB_ML_LIBC" == "musllinux" ]; then # PLAT is the same as $plat, # unless $plat is "aarch64", in which case it becomes "arm64v8" - local docker_image="multibuild/alpine3.18_{PLAT}" + local docker_image="multibuild/alpine3.22_{PLAT}" elif [ "$plat" == i686 ]; then local docker_image="matthewbrett/trusty:32" else # PLAT is the same as $plat, # unless $plat is "aarch64", in which case it becomes "arm64v8" - local docker_image="multibuild/focal_{PLAT}" + local docker_image="multibuild/noble_{PLAT}" fi else local docker_image="$DOCKER_TEST_IMAGE" diff --git a/multibuild/travis_steps.sh b/multibuild/travis_steps.sh index 773183fd..a5d6a62f 100644 --- a/multibuild/travis_steps.sh +++ b/multibuild/travis_steps.sh @@ -12,6 +12,7 @@ MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") if [ ! -d "$PWD/$WHEEL_SDIR" ]; then mkdir $PWD/$WHEEL_SDIR; fi if [[ "$(uname)" == "Darwin" ]]; then + if [ ! -f "$MULTIBUILD_DIR/travis_osx_steps.sh" ]; then exit 1; fi source $MULTIBUILD_DIR/travis_osx_steps.sh else source $MULTIBUILD_DIR/travis_linux_steps.sh