From 99c99b9f2a5af83a182ec53946c6101d851b5d9e Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 13 Sep 2022 09:45:24 -1000 Subject: [PATCH 01/38] [cd build] Add wheel builds for python 3.11 --- .github/workflows/wheels.yml | 22 +++++++++++++++++++ build_tools/github/Windows | 5 +++++ .../github/build_minimal_windows_image.sh | 11 ++++++++++ pyproject.toml | 2 ++ 4 files changed, 40 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1ffa56d2705bf..746b021e91eca 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -67,6 +67,10 @@ jobs: python: 310 bitness: 64 platform_id: win_amd64 + - os: windows-latest + python: 311 + bitness: 64 + platform_id: win_amd64 # Window 32 bit - os: windows-latest @@ -97,6 +101,14 @@ jobs: platform_id: manylinux_x86_64 manylinux_image: manylinux2014 + # On Python 3.11 only supports 64bit and is only available with manylinux2014 + - os: ubuntu-latest + python: 311 + bitness: 64 + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + scipy_image: scipy-1.10.0.dev0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + # MacOS x86_64 - os: macos-latest bitness: 64 @@ -110,6 +122,10 @@ jobs: bitness: 64 python: 310 platform_id: macosx_x86_64 + - os: macos-latest + bitness: 64 + python: 311 + platform_id: macosx_x86_64 # MacOS arm64 - os: macos-latest @@ -124,6 +140,10 @@ jobs: bitness: 64 python: 310 platform_id: macosx_arm64 + - os: macos-latest + bitness: 64 + python: 311 + platform_id: macosx_arm64 steps: - name: Checkout scikit-learn @@ -142,6 +162,7 @@ jobs: OPENBLAS_NUM_THREADS=2 SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 + PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} @@ -149,6 +170,7 @@ jobs: CIBW_TEST_SKIP: "*-macosx_arm64" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_BEFORE_TEST: if [[ ${{matrix.python}} == 311 ]]; then python -m pip install scipy==1.10.0.dev0; fi CIBW_TEST_REQUIRES: pytest pandas threadpoolctl CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} diff --git a/build_tools/github/Windows b/build_tools/github/Windows index 5ba35f790ca5e..5f31d23969997 100644 --- a/build_tools/github/Windows +++ b/build_tools/github/Windows @@ -5,10 +5,15 @@ FROM winamd64/python:$PYTHON_VERSION-windowsservercore ARG WHEEL_NAME ARG CONFTEST_NAME ARG CIBW_TEST_REQUIRES +ARG PIP_EXTRA_INDEX_URL # Copy and install the Windows wheel COPY $WHEEL_NAME $WHEEL_NAME COPY $CONFTEST_NAME $CONFTEST_NAME +RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL +# TODO: Remove when 3.11 for scipy images will be available on pypa +RUN echo $env:PYTHON_VERSION +RUN if ($env:PYTHON_VERSION -match '3.11') {python -m pip install scipy==1.10.0.dev0} RUN pip install $env:WHEEL_NAME # Install the testing dependencies diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index af17c7223498d..8095873d4b41c 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -22,10 +22,21 @@ cp $WHEEL_PATH $WHEEL_NAME # Dot the Python version for identyfing the base Docker image PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) +# TODO: Remove when 3.11 images will be available for +# windows +# scipy +# pandas +if [[ "$PYTHON_VERSION" == "3.11" ]]; then + PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc) + CIBW_TEST_REQUIRES="pytest threadpoolctl" + python -m pip install scipy==1.10.0.dev0 +fi + # Build a minimal Windows Docker image for testing the wheels docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \ --build-arg WHEEL_NAME=$WHEEL_NAME \ --build-arg CONFTEST_NAME=$CONFTEST_NAME \ --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \ + --build-arg PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" \ -f build_tools/github/Windows \ -t scikit-learn/minimal-windows . diff --git a/pyproject.toml b/pyproject.toml index 9b38a78966358..d59880f008e2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ requires = [ # see: https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg "oldest-supported-numpy", + # nightly scipy 1.10.0.dev0 are the first wheels available for python 3.11 + "scipy==1.10.0.dev0; python_version>='3.11'", "scipy>=1.3.2", ] From 99f2f3698806ed8f0cf008e1688587aeb1a7c174 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 15 Sep 2022 12:26:34 -1000 Subject: [PATCH 02/38] [cd build] Fix MacOS builds setting SETUPTOOLS_USE_DISTUTILS=stdlib. --- build_tools/github/build_wheels.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/github/build_wheels.sh b/build_tools/github/build_wheels.sh index 647b47492774b..cfbb534818350 100755 --- a/build_tools/github/build_wheels.sh +++ b/build_tools/github/build_wheels.sh @@ -31,6 +31,7 @@ if [[ "$RUNNER_OS" == "macOS" ]]; then export CFLAGS="$CFLAGS -I$PREFIX/include" export CXXFLAGS="$CXXFLAGS -I$PREFIX/include" export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp" + export SETUPTOOLS_USE_DISTUTILS=stdlib fi # The version of the built dependencies are specified From a005d02629755185d903e57748696f14066b8402 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Thu, 15 Sep 2022 13:34:02 -1000 Subject: [PATCH 03/38] Remove relic. --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 746b021e91eca..3235f6b83a899 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -107,7 +107,6 @@ jobs: bitness: 64 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 - scipy_image: scipy-1.10.0.dev0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # MacOS x86_64 - os: macos-latest From 18a1807fa2c0297f144052123cbea7f06a455847 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 16 Sep 2022 20:15:02 -1000 Subject: [PATCH 04/38] [cd build] Vendor msvcp140.dll. --- build_tools/github/vendor.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py index bbc941d8f25f7..c1dfa26e7bf98 100644 --- a/build_tools/github/vendor.py +++ b/build_tools/github/vendor.py @@ -17,7 +17,7 @@ VCOMP140_SRC_PATH = "C:\\Windows\\System32\\vcomp140.dll" VCRUNTIME140_SRC_PATH = "C:\\Windows\\System32\\vcruntime140.dll" VCRUNTIME140_1_SRC_PATH = "C:\\Windows\\System32\\vcruntime140_1.dll" - +MSVCP140_SRC_PATH = "C:\\Windows\\System32\\msvcp140.dll" def make_distributor_init_32_bits( distributor_init, vcomp140_dll_filename, vcruntime140_dll_filename @@ -65,12 +65,13 @@ def make_distributor_init_64_bits( vcomp140_dll_filename, vcruntime140_dll_filename, vcruntime140_1_dll_filename, + msvcp140_dll_filename, ): """Create a _distributor_init.py file for 64-bit architectures. This file is imported first when importing the sklearn package - so as to pre-load the vendored vcomp140.dll, vcruntime140.dll - and vcruntime140_1.dll. + so as to pre-load the vendored vcomp140.dll, vcruntime140.dll, + vcruntime140_1.dll and msvcp140.dll. """ with open(distributor_init, "wt") as f: f.write( @@ -97,13 +98,16 @@ def make_distributor_init_64_bits( vcomp140_dll_filename = op.join(libs_path, "{0}") vcruntime140_dll_filename = op.join(libs_path, "{1}") vcruntime140_1_dll_filename = op.join(libs_path, "{2}") + msvcp140_dll_filename = op.join(libs_path, "{3}") WinDLL(op.abspath(vcomp140_dll_filename)) WinDLL(op.abspath(vcruntime140_dll_filename)) WinDLL(op.abspath(vcruntime140_1_dll_filename)) + WinDLL(op.abspath(msvcp140_dll_filename)) """.format( vcomp140_dll_filename, vcruntime140_dll_filename, vcruntime140_1_dll_filename, + msvcp140_dll_filename, ) ) ) @@ -120,12 +124,16 @@ def main(wheel_dirname, bitness): if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64": raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.") + if not op.exists(MSVCP140_SRC_PATH) and bitness == "64": + raise ValueError(f"Could not find {MSVCP140_SRC_PATH}.") + if not op.isdir(wheel_dirname): raise RuntimeError(f"Could not find {wheel_dirname} file.") vcomp140_dll_filename = op.basename(VCOMP140_SRC_PATH) vcruntime140_dll_filename = op.basename(VCRUNTIME140_SRC_PATH) vcruntime140_1_dll_filename = op.basename(VCRUNTIME140_1_SRC_PATH) + msvcp140_dll_filename = op.basename(MSVCP140_SRC_PATH) target_folder = op.join(wheel_dirname, TARGET_FOLDER) distributor_init = op.join(wheel_dirname, DISTRIBUTOR_INIT) @@ -144,6 +152,9 @@ def main(wheel_dirname, bitness): print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.") shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder) + print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.") + shutil.copy2(MSVCP140_SRC_PATH, target_folder) + # Generate the _distributor_init file in the source tree print("Generating the '_distributor_init.py' file.") if bitness == "32": @@ -156,6 +167,7 @@ def main(wheel_dirname, bitness): vcomp140_dll_filename, vcruntime140_dll_filename, vcruntime140_1_dll_filename, + msvcp140_dll_filename ) From f7923b158628d009acbcecb446a48304d14c02dd Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 16 Sep 2022 20:30:34 -1000 Subject: [PATCH 05/38] Apply black --- build_tools/github/vendor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py index c1dfa26e7bf98..c93d1e9055490 100644 --- a/build_tools/github/vendor.py +++ b/build_tools/github/vendor.py @@ -17,7 +17,8 @@ VCOMP140_SRC_PATH = "C:\\Windows\\System32\\vcomp140.dll" VCRUNTIME140_SRC_PATH = "C:\\Windows\\System32\\vcruntime140.dll" VCRUNTIME140_1_SRC_PATH = "C:\\Windows\\System32\\vcruntime140_1.dll" -MSVCP140_SRC_PATH = "C:\\Windows\\System32\\msvcp140.dll" +MSVCP140_SRC_PATH = "C:\\Windows\\System32\\msvcp140.dll" + def make_distributor_init_32_bits( distributor_init, vcomp140_dll_filename, vcruntime140_dll_filename @@ -167,7 +168,7 @@ def main(wheel_dirname, bitness): vcomp140_dll_filename, vcruntime140_dll_filename, vcruntime140_1_dll_filename, - msvcp140_dll_filename + msvcp140_dll_filename, ) From 6add9af99e9e157c557d21bf7f83e17ab0a42bd7 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 23 Sep 2022 07:59:38 -1000 Subject: [PATCH 06/38] [cd build] Trigger build. From a48568bcbdd0b9eb2be113775f1e43a2039179df Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 7 Oct 2022 12:29:50 +0200 Subject: [PATCH 07/38] [cd build] Trigger CD since #24541 is merged From 2f411a9c73808235f9a65b36fda786f5498a98c9 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 7 Oct 2022 20:28:46 -1000 Subject: [PATCH 08/38] Apply suggestions from code review Co-authored-by: Julien Jerphanion --- .github/workflows/wheels.yml | 2 +- build_tools/github/build_wheels.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b5a6856249172..0013ac315cd6c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -101,7 +101,7 @@ jobs: platform_id: manylinux_x86_64 manylinux_image: manylinux2014 - # On Python 3.11 only supports 64bit and is only available with manylinux2014 + # NumPy on Python 3.11 only supports 64bit and is only available with manylinux2014 - os: ubuntu-latest python: 311 bitness: 64 diff --git a/build_tools/github/build_wheels.sh b/build_tools/github/build_wheels.sh index cfbb534818350..f14a8645c85aa 100755 --- a/build_tools/github/build_wheels.sh +++ b/build_tools/github/build_wheels.sh @@ -31,6 +31,10 @@ if [[ "$RUNNER_OS" == "macOS" ]]; then export CFLAGS="$CFLAGS -I$PREFIX/include" export CXXFLAGS="$CXXFLAGS -I$PREFIX/include" export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp" + # Disable the use of setuptools's vendored copy distutils when invoking setuptools + # See: https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html + # TODO: remove the definition of this environment variable when no + # reference to distutils exist in the code-base for building scikit-learn. export SETUPTOOLS_USE_DISTUTILS=stdlib fi From 7e3af0a36143af2e7390c373da73b5c096639739 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 7 Oct 2022 20:48:04 -1000 Subject: [PATCH 09/38] Clarify comments. --- build_tools/github/build_minimal_windows_image.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 8095873d4b41c..ad73b906af22d 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -23,9 +23,10 @@ cp $WHEEL_PATH $WHEEL_NAME PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) # TODO: Remove when 3.11 images will be available for -# windows -# scipy -# pandas +# windows (for now the docker image is tagged as 3.11-rc) +# TODO: Remove when 3.11 images will be available for +# scipy (for now only available from nightly anaconda index) +# pandas (no pandas 3.11 wheels for now) if [[ "$PYTHON_VERSION" == "3.11" ]]; then PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc) CIBW_TEST_REQUIRES="pytest threadpoolctl" From 4c47cb61e1dce29354ebdbd12b4127988c1b0d9e Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 00:16:28 -1000 Subject: [PATCH 10/38] [cd build] Install pandas wheels on windows 64 (available since the end of September). --- build_tools/github/build_minimal_windows_image.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index ad73b906af22d..80acf4c42534a 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -26,10 +26,8 @@ PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) # windows (for now the docker image is tagged as 3.11-rc) # TODO: Remove when 3.11 images will be available for # scipy (for now only available from nightly anaconda index) -# pandas (no pandas 3.11 wheels for now) if [[ "$PYTHON_VERSION" == "3.11" ]]; then PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc) - CIBW_TEST_REQUIRES="pytest threadpoolctl" python -m pip install scipy==1.10.0.dev0 fi From 00dbfd855c38f1af1dcc2f5ae644536b71407468 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 15:45:50 -1000 Subject: [PATCH 11/38] [cd build] scipy 1.9.2 wheels for 3.11 published on pypi (except for windows). --- .github/workflows/wheels.yml | 1 - build_tools/github/Windows | 2 +- pyproject.toml | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0013ac315cd6c..18f1c4ec51d5d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -169,7 +169,6 @@ jobs: CIBW_TEST_SKIP: "*-macosx_arm64" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} - CIBW_BEFORE_TEST: if [[ ${{matrix.python}} == 311 ]]; then python -m pip install scipy==1.10.0.dev0; fi CIBW_TEST_REQUIRES: pytest pandas threadpoolctl CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} diff --git a/build_tools/github/Windows b/build_tools/github/Windows index 5f31d23969997..7ed60662487bf 100644 --- a/build_tools/github/Windows +++ b/build_tools/github/Windows @@ -11,7 +11,7 @@ ARG PIP_EXTRA_INDEX_URL COPY $WHEEL_NAME $WHEEL_NAME COPY $CONFTEST_NAME $CONFTEST_NAME RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL -# TODO: Remove when 3.11 for scipy images will be available on pypa +# TODO: Remove when 3.11 scipy wheels for windows will be available on pypa RUN echo $env:PYTHON_VERSION RUN if ($env:PYTHON_VERSION -match '3.11') {python -m pip install scipy==1.10.0.dev0} RUN pip install $env:WHEEL_NAME diff --git a/pyproject.toml b/pyproject.toml index d59880f008e2b..9b38a78966358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,6 @@ requires = [ # see: https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg "oldest-supported-numpy", - # nightly scipy 1.10.0.dev0 are the first wheels available for python 3.11 - "scipy==1.10.0.dev0; python_version>='3.11'", "scipy>=1.3.2", ] From cbbf87e5c9925ec7ffab85cfbe2f71492ec97c2f Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 16:06:03 -1000 Subject: [PATCH 12/38] [cd build] scipy 1.9.2 wheels for 3.11 published on pypi. --- build_tools/github/Windows | 3 --- 1 file changed, 3 deletions(-) diff --git a/build_tools/github/Windows b/build_tools/github/Windows index 7ed60662487bf..f197d9300c21c 100644 --- a/build_tools/github/Windows +++ b/build_tools/github/Windows @@ -11,9 +11,6 @@ ARG PIP_EXTRA_INDEX_URL COPY $WHEEL_NAME $WHEEL_NAME COPY $CONFTEST_NAME $CONFTEST_NAME RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL -# TODO: Remove when 3.11 scipy wheels for windows will be available on pypa -RUN echo $env:PYTHON_VERSION -RUN if ($env:PYTHON_VERSION -match '3.11') {python -m pip install scipy==1.10.0.dev0} RUN pip install $env:WHEEL_NAME # Install the testing dependencies From b77b6a3866013cc76670ab262bf500595dc8b367 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:08:24 -1000 Subject: [PATCH 13/38] [cd build] force scipy version for win32. --- build_tools/github/build_minimal_windows_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 80acf4c42534a..5ad7b144391c8 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -10,6 +10,7 @@ if [[ "$BITNESS" == "32" ]]; then # 32-bit architectures are not supported # by the official Docker images: Tests will just be run # on the host (instead of the minimal Docker container). + python -m pip install 'scipy<=1.9.1' exit 0 fi @@ -28,7 +29,6 @@ PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) # scipy (for now only available from nightly anaconda index) if [[ "$PYTHON_VERSION" == "3.11" ]]; then PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc) - python -m pip install scipy==1.10.0.dev0 fi # Build a minimal Windows Docker image for testing the wheels From 40c1d037edccaae9920bc193fe87fcc0e4e9264d Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:21:22 -1000 Subject: [PATCH 14/38] [cd build] force scipy version for win32. --- .github/workflows/wheels.yml | 1 + build_tools/github/build_minimal_windows_image.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 18f1c4ec51d5d..02fbd1c860999 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,6 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" + CIBW_BEFORE_BUILD_WINDOWS: if [[ ${{matrix.platform_id}} == win32 ]]; then python -m pip install 'scipy<=1.9.1'; fi CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 5ad7b144391c8..77fcb9240e73b 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -10,7 +10,6 @@ if [[ "$BITNESS" == "32" ]]; then # 32-bit architectures are not supported # by the official Docker images: Tests will just be run # on the host (instead of the minimal Docker container). - python -m pip install 'scipy<=1.9.1' exit 0 fi From 45cee88bf9894d4b983173e41321f54c0484844d Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:33:19 -1000 Subject: [PATCH 15/38] [cd build] force scipy version for win32. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 02fbd1c860999..55a34f690e942 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: if [[ ${{matrix.platform_id}} == win32 ]]; then python -m pip install 'scipy<=1.9.1'; fi + CIBW_BEFORE_BUILD_WINDOWS: if ( '${{matrix.platform_id}}' -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From 18169330e3a2ede30c9f8b83b1994f8f897909d6 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:40:23 -1000 Subject: [PATCH 16/38] [cd build] force scipy version for win32. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 55a34f690e942..171d5c986c912 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: if ( '${{matrix.platform_id}}' -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} + CIBW_BEFORE_BUILD_WINDOWS: if ( "${{matrix.platform_id}}" -match "win32" ) {python -m pip install 'scipy<=1.9.1'} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From 26a41abdf153f8f2d426baa149efbea3f3dabfc5 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:50:47 -1000 Subject: [PATCH 17/38] [cd build] Experimenting with PowerShell. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 171d5c986c912..46f7d4d623c6b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: if ( "${{matrix.platform_id}}" -match "win32" ) {python -m pip install 'scipy<=1.9.1'} + CIBW_BEFORE_BUILD_WINDOWS: if ( $env:${{matrix.platform_id}} -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From 18eac63b2ac3723de8f3403f0759c61b7e618f0e Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 21:57:42 -1000 Subject: [PATCH 18/38] [cd build] Experimenting with PowerShell. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 46f7d4d623c6b..3b93ea8e0e9b2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: if ( $env:${{matrix.platform_id}} -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} + CIBW_BEFORE_BUILD_WINDOWS: set platform=${{matrix.platform_id}} ; if ( %platform -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From 0c4e80d89e08b2b84b980a4685779e34f4d94d67 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 22:08:12 -1000 Subject: [PATCH 19/38] [cd build] Experimenting with PowerShell. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3b93ea8e0e9b2..81b9bc3cb1f9c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: set platform=${{matrix.platform_id}} ; if ( %platform -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} + CIBW_BEFORE_BUILD_WINDOWS: set platform=${{matrix.platform_id}} ; if ( $env:platform -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From c51315e23901e70cd75e8e7baf118c5edf157d4b Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 22:25:28 -1000 Subject: [PATCH 20/38] [cd build] Experimenting with PowerShell. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 81b9bc3cb1f9c..95ddeac9cd729 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: set platform=${{matrix.platform_id}} ; if ( $env:platform -match 'win32' ) {python -m pip install 'scipy<=1.9.1'} + CIBW_BEFORE_BUILD_WINDOWS: bash -c "if [[ ${{matrix.platform_id}} == 'win32' ]]; then python -m pip install 'scipy<=1.9.1'; fi" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From fca7ca606368e2458c8f68478f75e92204c5f14e Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 22:42:25 -1000 Subject: [PATCH 21/38] [cd build] Set dependence in pyproject.toml. --- .github/workflows/wheels.yml | 1 - pyproject.toml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 95ddeac9cd729..18f1c4ec51d5d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,7 +167,6 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: bash -c "if [[ ${{matrix.platform_id}} == 'win32' ]]; then python -m pip install 'scipy<=1.9.1'; fi" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl diff --git a/pyproject.toml b/pyproject.toml index 9b38a78966358..0279ee6ca72ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ requires = [ # see: https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg "oldest-supported-numpy", + # the last scipy wheel available for win32 is 1.9.1 + "scipy<=1.9.1; sys_platform=='win32'", "scipy>=1.3.2", ] From 65dbb5920b8a4eda99cebb8397b831a6d61c2bf8 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 23:01:09 -1000 Subject: [PATCH 22/38] [cd build] Set dependence in pyproject.toml. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0279ee6ca72ea..df51eb800c79e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = [ "oldest-supported-numpy", # the last scipy wheel available for win32 is 1.9.1 - "scipy<=1.9.1; sys_platform=='win32'", + "scipy<=1.9.1; platform_machine=='win32'", "scipy>=1.3.2", ] From f0938dbee01864135d5bb546a831e53957db4663 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sat, 8 Oct 2022 23:13:40 -1000 Subject: [PATCH 23/38] [cd build] Set dependence in pyproject.toml. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index df51eb800c79e..34cd4b3006cde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = [ "oldest-supported-numpy", # the last scipy wheel available for win32 is 1.9.1 - "scipy<=1.9.1; platform_machine=='win32'", + "scipy<=1.9.1; platform_system=='win32'", "scipy>=1.3.2", ] From b4b6f5f85095f80285b13a01a3b8c0cf839444dc Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 11:40:35 -1000 Subject: [PATCH 24/38] [cd build] Install scipy 1.9.1 for tests. --- build_tools/github/build_minimal_windows_image.sh | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 77fcb9240e73b..997faa300bb54 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -10,6 +10,8 @@ if [[ "$BITNESS" == "32" ]]; then # 32-bit architectures are not supported # by the official Docker images: Tests will just be run # on the host (instead of the minimal Docker container). + # the last scipy wheel available for win32 is 1.9.1 + python -m pip install 'scipy<=1.9.1' exit 0 fi diff --git a/pyproject.toml b/pyproject.toml index 34cd4b3006cde..6bf95fd610c3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = [ "oldest-supported-numpy", # the last scipy wheel available for win32 is 1.9.1 - "scipy<=1.9.1; platform_system=='win32'", + "scipy<=1.9.1; sys_platform == 'win32'", "scipy>=1.3.2", ] From 2d2baed5bcd5699cd7d0af3637c708756c24c315 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 13:49:03 -1000 Subject: [PATCH 25/38] [cd build] Using platform_machine. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bf95fd610c3a..9b45f0faa55b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ requires = [ # see: https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg "oldest-supported-numpy", - # the last scipy wheel available for win32 is 1.9.1 - "scipy<=1.9.1; sys_platform == 'win32'", "scipy>=1.3.2", + # the last scipy wheel available for win32 is 1.9.1 + "scipy<=1.9.1; platform_machine == 'win32'", ] [tool.black] From 2d1cd7faaa7f0df3524b253d22245683ba71c7cb Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 13:57:50 -1000 Subject: [PATCH 26/38] [cd build] No way to use pyproject.toml deendencies. --- .github/workflows/wheels.yml | 1 + pyproject.toml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 18f1c4ec51d5d..95ddeac9cd729 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -167,6 +167,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" + CIBW_BEFORE_BUILD_WINDOWS: bash -c "if [[ ${{matrix.platform_id}} == 'win32' ]]; then python -m pip install 'scipy<=1.9.1'; fi" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl diff --git a/pyproject.toml b/pyproject.toml index 9b45f0faa55b8..9b38a78966358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,6 @@ requires = [ "oldest-supported-numpy", "scipy>=1.3.2", - # the last scipy wheel available for win32 is 1.9.1 - "scipy<=1.9.1; platform_machine == 'win32'", ] [tool.black] From 82a21b664b13bea9e83e6173251c87fe5d9f8b1f Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 15:05:22 -1000 Subject: [PATCH 27/38] [cd build] Prefer binaries when building. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 95ddeac9cd729..1548c9a804aa6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -163,11 +163,11 @@ jobs: SKLEARN_BUILD_PARALLEL=3 PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_CONFIG_SETTINGS: "--build-option=--prefer-binary" CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BEFORE_BUILD_WINDOWS: bash -c "if [[ ${{matrix.platform_id}} == 'win32' ]]; then python -m pip install 'scipy<=1.9.1'; fi" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl From 33fe747ae6c1e0245d253214d86dc68a8aa4d946 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 15:16:43 -1000 Subject: [PATCH 28/38] [cd build] Prefer binaries when building (via PIP_BUILD_OPTIONS). --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1548c9a804aa6..029a17bfb2c55 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -162,6 +162,7 @@ jobs: SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple + PIP_BUILD_OPTION=--prefer-binary CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_CONFIG_SETTINGS: "--build-option=--prefer-binary" CIBW_ARCHS: all From 995043503150afef8c8d4bc962f1cde87d61063c Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 15:31:58 -1000 Subject: [PATCH 29/38] [cd build] Prefer binaries when building (via PIP_BUILD_OPTIONS). --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 029a17bfb2c55..9b999f3379ffd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -164,7 +164,6 @@ jobs: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple PIP_BUILD_OPTION=--prefer-binary CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} - CIBW_CONFIG_SETTINGS: "--build-option=--prefer-binary" CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} From fd2371fe8e93b98cf912279e1c86e604088f508f Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Sun, 9 Oct 2022 15:44:02 -1000 Subject: [PATCH 30/38] [ci skip] Revert pip build options. --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9b999f3379ffd..18f1c4ec51d5d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -162,7 +162,6 @@ jobs: SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple - PIP_BUILD_OPTION=--prefer-binary CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} From 21e769d39527e615f1f36f58bddf5834c2f4494d Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 11 Oct 2022 10:28:15 -1000 Subject: [PATCH 31/38] [cd build] Trig wheel build. From 3210043727765975d1bdc2e5d258e410f2f57030 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 11 Oct 2022 11:17:44 -1000 Subject: [PATCH 32/38] [cd build] Add condition on extra index. --- .github/workflows/wheels.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 18f1c4ec51d5d..cb1f803e2defd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -153,6 +153,11 @@ jobs: with: python-version: '3.9' # update once build dependencies are available + - name: Set pip extra index + if: github.event_name == 'schedule' + env: + PIP_EXTRA_INDEX: https://pypi.anaconda.org/scipy-wheels-nightly/simple + - name: Build and test wheels env: CONFTEST_PATH: ${{ github.workspace }}/conftest.py @@ -161,7 +166,7 @@ jobs: OPENBLAS_NUM_THREADS=2 SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 - PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scipy-wheels-nightly/simple + PIP_EXTRA_INDEX_URL=${{ env.PIP_EXTRA_INDEX }} CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} From 657b563640dfa91a688ece4c1ab2c1c17e98fc2c Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 11 Oct 2022 11:22:02 -1000 Subject: [PATCH 33/38] [cd build] Add condition on extra index. --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index cb1f803e2defd..e8dbf42e40046 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -157,6 +157,7 @@ jobs: if: github.event_name == 'schedule' env: PIP_EXTRA_INDEX: https://pypi.anaconda.org/scipy-wheels-nightly/simple + run: echo ${{ env.PIP_EXTRA_INDEX }} - name: Build and test wheels env: From 6db7e7c791ba283f350bc2474e93a7bdb6f34604 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 11 Oct 2022 14:21:17 -1000 Subject: [PATCH 34/38] Reformat condition. Define PIP_PRE. --- .github/workflows/wheels.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e8dbf42e40046..ca2418ff3c3b4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -153,11 +153,14 @@ jobs: with: python-version: '3.9' # update once build dependencies are available - - name: Set pip extra index + - name: Set pip extra options if: github.event_name == 'schedule' env: PIP_EXTRA_INDEX: https://pypi.anaconda.org/scipy-wheels-nightly/simple - run: echo ${{ env.PIP_EXTRA_INDEX }} + PIP_PRE: 1 + run: | + echo "PIP_EXTRA_INDEX=${{ env.PIP_EXTRA_INDEX }}" >> $GITHUB_ENV + echo "PIP_PRE=${{ env.PIP_PRE }}" >> $GITHUB_ENV - name: Build and test wheels env: @@ -168,6 +171,7 @@ jobs: SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 PIP_EXTRA_INDEX_URL=${{ env.PIP_EXTRA_INDEX }} + PIP_PRE=${{ env.PIP_PRE }} CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} From ed2c6a07ad66d3356e730b3312c58fef0bcf9d3e Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Tue, 11 Oct 2022 19:48:39 -1000 Subject: [PATCH 35/38] Update .github/workflows/wheels.yml Co-authored-by: Thomas J. Fan --- .github/workflows/wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ca2418ff3c3b4..6a6e1a1cf35e2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -159,8 +159,9 @@ jobs: PIP_EXTRA_INDEX: https://pypi.anaconda.org/scipy-wheels-nightly/simple PIP_PRE: 1 run: | - echo "PIP_EXTRA_INDEX=${{ env.PIP_EXTRA_INDEX }}" >> $GITHUB_ENV - echo "PIP_PRE=${{ env.PIP_PRE }}" >> $GITHUB_ENV + # Test with nightly dependencies for nightly builds + echo "PIP_EXTRA_INDEX=https://pypi.anaconda.org/scipy-wheels-nightly/simple" >> $GITHUB_ENV + echo "PIP_PRE=1" >> $GITHUB_ENV - name: Build and test wheels env: From 5a43727bf03d5e357e3be96c1e8feb87bc21a1d7 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Fri, 14 Oct 2022 09:03:39 -1000 Subject: [PATCH 36/38] [cd build gh] Update vendor.py, remove bitness argument, add PIP_PRE variable. --- .github/workflows/wheels.yml | 25 ++---------- build_tools/github/Windows | 2 + .../github/build_minimal_windows_image.sh | 11 +----- build_tools/github/repair_windows_wheels.sh | 3 +- build_tools/github/test_windows_wheels.sh | 1 - build_tools/github/vendor.py | 39 ++++++++----------- 6 files changed, 23 insertions(+), 58 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a457b68bb6b76..00f63b667fbf2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,80 +57,64 @@ jobs: # https://github.com/scikit-learn/scikit-learn/issues/22530 - os: windows-2019 python: 38 - bitness: 64 platform_id: win_amd64 - os: windows-latest python: 39 - bitness: 64 platform_id: win_amd64 - os: windows-latest python: 310 - bitness: 64 platform_id: win_amd64 - os: windows-latest python: 311 - bitness: 64 platform_id: win_amd64 # Linux 64 bit manylinux2014 - os: ubuntu-latest python: 38 - bitness: 64 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 - os: ubuntu-latest python: 39 - bitness: 64 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014 - os: ubuntu-latest python: 310 - bitness: 64 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 # NumPy on Python 3.11 only supports 64bit and is only available with manylinux2014 - os: ubuntu-latest python: 311 - bitness: 64 platform_id: manylinux_x86_64 manylinux_image: manylinux2014 # MacOS x86_64 - os: macos-latest - bitness: 64 python: 38 platform_id: macosx_x86_64 - os: macos-latest - bitness: 64 python: 39 platform_id: macosx_x86_64 - os: macos-latest - bitness: 64 python: 310 platform_id: macosx_x86_64 - os: macos-latest - bitness: 64 python: 311 platform_id: macosx_x86_64 # MacOS arm64 - os: macos-latest - bitness: 64 python: 38 platform_id: macosx_arm64 - os: macos-latest - bitness: 64 python: 39 platform_id: macosx_arm64 - os: macos-latest - bitness: 64 python: 310 platform_id: macosx_arm64 - os: macos-latest - bitness: 64 python: 311 platform_id: macosx_arm64 @@ -145,9 +129,6 @@ jobs: - name: Set pip extra options if: github.event_name == 'schedule' - env: - PIP_EXTRA_INDEX: https://pypi.anaconda.org/scipy-wheels-nightly/simple - PIP_PRE: 1 run: | # Test with nightly dependencies for nightly builds echo "PIP_EXTRA_INDEX=https://pypi.anaconda.org/scipy-wheels-nightly/simple" >> $GITHUB_ENV @@ -168,11 +149,11 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} - CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} + CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} CIBW_TEST_REQUIRES: pytest pandas threadpoolctl CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh - CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }} + CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} CIBW_BUILD_VERBOSITY: 1 run: bash build_tools/github/build_wheels.sh diff --git a/build_tools/github/Windows b/build_tools/github/Windows index f197d9300c21c..93f6a6dfd0967 100644 --- a/build_tools/github/Windows +++ b/build_tools/github/Windows @@ -6,11 +6,13 @@ ARG WHEEL_NAME ARG CONFTEST_NAME ARG CIBW_TEST_REQUIRES ARG PIP_EXTRA_INDEX_URL +ARG PIP_PRE # Copy and install the Windows wheel COPY $WHEEL_NAME $WHEEL_NAME COPY $CONFTEST_NAME $CONFTEST_NAME RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL +RUN echo PIP_PRE = $env:PIP_PRE RUN pip install $env:WHEEL_NAME # Install the testing dependencies diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 997faa300bb54..31d1b967b0e42 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -4,16 +4,6 @@ set -e set -x PYTHON_VERSION=$1 -BITNESS=$2 - -if [[ "$BITNESS" == "32" ]]; then - # 32-bit architectures are not supported - # by the official Docker images: Tests will just be run - # on the host (instead of the minimal Docker container). - # the last scipy wheel available for win32 is 1.9.1 - python -m pip install 'scipy<=1.9.1' - exit 0 -fi TEMP_FOLDER="$HOME/AppData/Local/Temp" WHEEL_PATH=$(ls -d $TEMP_FOLDER/**/*/repaired_wheel/*) @@ -38,5 +28,6 @@ docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \ --build-arg CONFTEST_NAME=$CONFTEST_NAME \ --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \ --build-arg PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" \ + --build-arg PIP_PRE="$PIP_PRE" \ -f build_tools/github/Windows \ -t scikit-learn/minimal-windows . diff --git a/build_tools/github/repair_windows_wheels.sh b/build_tools/github/repair_windows_wheels.sh index de564fc177c89..cdd0c0c79d8c4 100755 --- a/build_tools/github/repair_windows_wheels.sh +++ b/build_tools/github/repair_windows_wheels.sh @@ -5,12 +5,11 @@ set -x WHEEL=$1 DEST_DIR=$2 -BITNESS=$3 # By default, the Windows wheels are not repaired. # In this case, we need to vendor VCRUNTIME140.dll wheel unpack "$WHEEL" WHEEL_DIRNAME=$(ls -d scikit_learn-*) -python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS" +python build_tools/github/vendor.py "$WHEEL_DIRNAME" wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR" rm -rf "$WHEEL_DIRNAME" diff --git a/build_tools/github/test_windows_wheels.sh b/build_tools/github/test_windows_wheels.sh index 38875b159506c..43a1a283e652c 100755 --- a/build_tools/github/test_windows_wheels.sh +++ b/build_tools/github/test_windows_wheels.sh @@ -4,7 +4,6 @@ set -e set -x PYTHON_VERSION=$1 -BITNESS=$2 docker container run \ --rm scikit-learn/minimal-windows \ diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py index c93d1e9055490..eeacff04c9173 100644 --- a/build_tools/github/vendor.py +++ b/build_tools/github/vendor.py @@ -114,7 +114,7 @@ def make_distributor_init_64_bits( ) -def main(wheel_dirname, bitness): +def main(wheel_dirname): """Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll.""" if not op.exists(VCOMP140_SRC_PATH): raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.") @@ -122,10 +122,10 @@ def main(wheel_dirname, bitness): if not op.exists(VCRUNTIME140_SRC_PATH): raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.") - if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64": + if not op.exists(VCRUNTIME140_1_SRC_PATH): raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.") - if not op.exists(MSVCP140_SRC_PATH) and bitness == "64": + if not op.exists(MSVCP140_SRC_PATH): raise ValueError(f"Could not find {MSVCP140_SRC_PATH}.") if not op.isdir(wheel_dirname): @@ -149,29 +149,22 @@ def main(wheel_dirname, bitness): print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.") shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder) - if bitness == "64": - print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.") - shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder) + print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.") + shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder) - print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.") - shutil.copy2(MSVCP140_SRC_PATH, target_folder) + print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.") + shutil.copy2(MSVCP140_SRC_PATH, target_folder) # Generate the _distributor_init file in the source tree - print("Generating the '_distributor_init.py' file.") - if bitness == "32": - make_distributor_init_32_bits( - distributor_init, vcomp140_dll_filename, vcruntime140_dll_filename - ) - else: - make_distributor_init_64_bits( - distributor_init, - vcomp140_dll_filename, - vcruntime140_dll_filename, - vcruntime140_1_dll_filename, - msvcp140_dll_filename, - ) + make_distributor_init_64_bits( + distributor_init, + vcomp140_dll_filename, + vcruntime140_dll_filename, + vcruntime140_1_dll_filename, + msvcp140_dll_filename, + ) if __name__ == "__main__": - _, wheel_file, bitness = sys.argv - main(wheel_file, bitness) + _, wheel_file = sys.argv + main(wheel_file) From 75e53ebe45730683a6d410d97638527703f56824 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Mon, 17 Oct 2022 12:01:46 -1000 Subject: [PATCH 37/38] [cd build] Add 3.11 wheel for arm64-graviton on travis. --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4d917a617e0f3..4f0bd8def013e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,16 @@ jobs: - CIBW_BUILD=cp310-manylinux_aarch64 - BUILD_WHEEL=true + - os: linux + arch: arm64-graviton2 + dist: focal + virt: vm + group: edge + if: type = cron or commit_message =~ /\[cd build\]/ + env: + - CIBW_BUILD=cp311-manylinux_aarch64 + - BUILD_WHEEL=true + install: source build_tools/travis/install.sh || travis_terminate 1 script: source build_tools/travis/script.sh || travis_terminate 1 after_success: source build_tools/travis/after_success.sh || travis_terminate 1 From 1c50c26a971a14917e8d7d15c33259f87fb618d6 Mon Sep 17 00:00:00 2001 From: Chiara Marmo Date: Wed, 19 Oct 2022 23:22:53 -1000 Subject: [PATCH 38/38] [cd build] Address comments. --- .github/workflows/wheels.yml | 10 ---------- build_tools/github/Windows | 4 ---- build_tools/github/build_minimal_windows_image.sh | 4 ---- build_tools/github/vendor.py | 1 + 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 00f63b667fbf2..49da927d67178 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -84,7 +84,6 @@ jobs: platform_id: manylinux_x86_64 manylinux_image: manylinux2014 - # NumPy on Python 3.11 only supports 64bit and is only available with manylinux2014 - os: ubuntu-latest python: 311 platform_id: manylinux_x86_64 @@ -127,13 +126,6 @@ jobs: with: python-version: '3.9' # update once build dependencies are available - - name: Set pip extra options - if: github.event_name == 'schedule' - run: | - # Test with nightly dependencies for nightly builds - echo "PIP_EXTRA_INDEX=https://pypi.anaconda.org/scipy-wheels-nightly/simple" >> $GITHUB_ENV - echo "PIP_PRE=1" >> $GITHUB_ENV - - name: Build and test wheels env: CONFTEST_PATH: ${{ github.workspace }}/conftest.py @@ -142,8 +134,6 @@ jobs: OPENBLAS_NUM_THREADS=2 SKLEARN_SKIP_NETWORK_TESTS=1 SKLEARN_BUILD_PARALLEL=3 - PIP_EXTRA_INDEX_URL=${{ env.PIP_EXTRA_INDEX }} - PIP_PRE=${{ env.PIP_PRE }} CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} CIBW_ARCHS: all CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} diff --git a/build_tools/github/Windows b/build_tools/github/Windows index 93f6a6dfd0967..5ba35f790ca5e 100644 --- a/build_tools/github/Windows +++ b/build_tools/github/Windows @@ -5,14 +5,10 @@ FROM winamd64/python:$PYTHON_VERSION-windowsservercore ARG WHEEL_NAME ARG CONFTEST_NAME ARG CIBW_TEST_REQUIRES -ARG PIP_EXTRA_INDEX_URL -ARG PIP_PRE # Copy and install the Windows wheel COPY $WHEEL_NAME $WHEEL_NAME COPY $CONFTEST_NAME $CONFTEST_NAME -RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL -RUN echo PIP_PRE = $env:PIP_PRE RUN pip install $env:WHEEL_NAME # Install the testing dependencies diff --git a/build_tools/github/build_minimal_windows_image.sh b/build_tools/github/build_minimal_windows_image.sh index 31d1b967b0e42..e43bf65b3bca5 100755 --- a/build_tools/github/build_minimal_windows_image.sh +++ b/build_tools/github/build_minimal_windows_image.sh @@ -16,8 +16,6 @@ PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2}) # TODO: Remove when 3.11 images will be available for # windows (for now the docker image is tagged as 3.11-rc) -# TODO: Remove when 3.11 images will be available for -# scipy (for now only available from nightly anaconda index) if [[ "$PYTHON_VERSION" == "3.11" ]]; then PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc) fi @@ -27,7 +25,5 @@ docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \ --build-arg WHEEL_NAME=$WHEEL_NAME \ --build-arg CONFTEST_NAME=$CONFTEST_NAME \ --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \ - --build-arg PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" \ - --build-arg PIP_PRE="$PIP_PRE" \ -f build_tools/github/Windows \ -t scikit-learn/minimal-windows . diff --git a/build_tools/github/vendor.py b/build_tools/github/vendor.py index eeacff04c9173..92e5fe6a8f205 100644 --- a/build_tools/github/vendor.py +++ b/build_tools/github/vendor.py @@ -156,6 +156,7 @@ def main(wheel_dirname): shutil.copy2(MSVCP140_SRC_PATH, target_folder) # Generate the _distributor_init file in the source tree + print("Generating the '_distributor_init.py' file.") make_distributor_init_64_bits( distributor_init, vcomp140_dll_filename,