CI Add wheel builds for Python 3.11

.github/workflows/wheels.yml

@@ -57,63 +57,66 @@ jobs:
           # https://github.com/scikit-learn/scikit-learn/issues/22530
           - os: windows-2019
             python: 38
-            bitness: 64
             platform_id: win_amd64
           - os: windows-latest
             python: 39
-            bitness: 64
             platform_id: win_amd64
           - os: windows-latest
             python: 310
-            bitness: 64
+            platform_id: win_amd64
+          - os: windows-latest
+            python: 311
             platform_id: win_amd64
 
           # Linux 64 bit manylinux2014
           - os: ubuntu-latest
             python: 38
-            bitness: 64
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
           - os: ubuntu-latest
             python: 39
-            bitness: 64
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
 
           # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014
           - os: ubuntu-latest
             python: 310
-            bitness: 64
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+
+          # NumPy on Python 3.11 only supports 64bit and is only available with manylinux2014
+          - os: ubuntu-latest
+            python: 311
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
 
           # MacOS x86_64
           - os: macos-latest
-            bitness: 64
             python: 38
             platform_id: macosx_x86_64
           - os: macos-latest
-            bitness: 64
             python: 39
             platform_id: macosx_x86_64
           - os: macos-latest
-            bitness: 64
             python: 310
             platform_id: macosx_x86_64
+          - os: macos-latest
+            python: 311
+            platform_id: macosx_x86_64
 
           # MacOS arm64
           - os: macos-latest
-            bitness: 64
             python: 38
             platform_id: macosx_arm64
           - os: macos-latest
-            bitness: 64
             python: 39
             platform_id: macosx_arm64
           - os: macos-latest
-            bitness: 64
             python: 310
             platform_id: macosx_arm64
+          - os: macos-latest
+            python: 311
+            platform_id: macosx_arm64
 
     steps:
       - name: Checkout scikit-learn
@@ -124,6 +127,13 @@ jobs:
         with:
           python-version: '3.9'  # update once build dependencies are available
 
+      - name: Set pip extra options
+        if: github.event_name == 'schedule'
+        run: |
+          # Test with nightly dependencies for nightly builds
+          echo "PIP_EXTRA_INDEX=https://pypi.anaconda.org/scipy-wheels-nightly/simple" >> $GITHUB_ENV
+          echo "PIP_PRE=1" >> $GITHUB_ENV
+
       - name: Build and test wheels
         env:
           CONFTEST_PATH: ${{ github.workspace }}/conftest.py
@@ -132,16 +142,18 @@ jobs:
                             OPENBLAS_NUM_THREADS=2
                             SKLEARN_SKIP_NETWORK_TESTS=1
                             SKLEARN_BUILD_PARALLEL=3
+                            PIP_EXTRA_INDEX_URL=${{ env.PIP_EXTRA_INDEX }}
+                            PIP_PRE=${{ env.PIP_PRE }}
           CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
           CIBW_ARCHS: all
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
           CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
           CIBW_TEST_SKIP: "*-macosx_arm64"
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
-          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir}
+          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }}
           CIBW_TEST_REQUIRES: pytest pandas threadpoolctl
           CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
-          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
+          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }}
           CIBW_BUILD_VERBOSITY: 1
 
         run: bash build_tools/github/build_wheels.sh

build_tools/github/Windows

@@ -5,10 +5,14 @@ FROM winamd64/python:$PYTHON_VERSION-windowsservercore
 ARG WHEEL_NAME
 ARG CONFTEST_NAME
 ARG CIBW_TEST_REQUIRES
+ARG PIP_EXTRA_INDEX_URL
+ARG PIP_PRE
 
 # Copy and install the Windows wheel
 COPY $WHEEL_NAME $WHEEL_NAME
 COPY $CONFTEST_NAME $CONFTEST_NAME
+RUN echo PIP_EXTRA_INDEX_URL = $env:PIP_EXTRA_INDEX_URL
+RUN echo PIP_PRE = $env:PIP_PRE
 RUN pip install $env:WHEEL_NAME
 
 # Install the testing dependencies

build_tools/github/build_minimal_windows_image.sh

@@ -4,14 +4,6 @@ set -e
 set -x
 
 PYTHON_VERSION=$1
-BITNESS=$2
-
-if [[ "$BITNESS" == "32" ]]; then
-    # 32-bit architectures are not supported
-    # by the official Docker images: Tests will just be run
-    # on the host (instead of the minimal Docker container).
-    exit 0
-fi
 
 TEMP_FOLDER="$HOME/AppData/Local/Temp"
 WHEEL_PATH=$(ls -d $TEMP_FOLDER/**/*/repaired_wheel/*)
@@ -22,10 +14,20 @@ cp $WHEEL_PATH $WHEEL_NAME
 # Dot the Python version for identyfing the base Docker image
 PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2})
 
+# TODO: Remove when 3.11 images will be available for
+# windows (for now the docker image is tagged as 3.11-rc)
+# TODO: Remove when 3.11 images will be available for
+# scipy (for now only available from nightly anaconda index)
+if [[ "$PYTHON_VERSION" == "3.11" ]]; then
+    PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc)
+fi
+
 # Build a minimal Windows Docker image for testing the wheels
 docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \
              --build-arg WHEEL_NAME=$WHEEL_NAME \
              --build-arg CONFTEST_NAME=$CONFTEST_NAME \
              --build-arg CIBW_TEST_REQUIRES="$CIBW_TEST_REQUIRES" \
+             --build-arg PIP_EXTRA_INDEX_URL="$PIP_EXTRA_INDEX_URL" \
+             --build-arg PIP_PRE="$PIP_PRE" \
              -f build_tools/github/Windows \
              -t scikit-learn/minimal-windows .

build_tools/github/build_wheels.sh

@@ -31,6 +31,11 @@ if [[ "$RUNNER_OS" == "macOS" ]]; then
     export CFLAGS="$CFLAGS -I$PREFIX/include"
     export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
     export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp"
+    # Disable the use of setuptools's vendored copy distutils when invoking setuptools
+    # See: https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html
+    # TODO: remove the definition of this environment variable when no
+    # reference to distutils exist in the code-base for building scikit-learn.
+    export SETUPTOOLS_USE_DISTUTILS=stdlib
 fi
 
 # The version of the built dependencies are specified

build_tools/github/repair_windows_wheels.sh

@@ -5,12 +5,11 @@ set -x
 
 WHEEL=$1
 DEST_DIR=$2
-BITNESS=$3
 
 # By default, the Windows wheels are not repaired.
 # In this case, we need to vendor VCRUNTIME140.dll
 wheel unpack "$WHEEL"
 WHEEL_DIRNAME=$(ls -d scikit_learn-*)
-python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS"
+python build_tools/github/vendor.py "$WHEEL_DIRNAME"
 wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR"
 rm -rf "$WHEEL_DIRNAME"

build_tools/github/test_windows_wheels.sh

@@ -4,7 +4,6 @@ set -e
 set -x
 
 PYTHON_VERSION=$1
-BITNESS=$2
 
 docker container run \
     --rm scikit-learn/minimal-windows \

build_tools/github/vendor.py

@@ -114,18 +114,18 @@ def make_distributor_init_64_bits(
         )
 
 
-def main(wheel_dirname, bitness):
+def main(wheel_dirname):
     """Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll."""
     if not op.exists(VCOMP140_SRC_PATH):
         raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.")
 
     if not op.exists(VCRUNTIME140_SRC_PATH):
         raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.")
 
-    if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64":
+    if not op.exists(VCRUNTIME140_1_SRC_PATH):
         raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.")
 
-    if not op.exists(MSVCP140_SRC_PATH) and bitness == "64":
+    if not op.exists(MSVCP140_SRC_PATH):
         raise ValueError(f"Could not find {MSVCP140_SRC_PATH}.")
 
     if not op.isdir(wheel_dirname):
@@ -149,29 +149,22 @@ def main(wheel_dirname, bitness):
     print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.")
     shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder)
 
-    if bitness == "64":
-        print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
-        shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
+    print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
+    shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
 
-        print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.")
-        shutil.copy2(MSVCP140_SRC_PATH, target_folder)
+    print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.")
+    shutil.copy2(MSVCP140_SRC_PATH, target_folder)
 
     # Generate the _distributor_init file in the source tree
-    print("Generating the '_distributor_init.py' file.")
-    if bitness == "32":
-        make_distributor_init_32_bits(
-            distributor_init, vcomp140_dll_filename, vcruntime140_dll_filename
-        )
-    else:
-        make_distributor_init_64_bits(
-            distributor_init,
-            vcomp140_dll_filename,
-            vcruntime140_dll_filename,
-            vcruntime140_1_dll_filename,
-            msvcp140_dll_filename,
-        )
+    make_distributor_init_64_bits(
+        distributor_init,
+        vcomp140_dll_filename,
+        vcruntime140_dll_filename,
+        vcruntime140_1_dll_filename,
+        msvcp140_dll_filename,
+    )
 
 
 if __name__ == "__main__":
-    _, wheel_file, bitness = sys.argv
-    main(wheel_file, bitness)
+    _, wheel_file = sys.argv
+    main(wheel_file)