CI Add wheel builds for Python 3.11 (#24446)

Co-authored-by: Julien Jerphanion <git@jjerphan.xyz>
Co-authored-by: Thomas J. Fan <thomasjpfan@gmail.com>

.github/workflows/wheels.yml

@@ -53,15 +53,15 @@ jobs:
           # https://github.com/scikit-learn/scikit-learn/issues/22530
           - os: windows-2019
             python: 38
-            bitness: 64
             platform_id: win_amd64
           - os: windows-latest
             python: 39
-            bitness: 64
             platform_id: win_amd64
           - os: windows-latest
             python: 310
-            bitness: 64
+            platform_id: win_amd64
+          - os: windows-latest
+            python: 311
             platform_id: win_amd64
 
           # Window 32 bit
@@ -77,49 +77,51 @@ jobs:
           # Linux 64 bit manylinux2014
           - os: ubuntu-latest
             python: 38
-            bitness: 64
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
           - os: ubuntu-latest
             python: 39
-            bitness: 64
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
 
           # NumPy on Python 3.10 only supports 64bit and is only available with manylinux2014
           - os: ubuntu-latest
             python: 310
-            bitness: 64
+            platform_id: manylinux_x86_64
+            manylinux_image: manylinux2014
+
+          - os: ubuntu-latest
+            python: 311
             platform_id: manylinux_x86_64
             manylinux_image: manylinux2014
 
           # MacOS x86_64
           - os: macos-latest
-            bitness: 64
             python: 38
             platform_id: macosx_x86_64
           - os: macos-latest
-            bitness: 64
             python: 39
             platform_id: macosx_x86_64
           - os: macos-latest
-            bitness: 64
             python: 310
             platform_id: macosx_x86_64
+          - os: macos-latest
+            python: 311
+            platform_id: macosx_x86_64
 
           # MacOS arm64
           - os: macos-latest
-            bitness: 64
             python: 38
             platform_id: macosx_arm64
           - os: macos-latest
-            bitness: 64
             python: 39
             platform_id: macosx_arm64
           - os: macos-latest
-            bitness: 64
             python: 310
             platform_id: macosx_arm64
+          - os: macos-latest
+            python: 311
+            platform_id: macosx_arm64
 
     steps:
       - name: Checkout scikit-learn
@@ -143,11 +145,11 @@ jobs:
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
           CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
           CIBW_TEST_SKIP: "*-macosx_arm64"
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
-          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir}
+          CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }}
           CIBW_TEST_REQUIRES: pytest pandas threadpoolctl
           CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh
-          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }} ${{ matrix.bitness }}
+          CIBW_TEST_COMMAND_WINDOWS: bash {project}/build_tools/github/test_windows_wheels.sh ${{ matrix.python }}
           CIBW_BUILD_VERBOSITY: 1
 
         run: bash build_tools/github/build_wheels.sh

.travis.yml

@@ -70,6 +70,16 @@ jobs:
         - CIBW_BUILD=cp310-manylinux_aarch64
         - BUILD_WHEEL=true
 
+    - os: linux
+      arch: arm64-graviton2
+      dist: focal
+      virt: vm
+      group: edge
+      if: type = cron or commit_message =~ /\[cd build\]/
+      env:
+        - CIBW_BUILD=cp311-manylinux_aarch64
+        - BUILD_WHEEL=true
+
 install: source build_tools/travis/install.sh || travis_terminate 1
 script: source build_tools/travis/script.sh || travis_terminate 1
 after_success: source build_tools/travis/after_success.sh || travis_terminate 1

build_tools/github/build_minimal_windows_image.sh

@@ -4,14 +4,6 @@ set -e
 set -x
 
 PYTHON_VERSION=$1
-BITNESS=$2
-
-if [[ "$BITNESS" == "32" ]]; then
-    # 32-bit architectures are not supported
-    # by the official Docker images: Tests will just be run
-    # on the host (instead of the minimal Docker container).
-    exit 0
-fi
 
 TEMP_FOLDER="$HOME/AppData/Local/Temp"
 WHEEL_PATH=$(ls -d $TEMP_FOLDER/**/*/repaired_wheel/*)
@@ -22,6 +14,12 @@ cp $WHEEL_PATH $WHEEL_NAME
 # Dot the Python version for identyfing the base Docker image
 PYTHON_VERSION=$(echo ${PYTHON_VERSION:0:1}.${PYTHON_VERSION:1:2})
 
+# TODO: Remove when 3.11 images will be available for
+# windows (for now the docker image is tagged as 3.11-rc)
+if [[ "$PYTHON_VERSION" == "3.11" ]]; then
+    PYTHON_VERSION=$(echo ${PYTHON_VERSION}-rc)
+fi
+
 # Build a minimal Windows Docker image for testing the wheels
 docker build --build-arg PYTHON_VERSION=$PYTHON_VERSION \
              --build-arg WHEEL_NAME=$WHEEL_NAME \

build_tools/github/build_wheels.sh

@@ -31,6 +31,11 @@ if [[ "$RUNNER_OS" == "macOS" ]]; then
     export CFLAGS="$CFLAGS -I$PREFIX/include"
     export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
     export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp"
+    # Disable the use of setuptools's vendored copy distutils when invoking setuptools
+    # See: https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html
+    # TODO: remove the definition of this environment variable when no
+    # reference to distutils exist in the code-base for building scikit-learn.
+    export SETUPTOOLS_USE_DISTUTILS=stdlib
 fi
 
 # The version of the built dependencies are specified

build_tools/github/repair_windows_wheels.sh

@@ -5,12 +5,11 @@ set -x
 
 WHEEL=$1
 DEST_DIR=$2
-BITNESS=$3
 
 # By default, the Windows wheels are not repaired.
 # In this case, we need to vendor VCRUNTIME140.dll
 wheel unpack "$WHEEL"
 WHEEL_DIRNAME=$(ls -d scikit_learn-*)
-python build_tools/github/vendor.py "$WHEEL_DIRNAME" "$BITNESS"
+python build_tools/github/vendor.py "$WHEEL_DIRNAME"
 wheel pack "$WHEEL_DIRNAME" -d "$DEST_DIR"
 rm -rf "$WHEEL_DIRNAME"

build_tools/github/test_windows_wheels.sh

@@ -4,7 +4,6 @@ set -e
 set -x
 
 PYTHON_VERSION=$1
-BITNESS=$2
 
 if [[ "$BITNESS" == "32" ]]; then
     # 32-bit architectures use the regular

build_tools/github/vendor.py

@@ -114,18 +114,18 @@ def make_distributor_init_64_bits(
         )
 
 
-def main(wheel_dirname, bitness):
+def main(wheel_dirname):
     """Embed vcomp140.dll, vcruntime140.dll and vcruntime140_1.dll."""
     if not op.exists(VCOMP140_SRC_PATH):
         raise ValueError(f"Could not find {VCOMP140_SRC_PATH}.")
 
     if not op.exists(VCRUNTIME140_SRC_PATH):
         raise ValueError(f"Could not find {VCRUNTIME140_SRC_PATH}.")
 
-    if not op.exists(VCRUNTIME140_1_SRC_PATH) and bitness == "64":
+    if not op.exists(VCRUNTIME140_1_SRC_PATH):
         raise ValueError(f"Could not find {VCRUNTIME140_1_SRC_PATH}.")
 
-    if not op.exists(MSVCP140_SRC_PATH) and bitness == "64":
+    if not op.exists(MSVCP140_SRC_PATH):
         raise ValueError(f"Could not find {MSVCP140_SRC_PATH}.")
 
     if not op.isdir(wheel_dirname):
@@ -149,29 +149,23 @@ def main(wheel_dirname, bitness):
     print(f"Copying {VCRUNTIME140_SRC_PATH} to {target_folder}.")
     shutil.copy2(VCRUNTIME140_SRC_PATH, target_folder)
 
-    if bitness == "64":
-        print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
-        shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
+    print(f"Copying {VCRUNTIME140_1_SRC_PATH} to {target_folder}.")
+    shutil.copy2(VCRUNTIME140_1_SRC_PATH, target_folder)
 
-        print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.")
-        shutil.copy2(MSVCP140_SRC_PATH, target_folder)
+    print(f"Copying {MSVCP140_SRC_PATH} to {target_folder}.")
+    shutil.copy2(MSVCP140_SRC_PATH, target_folder)
 
     # Generate the _distributor_init file in the source tree
     print("Generating the '_distributor_init.py' file.")
-    if bitness == "32":
-        make_distributor_init_32_bits(
-            distributor_init, vcomp140_dll_filename, vcruntime140_dll_filename
-        )
-    else:
-        make_distributor_init_64_bits(
-            distributor_init,
-            vcomp140_dll_filename,
-            vcruntime140_dll_filename,
-            vcruntime140_1_dll_filename,
-            msvcp140_dll_filename,
-        )
+    make_distributor_init_64_bits(
+        distributor_init,
+        vcomp140_dll_filename,
+        vcruntime140_dll_filename,
+        vcruntime140_1_dll_filename,
+        msvcp140_dll_filename,
+    )
 
 
 if __name__ == "__main__":
-    _, wheel_file, bitness = sys.argv
-    main(wheel_file, bitness)
+    _, wheel_file = sys.argv
+    main(wheel_file)