Maximize wheel build parallellization

ddelange · Jan 20, 2023 · bdb8fe5 · bdb8fe5
1 parent 93a7e02
commit bdb8fe5
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 47 deletions.
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
@@ -7,84 +7,96 @@ on:
   pull_request:
 
 jobs:
+  build_wheels_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - run: pip install cibuildwheel==2.12.0
+
+      - id: set-matrix
+        run: |
+          MATRIX_INCLUDE=$(
+            {
+              cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp |  jq -Rc '{"only": ., "os": "ubuntu-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp |  jq -Rc '{"only": ., "os": "macos-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp |  jq -Rc '{"only": ., "os": "windows-latest"}'
+            } | jq -sc
+          )
+          echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+        working-directory: packages/vaex-core/
+
   build_wheels:
-    name: Build wheel on ${{ matrix.os }}
+    needs: build_wheels_matrix
     runs-on: ${{ matrix.os }}
+    name: Build ${{ matrix.only }}
+
     strategy:
-      fail-fast: false
       matrix:
-        os: [ubuntu-18.04, macOS-latest, windows-latest]
-        python-version: [36, 37, 38, 39, 310]
+        include: ${{ fromJson(needs.build_wheels_matrix.outputs.include) }}
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
       with:
         submodules: true
 
     - name: Copy dll
-      if: (matrix.os == 'windows-latest')
+      if: matrix.os == 'windows-latest'
       uses: ./ci/actions/windll
 
-    - uses: actions/setup-python@v2
-      name: Install Python
-      with:
-        python-version: '3.7'
-
-    - name: Install cibuildwheel
-      run: |
-        python -m pip install cibuildwheel==2.8.1
-
-    - name: Install twine/wheel
-      run: |
-        python -m pip install twine wheel
-
     - name: chores
       if: matrix.os != 'windows-latest'
       run: |
-        mkdir dist
         mkdir packages/vaex-core/bin
         cp bin/install_pcre.sh packages/vaex-core/bin/
 
-    - name: Build vaex-core (windows)
-      if: (matrix.os == 'windows-latest')
-      env:
-        CIBW_BUILD: cp${{ matrix.python-version }}*64
-        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
-      run: |
-        python -m cibuildwheel --output-dir dist packages/vaex-core
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v2
 
-    - name: Build vaex-core (linux, osx)
-      if: (matrix.os != 'windows-latest')
+    - uses: pypa/cibuildwheel@v2.12.0
+      with:
+        only: ${{ matrix.only }}
+        package-dir: packages/vaex-core/
+        output-dir: packages/vaex-core/dist/
       env:
-        CIBW_BEFORE_BUILD: bin/install_pcre.sh
-        CIBW_BUILD: "cp${{ matrix.python-version }}*64 cp${{ matrix.python-version }}-macosx-*"
-        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
+        CIBW_BEFORE_BUILD: ${{ matrix.os == 'windows-latest' && '' || 'bin/install_pcre.sh' }}
         CIBW_BUILD_VERBOSITY: 2
-        # needed for m1 builds
+        CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+        CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
         CIBW_ENVIRONMENT_MACOS: "CFLAGS='-I/usr/local/include -L/usr/local/lib' CXXFLAGS='-I/usr/local/include -L/usr/local/lib' LDFLAGS='-L/usr/local/lib'"
-        CIBW_ARCHS_MACOS: "x86_64 arm64"
-      run: |
-        python -m cibuildwheel --output-dir dist packages/vaex-core
 
-    - name: Make source distribution
-      shell: bash
-      run: |
-          cd packages/vaex-core && python setup.py sdist
+    - uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
 
-    - name: Copy build artifacts
-      shell: bash
-      run: |
-          cp packages/vaex-core/dist/* dist/
+    - name: Make source distribution
+      run: python setup.py sdist
+      working-directory: packages/vaex-core/
 
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v3
       with:
         name: distributions
-        path: ./dist
+        path: packages/vaex-core/dist
 
     - name: Publish a Python distribution to PyPI
+      if: github.event_name == 'push'
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.pypi_password_vaex_core }}
       run: |
+        pip install twine wheel
         openssl sha256 dist/*
-        twine upload --skip-existing dist/vaex?core*
+        twine upload --skip-existing
+      working-directory: packages/vaex-core/
diff --git a/packages/vaex-core/setup.py b/packages/vaex-core/setup.py
@@ -183,12 +183,14 @@ def __str__(self):
       include_package_data=True,
       ext_modules=([extension_vaexfast] if on_rtd else [extension_vaexfast, extension_strings, extension_superutils, extension_superagg]) if not use_skbuild else [],
       zip_safe=False,
+      python_requires=">=3.6",
       classifiers=[
             "Programming Language :: Python :: 3.6",
             "Programming Language :: Python :: 3.7",
             "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
+            "Programming Language :: Python :: 3.11",
       ],
       extras_require={
           'all': ["gcsfs>=0.6.2", "s3fs"]