Merge branch 'scipy:main' into enh-rank-filt-1d

scipy · Apr 30, 2024 · a38a472 · a38a472
2 parents 8651e0b + 713bce9
commit a38a472
Show file tree

Hide file tree

Showing 120 changed files with 2,928 additions and 1,462 deletions.
diff --git a/.github/workflows/array_api.yml b/.github/workflows/array_api.yml
@@ -91,6 +91,7 @@ jobs:
         export OMP_NUM_THREADS=2
         # expand as new modules are added
         python dev.py --no-build test -b all -s cluster -- --durations 3 --timeout=60
+        python dev.py --no-build test -b all -s constants -- --durations 3 --timeout=60
         python dev.py --no-build test -b all -s fft -- --durations 3 --timeout=60
         python dev.py --no-build test -b all -t scipy.special.tests.test_support_alternative_backends -- --durations 3 --timeout=60
         python dev.py --no-build test -b all -t scipy._lib.tests.test_array_api

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -47,3 +47,8 @@ jobs:
         python tools/lint.py --diff-against origin/$GITHUB_BASE_REF
         python tools/unicode-check.py
         python tools/check_test_name.py
+
+    - name: Check that Python.h is first in any file including it.
+      shell: bash
+      run: |
+        python tools/check_python_h_first.py 
diff --git a/.github/workflows/linux_meson.yml → .github/workflows/linux.yml b/.github/workflows/linux_meson.yml → .github/workflows/linux.yml
@@ -1,4 +1,4 @@
-name: Linux Meson tests
+name: Linux tests
 
 on:
   push:
@@ -26,7 +26,7 @@ jobs:
     uses: ./.github/workflows/commit_message.yml
 
   test_meson:
-    name: Meson build
+    name: mypy (py3.10) & dev deps (py3.12), fast, dev.py
     needs: get_commit_message
     # If using act to run CI locally the github object does not exist and
     # the usual skipping should not be enforced
@@ -118,7 +118,7 @@ jobs:
       if: matrix.python-version == '3.10'
       run: |
         # Packages that are only needed for their annotations
-        python -m pip install mypy==1.0.0 types-psutil typing_extensions
+        python -m pip install mypy==1.10.0 types-psutil typing_extensions
         python -m pip install pybind11 sphinx
 
         python -u dev.py mypy
@@ -130,7 +130,7 @@ jobs:
 
   #################################################################################
   test_venv_install:
-    name: Pip install into venv
+    name: Install into venv, cluster only, pyAny/npAny, pip+cluster.test()
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -189,7 +189,7 @@ jobs:
   #################################################################################
   python_debug:
     # also uses the vcs->sdist->wheel route.
-    name: Python-debug & ATLAS
+    name: Python-debug & ATLAS & sdist+wheel, fast, py3.10/npMin, pip+pytest
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -219,7 +219,7 @@ jobs:
   #################################################################################
   gcc9:
     # Purpose is to examine builds with oldest-supported gcc and test with pydata/sparse.
-    name: Build with gcc-9
+    name: Oldest GCC & pydata/sparse, fast, py3.10/npMin, pip+pytest
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -271,7 +271,7 @@ jobs:
   #################################################################################
   prerelease_deps_coverage_64bit_blas:
     # TODO: re-enable ILP64 build.
-    name: Prerelease deps, coverage and 64-bit BLAS
+    name: Prerelease deps & coverage report, full, py3.10/npMin & py3.11/npPre, dev.py
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -357,7 +357,7 @@ jobs:
 
   #################################################################################
   linux_32bit:
-    name: Linux - 32 bit
+    name: 32-bit, fast, py3.10/npMin, dev.py
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1

diff --git a/.github/workflows/macos_meson.yml → .github/workflows/macos.yml b/.github/workflows/macos_meson.yml → .github/workflows/macos.yml
@@ -1,4 +1,4 @@
-name: macOS tests (meson)
+name: macOS tests
 
 on:
   push:
@@ -26,7 +26,7 @@ jobs:
     uses: ./.github/workflows/commit_message.yml
 
   test_meson:
-    name: Meson build
+    name: Conda & umfpack/scikit-sparse, fast, py3.11/npAny, dev.py
     needs: get_commit_message
     # If using act to run CI locally the github object does not exist and
     # the usual skipping should not be enforced
@@ -137,7 +137,7 @@ jobs:
 
 
   test_scipy_openblas:
-    name: M1 test - openblas
+    name: M1 & OpenBLAS, fast, py3.11/npAny, dev.py
     needs: get_commit_message
     # If using act to run CI locally the github object does not exist and
     # the usual skipping should not be enforced
@@ -165,14 +165,14 @@ jobs:
         sudo xcode-select -s /Applications/Xcode_15.2.app
         
         git submodule update --init
-        # for some reason gfortran is not on the path
-        GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran 
+        GFORTRAN_LOC=$(which gfortran-13)
         ln -s $GFORTRAN_LOC gfortran
         export PATH=$PWD:$PATH
 
-        # make sure we have openblas
+        # make sure we have openblas and gfortran dylibs
         bash tools/wheels/cibw_before_build_macos.sh $PWD
-        export DYLD_LIBRARY_PATH=/usr/local/gfortran/lib:/opt/arm64-builds/lib
+        GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`)
+        export DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib
         export PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig
     
         pip install click doit pydevtool rich_click meson cython pythran pybind11 ninja numpy
@@ -181,8 +181,9 @@ jobs:
         pip install pooch pytest hypothesis
         python dev.py -n test 
 
+
   test_meson_accelerate:
-    name: Meson build (Accelerate)
+    name: Accelerate, fast, py3.11/npAny, dev.py
     needs: get_commit_message
     # If using act to run CI locally the github object does not exist and
     # the usual skipping should not be enforced
@@ -210,11 +211,14 @@ jobs:
         sudo xcode-select -s /Applications/Xcode_15.2.app
 
         git submodule update --init
-        # for some reason gfortran is not on the path
-        GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran
+        GFORTRAN_LOC=$(which gfortran-13)
         ln -s $GFORTRAN_LOC gfortran
         export PATH=$PWD:$PATH
 
+        # Ensure we have gfortran dylib
+        GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`)
+        export DYLD_LIBRARY_PATH=$GFORTRAN_LIB
+
         pip install click doit pydevtool rich_click meson cython pythran pybind11 ninja numpy
         python dev.py build -C-Dblas=accelerate
 

diff --git a/.github/workflows/linux_musl.yml → .github/workflows/musllinux.yml b/.github/workflows/linux_musl.yml → .github/workflows/musllinux.yml
@@ -24,6 +24,7 @@ jobs:
     uses: ./.github/workflows/commit_message.yml
 
   musllinux_x86_64:
+    name: musl Ubuntu-latest, fast, py3.10/npAny, dev.py
     needs: get_commit_message
     runs-on: ubuntu-latest
     # If using act to run CI locally the github object does not exist and

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -59,7 +59,9 @@ jobs:
           echo github.ref ${{ github.ref }}
 
   build_wheels:
-    name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} ${{ matrix.buildplat[2] }}
+    name: Wheel, ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+      ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }}
+      ${{ matrix.buildplat[4] }}
     needs: get_commit_message
     if: >-
       contains(needs.get_commit_message.outputs.message, '1') ||
@@ -77,11 +79,13 @@ jobs:
         # should also be able to do multi-archs on a single entry, e.g.
         # [windows-2019, win*, "AMD64 x86"]. However, those two require a different compiler setup
         # so easier to separate out here.
-        - [ubuntu-22.04, manylinux, x86_64]
-        - [ubuntu-22.04, musllinux, x86_64]
-        - [macos-11, macosx, x86_64]
-        - [macos-14, macosx, arm64]
-        - [windows-2019, win, AMD64]
+        - [ubuntu-22.04, manylinux, x86_64, "", ""]
+        - [ubuntu-22.04, musllinux, x86_64, "", ""]
+        - [macos-12, macosx, x86_64, openblas, "10.9"]
+        - [macos-13, macosx, x86_64, accelerate, "14.0"]
+        - [macos-14, macosx, arm64, openblas, "12.0"]
+        - [macos-14, macosx, arm64, accelerate, "14.0"]
+        - [windows-2019, win, AMD64, "", ""]
 
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
         # python[0] is used to specify the python versions made by cibuildwheel
@@ -112,63 +116,53 @@ jobs:
         if: ${{ runner.os == 'Windows' && env.IS_32_BIT == 'false' }}
 
       - name: Setup macOS
-        if: matrix.buildplat[0] == 'macos-11' || matrix.buildplat[0] == 'macos-14'
+        if: startsWith( matrix.buildplat[0], 'macos-' )
         run: |
-          if [[ ${{ matrix.buildplat[2] }} == 'arm64' ]]; then
-            # macosx_arm64
-          
-            # use homebrew gfortran
-            sudo xcode-select -s /Applications/Xcode_15.2.app            
-            # for some reason gfortran is not on the path
-            GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran 
-            ln -s $GFORTRAN_LOC gfortran
+          if [[ ${{ matrix.buildplat[3] }} == 'accelerate' ]]; then
+            echo CIBW_CONFIG_SETTINGS=\"setup-args=-Dblas=accelerate\" >> "$GITHUB_ENV"
+            # Always use preinstalled gfortran for Accelerate builds
+            ln -s $(which gfortran-13) gfortran
             export PATH=$PWD:$PATH
             echo "PATH=$PATH" >> "$GITHUB_ENV"
-          
-            # location of the gfortran's libraries
-            GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`)
-
-            CIBW="MACOSX_DEPLOYMENT_TARGET=12.0\
-              MACOS_DEPLOYMENT_TARGET=12.0\
-              LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\
-              _PYTHON_HOST_PLATFORM=macosx-12.0-arm64\
-              PIP_PRE=1\
-              PIP_NO_BUILD_ISOLATION=false\
-              PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig\
-              PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-            echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV"
-          
-            CIBW="sudo xcode-select -s /Applications/Xcode_15.2.app"
-            echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV
-
-            echo "REPAIR_PATH=/opt/arm64-builds/lib" >> "$GITHUB_ENV"
-
-            CIBW="DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib delocate-listdeps {wheel} &&\
-              DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
-            echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV"
-      
+            LIB_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib))
+          fi
+          # Add libraries installed by cibw_before_build_macos.sh to path
+          if [[ ${{ matrix.buildplat[2] }} == 'arm64' ]]; then
+            LIB_PATH=$LIB_PATH:/opt/arm64-builds/lib
+          else
+            LIB_PATH=$LIB_PATH:/usr/local/lib
+          fi
+          if [[ ${{ matrix.buildplat[4] }} == '10.9' ]]; then
+            # Newest version of Xcode that supports macOS 10.9
+            XCODE_VER='13.4.1'
           else
-            # macosx_x86_64 with OpenBLAS
-            # setting SDKROOT necessary when using the gfortran compiler
-            # installed in cibw_before_build_macos.sh
-            # MACOS_DEPLOYMENT_TARGET is set because of
-            # https://github.com/mesonbuild/meson-python/pull/309. Once
-            # an update is released, then that environment variable can
-            # be removed.
-            CIBW="MACOSX_DEPLOYMENT_TARGET=10.9\
-              MACOS_DEPLOYMENT_TARGET=10.9\
-              SDKROOT=/Applications/Xcode_11.7.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk\
-              LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\
-              _PYTHON_HOST_PLATFORM=macosx-10.9-x86_64\
-              PIP_PRE=1\
-              PIP_NO_BUILD_ISOLATION=false\
-              PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-            echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV"  
-
-            CIBW="DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} &&\
-              DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
-            echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV"
+            XCODE_VER='15.2'
+          fi
+          CIBW="sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app"
+          echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV
+          # setting SDKROOT necessary when using the gfortran compiler
+          # installed in cibw_before_build_macos.sh
+          sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app
+          CIBW="MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }}\
+            LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH\
+            SDKROOT=$(xcrun --sdk macosx --show-sdk-path)\
+            PIP_PRE=1\
+            PIP_NO_BUILD_ISOLATION=false\
+            PKG_CONFIG_PATH=$LIB_PATH/pkgconfig\
+            PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+          echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV"
+
+          echo "REPAIR_PATH=$LIB_PATH" >> "$GITHUB_ENV"
+          GFORTRAN_LIB="\$(dirname \$(gfortran --print-file-name libgfortran.dylib))"
+          CIBW="DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-listdeps {wheel} &&\
+            DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-wheel --require-archs \
+            {delocate_archs} -w {dest_dir} {wheel}"
+          # Rename x86 Accelerate wheel to test on macOS 13 runner
+          if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then
+            CIBW+=" && mv {dest_dir}/\$(basename {wheel}) \
+              {dest_dir}/\$(echo \$(basename {wheel}) | sed 's/14_0/13_0/')"
           fi
+          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV"
 
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.17.0
@@ -195,10 +189,18 @@ jobs:
             PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
             PIP_NO_BUILD_ISOLATION=false
 
+      - name: Rename after test (macOS x86 Accelerate only)
+        # Rename x86 Accelerate wheel back so it targets macOS >= 14
+        if: matrix.buildplat[0] == 'macos-13' && matrix.buildplat[4] == '14.0'
+        run: |
+          mv ./wheelhouse/*.whl $(find ./wheelhouse -type f -name '*.whl' | sed 's/13_0/14_0/')
+
       - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
-          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.buildplat[2] }}
+          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
+            ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }}
+            ${{ matrix.buildplat[4] }}
 
       - uses: conda-incubator/setup-miniconda@v3
         with:
@@ -209,6 +211,7 @@ jobs:
           # build and test the wheel
           auto-update-conda: true
           python-version: "3.10"
+          miniconda-version: "latest"
 
       - name: Upload wheels
         if: success()

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -22,7 +22,7 @@ jobs:
     uses: ./.github/workflows/commit_message.yml
 
   test:
-    name: cp312 (meson) fast
+    name: fast, py3.12/npAny, dev.py
     needs: get_commit_message
     # Ensure (a) this doesn't run on forks by default, and
     #        (b) it does run with Act locally (`github` doesn't exist there)
@@ -62,7 +62,7 @@ jobs:
 
   #############################################################################
   full_dev_py_min_numpy:
-    name: cp310 (meson) full, dev.py, minimum numpy
+    name: full, py3.10/npMin, dev.py
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -103,7 +103,7 @@ jobs:
   #############################################################################
   full_build_sdist_wheel:
     # TODO: enable ILP64 once possible
-    name: cp311 (build sdist + wheel), full, no pythran
+    name: no pythran & sdist+wheel, full, py3.11/npPre, pip+pytest
     needs: get_commit_message
     if: >
       needs.get_commit_message.outputs.message == 1
@@ -157,4 +157,3 @@ jobs:
           cd $RUNNER_TEMP
           # run full test suite
           pytest --pyargs scipy
-
diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt
@@ -281,3 +281,8 @@ Name: array-api-compat
 Files: scipy/_lib/array-api-compat/*
 License: MIT
   For details, see scipy/_lib/array-api-compat/LICENCE
+
+Name: Tempita
+Files: scipy/_build_utils/tempita/*
+License: MIT
+  For details, see scipy/_build_utils/tempita/LICENCE.txt