Merge pull request #7483 from stuartarchibald/fix/7176

NumPy 1.21 support

README.rst

@@ -50,7 +50,7 @@ Dependencies
 
 * Python versions: 3.7-3.9
 * llvmlite 0.38.*
-* NumPy >=1.17 (can build with 1.11 for ABI compatibility).
+* NumPy >=1.18 (can build with 1.11 for ABI compatibility).
 
 Optionally:
 

azure-pipelines.yml

@@ -12,14 +12,14 @@ jobs:
     name: macOS
     vmImage: macOS-10.14
     matrix:
-      py37_np117:
+      py37_np118:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: 'azure_ci'
         TEST_START_INDEX: 0
-      py39_np120:
+      py39_np121:
         PYTHON: '3.9'
-        NUMPY: '1.20'
+        NUMPY: '1.21'
         CONDA_ENV: 'azure_ci'
         TEST_START_INDEX: 1
 
@@ -28,92 +28,92 @@ jobs:
     name: Linux
     vmImage: ubuntu-18.04
     matrix:
-      py37_np117_32bit:
+      py37_np118_32bit:
         # 32 bit linux only has np 1.15
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         BITS32: yes
         TEST_START_INDEX: 2
-      py37_np117_cov:
+      py37_np118_cov:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         RUN_COVERAGE: yes
         RUN_FLAKE8: yes
         RUN_MYPY: yes
         TEST_START_INDEX: 3
-      py37_np117_vanilla:
+      py37_np118_vanilla:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         VANILLA_INSTALL: yes
         TEST_START_INDEX: 4
-      py37_np117_tbb:
+      py37_np118_tbb:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         TEST_THREADING: 'tbb'
         TEST_START_INDEX: 5
-      py37_np117_omp:
+      py37_np118_omp:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         TEST_THREADING: omp
         TEST_START_INDEX: 6
-      py37_np117_workqueue:
+      py37_np118_workqueue:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: azure_ci
         TEST_THREADING: workqueue
         TEST_START_INDEX: 7
-      py37_np118_doc:
+      py37_np119_doc:
         PYTHON: '3.7'
-        NUMPY: '1.18'
+        NUMPY: '1.19'
         CONDA_ENV: azure_ci
         BUILD_DOC: yes
         TEST_START_INDEX: 8
-      py37_np118_pickle5:
+      py37_np119_pickle5:
         PYTHON: '3.7'
-        NUMPY: '1.18'
+        NUMPY: '1.19'
         CONDA_ENV: azure_ci
         TEST_PICKLE5: yes
         TEST_START_INDEX: 9
-      py37_np119_svml:
+      py37_np120_svml:
         PYTHON: '3.7'
-        NUMPY: '1.19'
+        NUMPY: '1.20'
         CONDA_ENV: azure_ci
         TEST_SVML: yes
         TEST_START_INDEX: 10
-      py37_np120:
+      py37_np121:
         PYTHON: '3.7'
-        NUMPY: '1.20'
+        NUMPY: '1.21'
         CONDA_ENV: azure_ci
         TEST_START_INDEX: 11
-      py38_np118:
+      py38_np119:
         PYTHON: '3.8'
-        NUMPY: '1.18'
+        NUMPY: '1.19'
         CONDA_ENV: azure_ci
         TEST_START_INDEX: 12
-      py38_np119_typeguard:
+      py38_np120_typeguard:
         PYTHON: '3.8'
-        NUMPY: '1.19'
+        NUMPY: '1.20'
         CONDA_ENV: azure_ci
         RUN_TYPEGUARD: yes
         TEST_START_INDEX: 13
-      py38_np120:
+      py38_np121:
         PYTHON: '3.8'
-        NUMPY: '1.20'
+        NUMPY: '1.21'
         CONDA_ENV: azure_ci
         TEST_START_INDEX: 14
-      py39_np119:
+      py39_np120:
         PYTHON: '3.9'
-        NUMPY: '1.19'
+        NUMPY: '1.20'
         CONDA_ENV: azure_ci
         TEST_START_INDEX: 15
-      py39_np120:
+      py39_np121:
         PYTHON: '3.9'
-        NUMPY: '1.20'
+        NUMPY: '1.21'
         CONDA_ENV: azure_ci
         TEST_START_INDEX: 16
 

buildscripts/azure/azure-windows.yml

@@ -8,14 +8,14 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py39_np120:
+      py39_np121:
         PYTHON: '3.9'
-        NUMPY: '1.20'
+        NUMPY: '1.21'
         CONDA_ENV: 'testenv'
         TEST_START_INDEX: 17
-      py37_np117:
+      py37_np118:
         PYTHON: '3.7'
-        NUMPY: '1.17'
+        NUMPY: '1.18'
         CONDA_ENV: 'testenv'
         TEST_START_INDEX: 18
 

buildscripts/condarecipe.local/meta.yaml

@@ -42,8 +42,8 @@ requirements:
     #       available.
     - tbb-devel >=2021       # [not (armv6l or armv7l or aarch64 or linux32 or ppc64le)]
   run:
-    - python >=3.6
-    - numpy >=1.17
+    - python >=3.7
+    - numpy >=1.18
     - setuptools
     # On channel https://anaconda.org/numba/
     - llvmlite >=0.38.0dev0,<0.38

docs/source/user/5minguide.rst

@@ -18,7 +18,7 @@ Out of the box Numba works with the following:
   support on M1/Arm64.
 * GPUs: Nvidia CUDA.
 * CPython
-* NumPy 1.17 - latest
+* NumPy 1.18 - latest
 
 How do I get it?
 ----------------

docs/source/user/installing.rst

@@ -5,7 +5,7 @@ Installation
 Compatibility
 -------------
 
-Numba is compatible with Python 3.7 or later, and Numpy versions 1.17 or later.
+Numba is compatible with Python 3.7 or later, and Numpy versions 1.18 or later.
 
 Our supported platforms are:
 

numba/__init__.py

@@ -132,8 +132,8 @@ def _ensure_critical_deps():
     if PYVERSION < (3, 7):
         raise ImportError("Numba needs Python 3.7 or greater")
 
-    if numpy_version < (1, 17):
-        raise ImportError("Numba needs NumPy 1.17 or greater")
+    if numpy_version < (1, 18):
+        raise ImportError("Numba needs NumPy 1.18 or greater")
 
     try:
         import scipy

numba/np/ufunc/_internal.c

@@ -276,6 +276,7 @@ static PyMemberDef dufunc_members[] = {
  */
 
 static struct _ufunc_dispatch {
+    /* Note that the following may also hold `_PyCFunctionFastWithKeywords` */
     PyCFunctionWithKeywords ufunc_reduce;
     PyCFunctionWithKeywords ufunc_accumulate;
     PyCFunctionWithKeywords ufunc_reduceat;
@@ -286,7 +287,7 @@ static struct _ufunc_dispatch {
 } ufunc_dispatch;
 
 static int
-init_ufunc_dispatch(void)
+init_ufunc_dispatch(int *numpy_uses_fastcall)
 {
     int result = 0;
     PyMethodDef * crnt = PyUFunc_Type.tp_methods;
@@ -329,6 +330,16 @@ init_ufunc_dispatch(void)
             result = -1; /* Unknown method */
         }
         if (result < 0) break;
+
+        /* Check whether NumPy uses fastcall (ufunc.at never uses it) */
+        if (strncmp(crnt_name, "at", 3) != 0) {
+            if (*numpy_uses_fastcall == -1) {
+                *numpy_uses_fastcall = crnt->ml_flags & METH_FASTCALL;
+            }
+            else if (*numpy_uses_fastcall != (crnt->ml_flags & METH_FASTCALL)) {
+                return -1;
+            }
+        }
     }
     if (result == 0) {
         /* Sanity check. */
@@ -344,6 +355,7 @@ init_ufunc_dispatch(void)
     return result;
 }
 
+
 static PyObject *
 dufunc_reduce(PyDUFuncObject * self, PyObject * args, PyObject *kws)
 {
@@ -368,6 +380,47 @@ dufunc_outer(PyDUFuncObject * self, PyObject * args, PyObject *kws)
     return ufunc_dispatch.ufunc_outer((PyObject*)self->ufunc, args, kws);
 }
 
+
+/*
+ * The following are the vectorcall versions of the above, since NumPy
+ * uses the FASTCALL/Vectorcall protocol starting with version 1.21.
+ * The only NumPy versions supporting vectorcall use Python 3.7 or higher.
+ */
+static PyObject *
+dufunc_reduce_fast(PyDUFuncObject * self,
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
+{
+    return ((_PyCFunctionFastWithKeywords)ufunc_dispatch.ufunc_reduce)(
+            (PyObject*)self->ufunc, args, len_args, kwnames);
+}
+
+static PyObject *
+dufunc_reduceat_fast(PyDUFuncObject * self,
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
+{
+    return ((_PyCFunctionFastWithKeywords)ufunc_dispatch.ufunc_reduceat)(
+            (PyObject*)self->ufunc, args, len_args, kwnames);
+}
+
+
+static PyObject *
+dufunc_accumulate_fast(PyDUFuncObject * self,
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
+{
+    return ((_PyCFunctionFastWithKeywords)ufunc_dispatch.ufunc_accumulate)(
+            (PyObject*)self->ufunc, args, len_args, kwnames);
+}
+
+
+static PyObject *
+dufunc_outer_fast(PyDUFuncObject * self,
+        PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames)
+{
+    return ((_PyCFunctionFastWithKeywords)ufunc_dispatch.ufunc_outer)(
+            (PyObject*)self->ufunc, args, len_args, kwnames);
+}
+
+
 #if NPY_API_VERSION >= 0x00000008
 static PyObject *
 dufunc_at(PyDUFuncObject * self, PyObject * args)
@@ -568,6 +621,41 @@ static struct PyMethodDef dufunc_methods[] = {
     {NULL, NULL, 0, NULL}           /* sentinel */
 };
 
+
+/*
+ * If Python is new enough, NumPy may use fastcall.  In that case we have to
+ * also use fastcall for simplicity and speed.
+ */
+static struct PyMethodDef dufunc_methods_fast[] = {
+    {"reduce",
+        (PyCFunction)dufunc_reduce_fast,
+        METH_FASTCALL | METH_KEYWORDS, NULL },
+    {"accumulate",
+        (PyCFunction)dufunc_accumulate_fast,
+        METH_FASTCALL | METH_KEYWORDS, NULL },
+    {"reduceat",
+        (PyCFunction)dufunc_reduceat_fast,
+        METH_FASTCALL | METH_KEYWORDS, NULL },
+    {"outer",
+        (PyCFunction)dufunc_outer_fast,
+        METH_FASTCALL | METH_KEYWORDS, NULL},
+#if NPY_API_VERSION >= 0x00000008
+    {"at",
+        (PyCFunction)dufunc_at,
+        METH_VARARGS, NULL},
+#endif
+    {"_compile_for_args",
+        (PyCFunction)dufunc__compile_for_args,
+        METH_VARARGS | METH_KEYWORDS,
+        "Abstract method: subclasses should overload _compile_for_args() to compile the ufunc at the given arguments' types."},
+    {"_add_loop",
+        (PyCFunction)dufunc__add_loop,
+        METH_VARARGS,
+        NULL},
+    {NULL, NULL, 0, NULL}           /* sentinel */
+};
+
+
 static PyObject *
 dufunc_getfrozen(PyDUFuncObject * self, void * closure)
 {
@@ -681,8 +769,15 @@ MOD_INIT(_internal)
         return MOD_ERROR_VAL;
 
     PyDUFunc_Type.tp_new = PyType_GenericNew;
-    if (init_ufunc_dispatch() <= 0)
+
+    int numpy_uses_fastcall = -1;
+    if (init_ufunc_dispatch(&numpy_uses_fastcall) <= 0)
         return MOD_ERROR_VAL;
+
+    if (numpy_uses_fastcall) {
+        PyDUFunc_Type.tp_methods = dufunc_methods_fast;
+    }
+
     if (PyType_Ready(&PyDUFunc_Type) < 0)
         return MOD_ERROR_VAL;
     Py_INCREF(&PyDUFunc_Type);

numba/tests/test_array_reductions.py

@@ -686,8 +686,11 @@ def _do_check_nptimedelta(self, pyfunc, arr):
         np.random.shuffle(arr)
         self.assertPreciseEqual(cfunc(arr), pyfunc(arr))
         # Test with a NaT
-        arr[arr.size // 2] = 'NaT'
-        self.assertPreciseEqual(cfunc(arr), pyfunc(arr))
+        if numpy_version != (1, 21) and 'median' not in pyfunc.__name__:
+            # There's problems with NaT handling in "median" on at least NumPy
+            # 1.21.{3, 4}. See https://github.com/numpy/numpy/issues/20376
+            arr[arr.size // 2] = 'NaT'
+            self.assertPreciseEqual(cfunc(arr), pyfunc(arr))
         if 'median' not in pyfunc.__name__:
             # Test with (val, NaT)^N (and with the random NaT from above)
             # use a loop, there's some weird thing/bug with arr[1::2] = 'NaT'

setup.py

@@ -22,7 +22,7 @@
 min_python_version = "3.7"
 max_python_version = "3.10"  # exclusive
 min_numpy_build_version = "1.11"
-min_numpy_run_version = "1.17"
+min_numpy_run_version = "1.18"
 min_llvmlite_version = "0.38.0dev0"
 max_llvmlite_version = "0.39"