From ca93db4ede843c7c9806f05195568f1a7e8e20df Mon Sep 17 00:00:00 2001 From: Tyler Reddy Date: Thu, 10 Dec 2020 20:01:39 -0700 Subject: [PATCH 01/13] BUG: pavement.py fixes * the changes made to `pavement.py`, as described in gh-13221, modernized some file handling code blocks as suggested by static analysis * however, this broke the usage of `paver release` (which is not tested in CI), because the meaning of those code blocks was changed from passing a path-like object to `os.path.basename()` to passing an open file handle to `basename()` * this two-character patch restores the originally intended meaning (passing a path-like object to `basename()`) while preserving the modernized file context managers; I confirmed locally that `paver release` works again with this patch --- pavement.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pavement.py b/pavement.py index c29687462c52..5cf31e25ad1a 100644 --- a/pavement.py +++ b/pavement.py @@ -209,7 +209,7 @@ def compute_md5(idirs): for fn in sorted(released): with open(fn, 'rb') as f: m = md5(f.read()) - checksums.append('%s %s' % (m.hexdigest(), os.path.basename(f))) + checksums.append('%s %s' % (m.hexdigest(), os.path.basename(fn))) return checksums @@ -221,7 +221,7 @@ def compute_sha256(idirs): for fn in sorted(released): with open(fn, 'rb') as f: m = sha256(f.read()) - checksums.append('%s %s' % (m.hexdigest(), os.path.basename(f))) + checksums.append('%s %s' % (m.hexdigest(), os.path.basename(fn))) return checksums From cc63d57dcb4ba3c8af757d4eefcc96e2de3acfa7 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 18:51:25 -0500 Subject: [PATCH 02/13] BUG: fix cval handling for complex inputs in ndimage.interpolation --- scipy/ndimage/interpolation.py | 40 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/scipy/ndimage/interpolation.py b/scipy/ndimage/interpolation.py index 2e85cdfeb7dd..b75f8a8fd70a 100644 --- a/scipy/ndimage/interpolation.py +++ b/scipy/ndimage/interpolation.py @@ -329,12 +329,14 @@ def geometric_transform(input, mapping, output_shape=None, output = _ni_support._get_output(output, input, shape=output_shape, complex_output=complex_output) if complex_output: - kwargs = dict(order=order, mode=mode, cval=cval, prefilter=prefilter, + kwargs = dict(order=order, mode=mode, prefilter=prefilter, output_shape=output_shape, extra_arguments=extra_arguments, extra_keywords=extra_keywords) - geometric_transform(input.real, mapping, output=output.real, **kwargs) - geometric_transform(input.imag, mapping, output=output.imag, **kwargs) + geometric_transform(input.real, mapping, output=output.real, + cval=numpy.real(cval), **kwargs) + geometric_transform(input.imag, mapping, output=output.imag, + cval=numpy.imag(cval), **kwargs) return output if prefilter and order > 1: @@ -437,9 +439,11 @@ def map_coordinates(input, coordinates, output=None, order=3, output = _ni_support._get_output(output, input, shape=output_shape, complex_output=complex_output) if complex_output: - kwargs = dict(order=order, mode=mode, cval=cval, prefilter=prefilter) - map_coordinates(input.real, coordinates, output=output.real, **kwargs) - map_coordinates(input.imag, coordinates, output=output.imag, **kwargs) + kwargs = dict(order=order, mode=mode, prefilter=prefilter) + map_coordinates(input.real, coordinates, output=output.real, + cval=numpy.real(cval), **kwargs) + map_coordinates(input.imag, coordinates, output=output.imag, + cval=numpy.imag(cval), **kwargs) return output if prefilter and order > 1: padded, npad = _prepad_for_spline_filter(input, mode, cval) @@ -551,9 +555,11 @@ def affine_transform(input, matrix, offset=0.0, output_shape=None, complex_output=complex_output) if complex_output: kwargs = dict(offset=offset, output_shape=output_shape, order=order, - mode=mode, cval=cval, prefilter=prefilter) - affine_transform(input.real, matrix, output=output.real, **kwargs) - affine_transform(input.imag, matrix, output=output.imag, **kwargs) + mode=mode, prefilter=prefilter) + affine_transform(input.real, matrix, output=output.real, + cval=numpy.real(cval), **kwargs) + affine_transform(input.imag, matrix, output=output.imag, + cval=numpy.imag(cval), **kwargs) return output if prefilter and order > 1: padded, npad = _prepad_for_spline_filter(input, mode, cval) @@ -655,9 +661,11 @@ def shift(input, shift, output=None, order=3, mode='constant', cval=0.0, # import under different name to avoid confusion with shift parameter from scipy.ndimage.interpolation import shift as _shift - kwargs = dict(order=order, mode=mode, cval=cval, prefilter=prefilter) - _shift(input.real, shift, output=output.real, **kwargs) - _shift(input.imag, shift, output=output.imag, **kwargs) + kwargs = dict(order=order, mode=mode, prefilter=prefilter) + _shift(input.real, shift, output=output.real, cval=numpy.real(cval), + **kwargs) + _shift(input.imag, shift, output=output.imag, cval=numpy.imag(cval), + **kwargs) return output if prefilter and order > 1: padded, npad = _prepad_for_spline_filter(input, mode, cval) @@ -763,9 +771,11 @@ def zoom(input, zoom, output=None, order=3, mode='constant', cval=0.0, # import under different name to avoid confusion with zoom parameter from scipy.ndimage.interpolation import zoom as _zoom - kwargs = dict(order=order, mode=mode, cval=cval, prefilter=prefilter) - _zoom(input.real, zoom, output=output.real, **kwargs) - _zoom(input.imag, zoom, output=output.imag, **kwargs) + kwargs = dict(order=order, mode=mode, prefilter=prefilter) + _zoom(input.real, zoom, output=output.real, cval=numpy.real(cval), + **kwargs) + _zoom(input.imag, zoom, output=output.imag, cval=numpy.imag(cval), + **kwargs) return output if prefilter and order > 1: padded, npad = _prepad_for_spline_filter(input, mode, cval) From 7fd88bcba9d4f8693b7a6dade036bf4c75510453 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 18:52:11 -0500 Subject: [PATCH 03/13] TST: test cval handling in ndimage.interpolation with complex inputs --- scipy/ndimage/tests/test_interpolation.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scipy/ndimage/tests/test_interpolation.py b/scipy/ndimage/tests/test_interpolation.py index 389c0add99d1..f696f9fb0eed 100644 --- a/scipy/ndimage/tests/test_interpolation.py +++ b/scipy/ndimage/tests/test_interpolation.py @@ -924,6 +924,26 @@ def test_shift05(self, order, dtype): out = ndimage.shift(data, [0, 1], order=order) assert_array_almost_equal(out, expected) + @pytest.mark.parametrize('order', range(0, 6)) + @pytest.mark.parametrize('mode', ['constant', 'grid-constant']) + @pytest.mark.parametrize('dtype', [numpy.float64, numpy.complex128]) + def test_shift_with_nonzero_cval(self, order, mode, dtype): + data = numpy.array([[1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1]], dtype=dtype) + + expected = numpy.array([[0, 1, 1, 1], + [0, 1, 1, 1], + [0, 1, 1, 1]], dtype=dtype) + + if data.dtype.kind == 'c': + data -= 1j * data + expected -= 1j * expected + cval = 5.0 + expected[:, 0] = cval # specific to shift of [0, 1] used below + out = ndimage.shift(data, [0, 1], order=order, mode=mode, cval=cval) + assert_array_almost_equal(out, expected) + @pytest.mark.parametrize('order', range(0, 6)) def test_shift06(self, order): data = numpy.array([[4, 1, 3, 2], From 369e4f851330dbb7d64e5897b7acf34a90e5992a Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 18:52:35 -0500 Subject: [PATCH 04/13] BUG: fix cval handling for complex inputs in ndimage.filters --- scipy/ndimage/filters.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/scipy/ndimage/filters.py b/scipy/ndimage/filters.py index e2be8e18a902..3564c77d4fcc 100644 --- a/scipy/ndimage/filters.py +++ b/scipy/ndimage/filters.py @@ -51,23 +51,32 @@ def _invalid_origin(origin, lenw): return (origin < -(lenw // 2)) or (origin > (lenw - 1) // 2) -def _complex_via_real_components(func, input, weights, output, **kwargs): +def _complex_via_real_components(func, input, weights, output, cval, **kwargs): """Complex convolution via a linear combination of real convolutions.""" complex_input = input.dtype.kind == 'c' complex_weights = weights.dtype.kind == 'c' if complex_input and complex_weights: # real component of the output - func(input.real, weights.real, output=output.real, **kwargs) - output.real -= func(input.imag, weights.imag, output=None, **kwargs) + func(input.real, weights.real, output=output.real, + cval=numpy.real(cval), **kwargs) + output.real -= func(input.imag, weights.imag, output=None, + cval=numpy.imag(cval), **kwargs) # imaginary component of the output - func(input.real, weights.imag, output=output.imag, **kwargs) - output.imag += func(input.imag, weights.real, output=None, **kwargs) + func(input.real, weights.imag, output=output.imag, + cval=numpy.real(cval), **kwargs) + output.imag += func(input.imag, weights.real, output=None, + cval=numpy.imag(cval), **kwargs) elif complex_input: - func(input.real, weights, output=output.real, **kwargs) - func(input.imag, weights, output=output.imag, **kwargs) + func(input.real, weights, output=output.real, cval=numpy.real(cval), + **kwargs) + func(input.imag, weights, output=output.imag, cval=numpy.imag(cval), + **kwargs) else: - func(input, weights.real, output=output.real, **kwargs) - func(input, weights.imag, output=output.imag, **kwargs) + if numpy.iscomplexobj(cval): + raise ValueError("Cannot provide a complex-valued cval when the " + "input is real.") + func(input, weights.real, output=output.real, cval=cval, **kwargs) + func(input, weights.imag, output=output.imag, cval=cval, **kwargs) return output @@ -104,10 +113,10 @@ def correlate1d(input, weights, axis=-1, output=None, mode="reflect", if complex_weights: weights = weights.conj() weights = weights.astype(numpy.complex128, copy=False) - kwargs = dict(axis=axis, mode=mode, cval=cval, origin=origin) + kwargs = dict(axis=axis, mode=mode, origin=origin) output = _ni_support._get_output(output, input, complex_output=True) return _complex_via_real_components(correlate1d, input, weights, - output, **kwargs) + output, cval, **kwargs) output = _ni_support._get_output(output, input) weights = numpy.asarray(weights, dtype=numpy.float64) @@ -638,12 +647,12 @@ def _correlate_or_convolve(input, weights, output, mode, cval, origin, # As for numpy.correlate, conjugate weights rather than input. weights = weights.conj() kwargs = dict( - mode=mode, cval=cval, origin=origin, convolution=convolution + mode=mode, origin=origin, convolution=convolution ) output = _ni_support._get_output(output, input, complex_output=True) return _complex_via_real_components(_correlate_or_convolve, input, - weights, output, **kwargs) + weights, output, cval, **kwargs) origins = _ni_support._normalize_sequence(origin, input.ndim) weights = numpy.asarray(weights, dtype=numpy.float64) @@ -889,9 +898,9 @@ def uniform_filter1d(input, size, axis=-1, output=None, origin) else: _nd_image.uniform_filter1d(input.real, size, axis, output.real, mode, - cval, origin) + numpy.real(cval), origin) _nd_image.uniform_filter1d(input.imag, size, axis, output.imag, mode, - cval, origin) + numpy.imag(cval), origin) return output From 1e3b3caea7f4688db3e117709f29de6991ec393a Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 19:16:55 -0500 Subject: [PATCH 05/13] BUG: avoid segfault if mode='grid-constant' is passed to ndimage.filter functions --- scipy/ndimage/src/ni_support.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scipy/ndimage/src/ni_support.c b/scipy/ndimage/src/ni_support.c index 84f12ca00156..a6e3ff06a339 100644 --- a/scipy/ndimage/src/ni_support.c +++ b/scipy/ndimage/src/ni_support.c @@ -310,6 +310,7 @@ int NI_ExtendLine(double *buffer, npy_intp line_length, break; /* kkkkkkkk|abcd]kkkkkkkk */ case NI_EXTEND_CONSTANT: + case NI_EXTEND_GRID_CONSTANT: val = extend_value; dst = buffer; while (size_before--) { @@ -670,6 +671,7 @@ int NI_InitFilterOffsets(PyArrayObject *array, npy_bool *footprint, } break; case NI_EXTEND_CONSTANT: + case NI_EXTEND_GRID_CONSTANT: if (cc < 0 || cc >= len) cc = *border_flag_value; break; From 3573c257d76e33be43a733da3704558bf9847b9e Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 19:52:17 -0500 Subject: [PATCH 06/13] TST: test correlate and convolve functions with complex cval --- scipy/ndimage/tests/test_filters.py | 117 ++++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 16 deletions(-) diff --git a/scipy/ndimage/tests/test_filters.py b/scipy/ndimage/tests/test_filters.py index a2d68e8115bd..a66bd306469a 100644 --- a/scipy/ndimage/tests/test_filters.py +++ b/scipy/ndimage/tests/test_filters.py @@ -20,7 +20,8 @@ def sumsq(a, b): return math.sqrt(((a - b)**2).sum()) -def _complex_correlate(array, kernel, real_dtype, convolve=False): +def _complex_correlate(array, kernel, real_dtype, convolve=False, + mode="reflect", cval=0, ): """Utility to perform a reference complex-valued convolutions. When convolve==False, correlation is performed instead @@ -36,40 +37,54 @@ def _complex_correlate(array, kernel, real_dtype, convolve=False): if not convolve: kernel = kernel.conj() if complex_array and complex_kernel: + # use: real(cval) for array.real component + # imag(cval) for array.imag component output = ( - func(array.real, kernel.real, output=real_dtype) - - func(array.imag, kernel.imag, output=real_dtype) + - 1j * func(array.imag, kernel.real, output=real_dtype) + - 1j * func(array.real, kernel.imag, output=real_dtype) + func(array.real, kernel.real, output=real_dtype, + mode=mode, cval=numpy.real(cval)) - + func(array.imag, kernel.imag, output=real_dtype, + mode=mode, cval=numpy.imag(cval)) + + 1j * func(array.imag, kernel.real, output=real_dtype, + mode=mode, cval=numpy.imag(cval)) + + 1j * func(array.real, kernel.imag, output=real_dtype, + mode=mode, cval=numpy.real(cval)) ) elif complex_array: output = ( - func(array.real, kernel, output=real_dtype) + - 1j * func(array.imag, kernel, output=real_dtype) + func(array.real, kernel, output=real_dtype, mode=mode, + cval=numpy.real(cval)) + + 1j * func(array.imag, kernel, output=real_dtype, mode=mode, + cval=numpy.imag(cval)) ) elif complex_kernel: + # real array so cval is real too output = ( - func(array, kernel.real, output=real_dtype) + - 1j * func(array, kernel.imag, output=real_dtype) + func(array, kernel.real, output=real_dtype, mode=mode, cval=cval) + + 1j * func(array, kernel.imag, output=real_dtype, mode=mode, + cval=cval) ) return output class TestNdimageFilters: - def _validate_complex(self, array, kernel, type2): + def _validate_complex(self, array, kernel, type2, mode='reflect', cval=0): # utility for validating complex-valued correlations real_dtype = numpy.asarray([], dtype=type2).real.dtype expected = _complex_correlate( - array, kernel, real_dtype, convolve=False + array, kernel, real_dtype, convolve=False, mode=mode, cval=cval ) if array.ndim == 1: - correlate = functools.partial(ndimage.correlate1d, axis=-1) - convolve = functools.partial(ndimage.convolve1d, axis=-1) + correlate = functools.partial(ndimage.correlate1d, axis=-1, + mode=mode, cval=cval) + convolve = functools.partial(ndimage.convolve1d, axis=-1, + mode=mode, cval=cval) else: - correlate = ndimage.correlate - convolve = ndimage.convolve + correlate = functools.partial(ndimage.correlate, mode=mode, + cval=cval) + convolve = functools.partial(ndimage.convolve, mode=mode, + cval=cval) # test correlate output dtype output = correlate(array, kernel, output=type2) @@ -84,7 +99,7 @@ def _validate_complex(self, array, kernel, type2): # test convolve output dtype output = convolve(array, kernel, output=type2) expected = _complex_correlate( - array, kernel, real_dtype, convolve=True + array, kernel, real_dtype, convolve=True, mode=mode, cval=cval, ) assert_array_almost_equal(expected, output) assert_equal(output.dtype.type, type2) @@ -499,6 +514,36 @@ def test_correlate_complex_kernel(self, dtype_input, dtype_kernel, [4, 5, 6]], dtype_input) self._validate_complex(array, kernel, dtype_output) + @pytest.mark.parametrize('dtype_kernel', complex_types) + @pytest.mark.parametrize('dtype_input', types) + @pytest.mark.parametrize('dtype_output', complex_types) + @pytest.mark.parametrize('mode', ['grid-constant', 'constant']) + def test_correlate_complex_kernel_cval(self, dtype_input, dtype_kernel, + dtype_output, mode): + # test use of non-zero cval with complex inputs + # also verifies that mode 'grid-constant' does not segfault + kernel = numpy.array([[1, 0], + [0, 1 + 1j]], dtype_kernel) + array = numpy.array([[1, 2, 3], + [4, 5, 6]], dtype_input) + self._validate_complex(array, kernel, dtype_output, mode=mode, + cval=5.0) + + @pytest.mark.parametrize('dtype_kernel', complex_types) + @pytest.mark.parametrize('dtype_input', types) + def test_correlate_complex_kernel_invalid_cval(self, dtype_input, + dtype_kernel): + # cannot give complex cval with a real image + kernel = numpy.array([[1, 0], + [0, 1 + 1j]], dtype_kernel) + array = numpy.array([[1, 2, 3], + [4, 5, 6]], dtype_input) + for func in [ndimage.convolve, ndimage.correlate, ndimage.convolve1d, + ndimage.correlate1d]: + with pytest.raises(ValueError): + func(array, kernel, mode='constant', cval=5.0 + 1.0j, + output=numpy.complex64) + @pytest.mark.parametrize('dtype_kernel', complex_types) @pytest.mark.parametrize('dtype_input', types) @pytest.mark.parametrize('dtype_output', complex_types) @@ -508,6 +553,16 @@ def test_correlate1d_complex_kernel(self, dtype_input, dtype_kernel, array = numpy.array([1, 2, 3, 4, 5, 6], dtype_input) self._validate_complex(array, kernel, dtype_output) + @pytest.mark.parametrize('dtype_kernel', complex_types) + @pytest.mark.parametrize('dtype_input', types) + @pytest.mark.parametrize('dtype_output', complex_types) + def test_correlate_complex_kernel_cval(self, dtype_input, dtype_kernel, + dtype_output): + kernel = numpy.array([1, 1 + 1j], dtype_kernel) + array = numpy.array([1, 2, 3, 4, 5, 6], dtype_input) + self._validate_complex(array, kernel, dtype_output, mode='constant', + cval=5.0) + @pytest.mark.parametrize('dtype_kernel', types) @pytest.mark.parametrize('dtype_input', complex_types) @pytest.mark.parametrize('dtype_output', complex_types) @@ -528,6 +583,16 @@ def test_correlate1d_complex_input(self, dtype_input, dtype_kernel, array = numpy.array([1, 2j, 3, 1 + 4j, 5, 6j], dtype_input) self._validate_complex(array, kernel, dtype_output) + @pytest.mark.parametrize('dtype_kernel', types) + @pytest.mark.parametrize('dtype_input', complex_types) + @pytest.mark.parametrize('dtype_output', complex_types) + def test_correlate1d_complex_input_cval(self, dtype_input, dtype_kernel, + dtype_output): + kernel = numpy.array([1, 0, 1], dtype_kernel) + array = numpy.array([1, 2j, 3, 1 + 4j, 5, 6j], dtype_input) + self._validate_complex(array, kernel, dtype_output, mode='constant', + cval=5 - 3j) + @pytest.mark.parametrize('dtype', complex_types) @pytest.mark.parametrize('dtype_output', complex_types) def test_correlate_complex_input_and_kernel(self, dtype, dtype_output): @@ -537,6 +602,17 @@ def test_correlate_complex_input_and_kernel(self, dtype, dtype_output): [1 + 4j, 5, 6j]], dtype) self._validate_complex(array, kernel, dtype_output) + @pytest.mark.parametrize('dtype', complex_types) + @pytest.mark.parametrize('dtype_output', complex_types) + def test_correlate_complex_input_and_kernel_cval(self, dtype, + dtype_output): + kernel = numpy.array([[1, 0], + [0, 1 + 1j]], dtype) + array = numpy.array([[1, 2, 3], + [4, 5, 6]], dtype) + self._validate_complex(array, kernel, dtype_output, mode='constant', + cval=5.0 + 2.0j) + @pytest.mark.parametrize('dtype', complex_types) @pytest.mark.parametrize('dtype_output', complex_types) def test_correlate1d_complex_input_and_kernel(self, dtype, dtype_output): @@ -544,6 +620,15 @@ def test_correlate1d_complex_input_and_kernel(self, dtype, dtype_output): array = numpy.array([1, 2j, 3, 1 + 4j, 5, 6j], dtype) self._validate_complex(array, kernel, dtype_output) + @pytest.mark.parametrize('dtype', complex_types) + @pytest.mark.parametrize('dtype_output', complex_types) + def test_correlate1d_complex_input_and_kernel_cval(self, dtype, + dtype_output): + kernel = numpy.array([1, 1 + 1j], dtype) + array = numpy.array([1, 2j, 3, 1 + 4j, 5, 6j], dtype) + self._validate_complex(array, kernel, dtype_output, mode='constant', + cval=5.0 + 2.0j) + def test_gauss01(self): input = numpy.array([[1, 2, 3], [2, 4, 6]], numpy.float32) From f4eebf644f39ae2f84f78a53850c0a1de46a2d26 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 15 Dec 2020 20:11:20 -0500 Subject: [PATCH 07/13] fix copy/paste error in test name --- scipy/ndimage/tests/test_filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scipy/ndimage/tests/test_filters.py b/scipy/ndimage/tests/test_filters.py index a66bd306469a..5077a3536416 100644 --- a/scipy/ndimage/tests/test_filters.py +++ b/scipy/ndimage/tests/test_filters.py @@ -556,8 +556,8 @@ def test_correlate1d_complex_kernel(self, dtype_input, dtype_kernel, @pytest.mark.parametrize('dtype_kernel', complex_types) @pytest.mark.parametrize('dtype_input', types) @pytest.mark.parametrize('dtype_output', complex_types) - def test_correlate_complex_kernel_cval(self, dtype_input, dtype_kernel, - dtype_output): + def test_correlate1d_complex_kernel_cval(self, dtype_input, dtype_kernel, + dtype_output): kernel = numpy.array([1, 1 + 1j], dtype_kernel) array = numpy.array([1, 2, 3, 4, 5, 6], dtype_input) self._validate_complex(array, kernel, dtype_output, mode='constant', From f7045e13f3eac135ba469c7914a19b68c14b21a4 Mon Sep 17 00:00:00 2001 From: peterbell10 Date: Wed, 16 Dec 2020 17:43:32 +0000 Subject: [PATCH 08/13] BUG,MAINT: Ensure all Pool objects are closed by with statements (#13253) --- scipy/_lib/tests/test__util.py | 5 +---- scipy/integrate/_quad_vec.py | 3 +-- scipy/optimize/tests/test_minpack.py | 5 +---- scipy/stats/stats.py | 4 ++-- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/scipy/_lib/tests/test__util.py b/scipy/_lib/tests/test__util.py index bc891157e0a4..e79575c202a1 100644 --- a/scipy/_lib/tests/test__util.py +++ b/scipy/_lib/tests/test__util.py @@ -124,8 +124,7 @@ def test_mapwrapper_parallel(): assert_(excinfo.type is ValueError) # can also set a PoolWrapper up with a map-like callable instance - try: - p = Pool(2) + with Pool(2) as p: q = MapWrapper(p.map) assert_(q._own_pool is False) @@ -135,8 +134,6 @@ def test_mapwrapper_parallel(): # because it didn't create it out = p.map(np.sin, in_arg) assert_equal(list(out), out_arg) - finally: - p.close() # get our custom ones and a few from the "import *" cases diff --git a/scipy/integrate/_quad_vec.py b/scipy/integrate/_quad_vec.py index d8cdd6e0e4ae..9f838e94b6ea 100644 --- a/scipy/integrate/_quad_vec.py +++ b/scipy/integrate/_quad_vec.py @@ -267,7 +267,6 @@ def quad_vec(f, a, b, epsabs=1e-200, epsrel=1e-8, norm='2', cache_size=100e6, li else: norm_func = norm_funcs[norm] - mapwrapper = MapWrapper(workers) parallel_count = 128 min_intervals = 2 @@ -341,7 +340,7 @@ def quad_vec(f, a, b, epsabs=1e-200, epsrel=1e-8, norm='2', cache_size=100e6, li } # Process intervals - with mapwrapper: + with MapWrapper(workers) as mapwrapper: ier = NOT_CONVERGED while intervals and len(intervals) < limit: diff --git a/scipy/optimize/tests/test_minpack.py b/scipy/optimize/tests/test_minpack.py index 5bf1e25029a5..32c9109e05e9 100644 --- a/scipy/optimize/tests/test_minpack.py +++ b/scipy/optimize/tests/test_minpack.py @@ -39,11 +39,8 @@ def dummy_func(x, shape): def sequence_parallel(fs): - pool = ThreadPool(len(fs)) - try: + with ThreadPool(len(fs)) as pool: return pool.map(lambda f: f(), fs) - finally: - pool.terminate() # Function and Jacobian for tests of solvers for systems of nonlinear diff --git a/scipy/stats/stats.py b/scipy/stats/stats.py index c472223dc807..2b7dac89c87a 100644 --- a/scipy/stats/stats.py +++ b/scipy/stats/stats.py @@ -4840,9 +4840,9 @@ def _perm_test(x, y, stat, reps=1000, workers=-1, random_state=None): size=4, dtype=np.uint32)) for _ in range(reps)] # parallelizes with specified workers over number of reps and set seeds - mapwrapper = MapWrapper(workers) parallelp = _ParallelP(x=x, y=y, random_states=random_states) - null_dist = np.array(list(mapwrapper(parallelp, range(reps)))) + with MapWrapper(workers) as mapwrapper: + null_dist = np.array(list(mapwrapper(parallelp, range(reps)))) # calculate p-value and significant permutation map through list pvalue = (null_dist >= stat).sum() / reps From f5c1695c5ce293723959e6dd1110f4f35688cf29 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Fri, 18 Dec 2020 16:32:48 +1100 Subject: [PATCH 09/13] CI: fix macOS --- .github/workflows/macos.yml | 84 +++++++++++++++---------------------- 1 file changed, 34 insertions(+), 50 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1d3ed3781a52..0a165c94c071 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -14,12 +14,12 @@ on: jobs: test_macos: name: macOS Test Matrix - if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + # if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" runs-on: macos-latest strategy: max-parallel: 3 matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7] numpy-version: ['--upgrade numpy'] steps: @@ -29,59 +29,44 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Setup openblas + - name: Setup gfortran run: | - # this setup is originally taken from the .travis.yml - - # virtualenv needed for the multibuild steps - pip install virtualenv - brew install libmpc gcc@6 suitesparse swig - # The openblas binary used here was built using a gfortran older than 7, - # so it needs the older abi libgfortran. - export FC=gfortran-6 - export CC=gcc-6 - export CXX=g++-6 - mkdir gcc_aliases - pushd gcc_aliases - ln -s `which gcc-6` gcc - ln -s `which g++-6` g++ - ln -s `which gfortran-6` gfortran - # make gcc aliases in current dir - export PATH=$PWD/gcc_aliases:$PATH - popd - touch config.sh - git clone --depth=1 https://github.com/matthew-brett/multibuild.git + # this is based on the numpy azure pipeline setup. + set -xe + # same version of gfortran as the open-libs and numpy-wheel builds + curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg + GFORTRAN_SHA256=$(shasum -a 256 gfortran.dmg) + KNOWN_SHA256="d2d5ca5ba8332d63bbe23a07201c4a0a5d7e09ee56f0298a96775f928c3c4b30 gfortran.dmg" + if [ "$GFORTRAN_SHA256" != "$KNOWN_SHA256" ]; then + echo sha256 mismatch + exit 1 + fi + hdiutil attach -mountpoint /Volumes/gfortran gfortran.dmg + sudo installer -pkg /Volumes/gfortran/gfortran.pkg -target / + otool -L /usr/local/gfortran/lib/libgfortran.3.dylib + # Manually symlink gfortran-4.9 to plain gfortran for f2py. + # No longer needed after Feb 13 2020 as gfortran is already present + # and the attempted link errors. Keep this for future reference. + # ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran - # designed for travis, but probably work on github actions - source multibuild/common_utils.sh - source multibuild/travis_steps.sh - before_install - export CFLAGS="-arch x86_64" - export CXXFLAGS="-arch x86_64" - printenv + - name: Setup openblas + run: | + # this is based on the numpy azure pipeline setup. + set -xe + target=$(python tools/openblas_support.py) + ls -lR $target + # manually link to appropriate system paths + cp $target/lib/lib* /usr/local/lib/ + cp $target/include/* /usr/local/include/ - # Grab openblas - OPENBLAS_PATH=$(python tools/openblas_support.py) - # Copy it to the working directory - mv $OPENBLAS_PATH ./ + # otool -L /usr/local/lib/libopenblas* - # Modify the openblas dylib so it can be used in its current location - # Also make it use the current install location for libgfortran, libquadmath, and libgcc_s. - pushd openblas/lib - install_name_tool -id $PWD/libopenblasp-r*.dylib libopenblas.dylib - install_name_tool -change /usr/local/gfortran/lib/libgfortran.3.dylib /usr/local/Cellar/gcc@6/6.5.0_5/lib/gcc/6/libgfortran.3.dylib libopenblas.dylib - install_name_tool -change /usr/local/gfortran/lib/libquadmath.0.dylib /usr/local/Cellar/gcc@6/6.5.0_5/lib/gcc/6/libquadmath.0.dylib libopenblas.dylib - install_name_tool -change /usr/local/gfortran/lib/libgcc_s.1.dylib /usr/local/Cellar/gcc@6/6.5.0_5/lib/gcc/6/libgcc_s.1.dylib libopenblas.dylib - popd echo "[openblas]" > site.cfg echo "libraries = openblas" >> site.cfg - echo "library_dirs = $PWD/openblas/lib" >> site.cfg - echo "include_dirs = $PWD/openblas/include" >> site.cfg - echo "runtime_library_dirs = $PWD/openblas/lib" >> site.cfg - # remove a spurious gcc/gfortran toolchain install - rm -rf /usr/local/Cellar/gcc/9.2.0_2 - # - export PATH="$PATH:$PWD/openblas" + echo "library_dirs = /usr/local/lib" >> site.cfg + echo "include_dirs = /usr/local/include" >> site.cfg + echo "runtime_library_dirs = /usr/local/lib" >> site.cfg + - name: Install packages run: | pip install ${{ matrix.numpy-version }} @@ -89,5 +74,4 @@ jobs: - name: Test SciPy run: | - export DYLD_LIBRARY_PATH=/usr/local/Cellar/gcc@6/6.5.0_5/lib/gcc/6/:$DYLD_LIBRARY_PATH python -u runtests.py From 6558739e6c67984f607e70695ccbf419a53ed175 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Fri, 18 Dec 2020 16:55:16 +1100 Subject: [PATCH 10/13] CI: reactivate CI selectivity --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0a165c94c071..9622b38b0eb4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -14,7 +14,7 @@ on: jobs: test_macos: name: macOS Test Matrix - # if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" runs-on: macos-latest strategy: max-parallel: 3 From caa3eb278f903fb7658c42f75f409f85620f7228 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Fri, 18 Dec 2020 16:59:09 +1100 Subject: [PATCH 11/13] CI: a few other packages --- .github/workflows/macos.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 9622b38b0eb4..2136ee2b0ec7 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -19,7 +19,7 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: [3.7] + python-version: [3.7, 3.8, 3.9] numpy-version: ['--upgrade numpy'] steps: @@ -31,7 +31,7 @@ jobs: - name: Setup gfortran run: | - # this is based on the numpy azure pipeline setup. + # this is taken verbatim from the numpy azure pipeline setup. set -xe # same version of gfortran as the open-libs and numpy-wheel builds curl -L https://github.com/MacPython/gfortran-install/raw/master/archives/gfortran-4.9.0-Mavericks.dmg -o gfortran.dmg @@ -51,7 +51,7 @@ jobs: - name: Setup openblas run: | - # this is based on the numpy azure pipeline setup. + # this is taken verbatim from the numpy azure pipeline setup. set -xe target=$(python tools/openblas_support.py) ls -lR $target @@ -67,6 +67,10 @@ jobs: echo "include_dirs = /usr/local/include" >> site.cfg echo "runtime_library_dirs = /usr/local/lib" >> site.cfg + - name: A few other packages + run: | + brew install libmpc suitesparse swig + - name: Install packages run: | pip install ${{ matrix.numpy-version }} From 5e5ec9ff41d698edeb38a8b8b22f80f791824ec1 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Sat, 19 Dec 2020 16:52:13 -0500 Subject: [PATCH 12/13] CI: github actions: In the linux dbg tests, update apt before install packages. --- .github/workflows/linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 14ad8c11c984..d7ce4c7c84f1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,6 +19,7 @@ jobs: - uses: actions/checkout@v2 - name: Configuring Test Environment run: | + sudo apt-get update sudo apt install python3.7-dbg python3.7-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache swig libmpc-dev free -m python3.7-dbg --version # just to check From 550f87e8bc3aff58818679be2716e2b4fcb42691 Mon Sep 17 00:00:00 2001 From: Tyler Reddy Date: Mon, 21 Dec 2020 09:45:29 -0700 Subject: [PATCH 13/13] DOC: update 1.6.0 relnotes * update the SciPy `1.6.0` (rc2) release notes following backports to the maintenance branch * I've added Matti Picus to the author list because they made essential contributions to the wheels repo that were not picked up by our automated scripts * backports included for rc2 (so far): gh-13226 gh-13249 gh-13253 gh-13260 gh-13269 --- doc/release/1.6.0-notes.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/release/1.6.0-notes.rst b/doc/release/1.6.0-notes.rst index 34280f3e525a..c820b087e071 100644 --- a/doc/release/1.6.0-notes.rst +++ b/doc/release/1.6.0-notes.rst @@ -402,6 +402,7 @@ Authors * Sambit Panda * Dima Pasechnik * Tirth Patel + +* Matti Picus * Paweł Redzyński + * Vladimir Philipenko + * Philipp Thölke + @@ -449,7 +450,7 @@ Authors * ZhihuiChen0903 + * Jacob Zhong + -A total of 121 people contributed to this release. +A total of 122 people contributed to this release. People with a "+" by their names contributed a patch for the first time. This list of names is automatically generated, and may not be fully complete. @@ -589,6 +590,8 @@ Issues closed for 1.6.0 * `#13182 `__: Key appears twice in \`test_optimize.test_show_options\` * `#13191 `__: \`scipy.linalg.lapack.dgesjv\` overwrites original arrays if... * `#13207 `__: TST: Erratic test failure in test_cossin_separate +* `#13221 `__: BUG: pavement.py glitch +* `#13248 `__: ndimage: improper cval handling for complex-valued inputs Pull requests for 1.6.0 ----------------------- @@ -927,6 +930,13 @@ Pull requests for 1.6.0 * `#13190 `__: BUG: optimize: fix a duplicate key bug for \`test_show_options\` * `#13192 `__: BUG:linalg: Add overwrite option to gejsv wrapper * `#13194 `__: BUG: slsqp should be able to use rel_step +* `#13199 `__: [skip travis] DOC: 1.6.0 release notes * `#13203 `__: fix typos * `#13209 `__: TST:linalg: set the seed for cossin test * `#13212 `__: [DOC] Backtick and directive consistency. +* `#13217 `__: REL: add necessary setuptools and numpy version pins in pyproject.toml... +* `#13226 `__: BUG: pavement.py file handle fixes +* `#13249 `__: Handle cval correctly for ndimage functions with complex-valued... +* `#13253 `__: BUG,MAINT: Ensure all Pool objects are closed +* `#13260 `__: CI: fix macOS testing +* `#13269 `__: CI: github actions: In the linux dbg tests, update apt before...