From 3d805061e7f1b32df7c1cd5b40d272b34d50e487 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 23 Feb 2022 15:45:40 +0100 Subject: [PATCH] Bump up dependency versions on the CI config (#1272) --- azure-pipelines.yml | 24 +++---- conftest.py | 2 +- continuous_integration/install.sh | 2 +- joblib/backports.py | 101 +++++++++++++++++++++++++++++- joblib/compressor.py | 2 +- joblib/test/test_numpy_pickle.py | 10 +-- 6 files changed, 121 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b15e39499..90e59f602 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,20 +43,20 @@ jobs: PYTHON_VERSION: "3.9" # SKIP_TESTS: "true" SKLEARN_TESTS: "true" - linux_py38_distributed: + linux_py310_distributed: # To be updated regularly to use the most recent versions of the # dependencies. imageName: 'ubuntu-latest' - PYTHON_VERSION: "3.8" - EXTRA_CONDA_PACKAGES: "numpy=1.18 distributed=2.17" + PYTHON_VERSION: "3.10" + EXTRA_CONDA_PACKAGES: "numpy=1.22 distributed=2022.2.0" linux_py37_distributed: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" EXTRA_CONDA_PACKAGES: "numpy=1.15 distributed=2.13" - linux_py37_cython: + linux_py310_cython: imageName: 'ubuntu-latest' - PYTHON_VERSION: "3.7" - EXTRA_CONDA_PACKAGES: "numpy=1.15" + PYTHON_VERSION: "3.10" + EXTRA_CONDA_PACKAGES: "numpy=1.22" CYTHON: "true" linux_py37_no_multiprocessing_no_lzma: imageName: 'ubuntu-latest' @@ -68,15 +68,15 @@ jobs: imageName: 'ubuntu-latest' PYTHON_VERSION: "3.7" - windows_py38: + windows_py310: imageName: "windows-latest" - PYTHON_VERSION: "3.8" - EXTRA_CONDA_PACKAGES: "numpy=1.18" + PYTHON_VERSION: "3.10" + EXTRA_CONDA_PACKAGES: "numpy=1.22" - macos_py38: + macos_py310: imageName: "macos-latest" - PYTHON_VERSION: "3.8" - EXTRA_CONDA_PACKAGES: "numpy=1.18" + PYTHON_VERSION: "3.10" + EXTRA_CONDA_PACKAGES: "numpy=1.22" macos_py37_no_numpy: imageName: "macos-latest" PYTHON_VERSION: "3.7" diff --git a/conftest.py b/conftest.py index e246e951f..875e9b9b9 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,10 @@ -from distutils.version import LooseVersion import pytest from _pytest.doctest import DoctestItem import logging from joblib.parallel import mp +from joblib.backports import LooseVersion try: import lz4 except ImportError: diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index c942d8985..13cb8df75 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -15,7 +15,7 @@ create_new_conda_env() { # TODO: unpin pytest once it no longer causes test errors because of # PytestRemovedIn8Warning warnings TO_INSTALL="python=$PYTHON_VERSION pip pytest<7.0 $EXTRA_CONDA_PACKAGES" - conda create -n testenv --yes $TO_INSTALL + conda create -n testenv --yes -c conda-forge $TO_INSTALL source activate testenv } diff --git a/joblib/backports.py b/joblib/backports.py index cb2f7233d..3a14f1076 100644 --- a/joblib/backports.py +++ b/joblib/backports.py @@ -2,13 +2,112 @@ Backports of fixes for joblib dependencies """ import os +import re import time -from distutils.version import LooseVersion from os.path import basename from multiprocessing import util +class Version: + """Backport from deprecated distutils + + We maintain this backport to avoid introducing a new dependency on + `packaging`. + + We might rexplore this choice in the future if all major Python projects + introduce a dependency on packaging anyway. + """ + + def __init__(self, vstring=None): + if vstring: + self.parse(vstring) + + def __repr__(self): + return "%s ('%s')" % (self.__class__.__name__, str(self)) + + def __eq__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c == 0 + + def __lt__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c < 0 + + def __le__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c <= 0 + + def __gt__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c > 0 + + def __ge__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c >= 0 + + +class LooseVersion(Version): + """Backport from deprecated distutils + + We maintain this backport to avoid introducing a new dependency on + `packaging`. + + We might rexplore this choice in the future if all major Python projects + introduce a dependency on packaging anyway. + """ + + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + + def __init__(self, vstring=None): + if vstring: + self.parse(vstring) + + def parse(self, vstring): + # I've given up on thinking I can reconstruct the version string + # from the parsed tuple -- so I just store the string here for + # use by __str__ + self.vstring = vstring + components = [x for x in self.component_re.split(vstring) + if x and x != '.'] + for i, obj in enumerate(components): + try: + components[i] = int(obj) + except ValueError: + pass + + self.version = components + + def __str__(self): + return self.vstring + + def __repr__(self): + return "LooseVersion ('%s')" % str(self) + + def _cmp(self, other): + if isinstance(other, str): + other = LooseVersion(other) + elif not isinstance(other, LooseVersion): + return NotImplemented + + if self.version == other.version: + return 0 + if self.version < other.version: + return -1 + if self.version > other.version: + return 1 + + try: import numpy as np diff --git a/joblib/compressor.py b/joblib/compressor.py index 840fabe06..8361d37d4 100644 --- a/joblib/compressor.py +++ b/joblib/compressor.py @@ -2,7 +2,7 @@ import io import zlib -from distutils.version import LooseVersion +from joblib.backports import LooseVersion try: from threading import RLock diff --git a/joblib/test/test_numpy_pickle.py b/joblib/test/test_numpy_pickle.py index 2135e8fa7..8019ca536 100644 --- a/joblib/test/test_numpy_pickle.py +++ b/joblib/test/test_numpy_pickle.py @@ -282,11 +282,13 @@ def test_compress_mmap_mode_warning(tmpdir): numpy_pickle.dump(a, this_filename, compress=1) with warns(UserWarning) as warninfo: numpy_pickle.load(this_filename, mmap_mode='r+') + warninfo = [w.message for w in warninfo] assert len(warninfo) == 1 - assert (str(warninfo[0].message) == - 'mmap_mode "%(mmap_mode)s" is not compatible with compressed ' - 'file %(filename)s. "%(mmap_mode)s" flag will be ignored.' % - {'filename': this_filename, 'mmap_mode': 'r+'}) + assert ( + str(warninfo[0]) == + 'mmap_mode "r+" is not compatible with compressed ' + f'file {this_filename}. "r+" flag will be ignored.' + ) @with_numpy