diff --git a/CHANGES.rst b/CHANGES.rst index 62bf7edc0..cecb65c53 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,10 @@ Latest changes Development version ------------------- +- Fix argument 'max_nbytes' being used if explicitely set to something + else than None even though 'mmap_mode' was set to None. + https://github.com/joblib/joblib/pull/1325 + - Make sure that joblib works even when multiprocessing is not available, for instance with Pyodide https://github.com/joblib/joblib/pull/1256 diff --git a/joblib/parallel.py b/joblib/parallel.py index 1c2fe18f7..0bb368064 100644 --- a/joblib/parallel.py +++ b/joblib/parallel.py @@ -495,9 +495,9 @@ class Parallel(Logger): parallel_backend. verbose: int, optional The verbosity level: if non zero, progress messages are - printed. Above 50, the output is sent to stdout. + printed. If above 50, the output is sent to stdout. The frequency of the messages increases with the verbosity level. - If it more than 10, all iterations are reported. + If above 10, all iterations are reported. timeout: float, optional Timeout limit for each task to complete. If any task takes longer a TimeOutError will be raised. Only applied when n_jobs != 1 @@ -540,7 +540,8 @@ class Parallel(Logger): Threshold on the size of arrays passed to the workers that triggers automated memory mapping in temp_folder. Can be an int in Bytes, or a human-readable string, e.g., '1M' for 1 megabyte. - Use None to disable memmapping of large arrays. + Use None to disable automatic memmapping of large arrays (small + arrays can ignore this argument). Only active when backend="loky" or "multiprocessing". mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, default: 'r' Memmapping mode for numpy arrays passed to workers. None will @@ -688,6 +689,9 @@ def __init__(self, n_jobs=None, backend=None, verbose=0, timeout=None, self._id = uuid4().hex self._reducer_callback = None + if mmap_mode is None: + max_nbytes = None + if isinstance(max_nbytes, str): max_nbytes = memstr_to_bytes(max_nbytes) diff --git a/joblib/test/test_memmapping.py b/joblib/test/test_memmapping.py index b752027d8..aded3dc8d 100644 --- a/joblib/test/test_memmapping.py +++ b/joblib/test/test_memmapping.py @@ -888,6 +888,35 @@ def test_memmapping_pool_for_large_arrays_disabled(factory, tmpdir): del p +@with_numpy +@with_multiprocessing +@parametrize("factory", [MemmappingPool, TestExecutor.get_memmapping_executor], + ids=["multiprocessing", "loky"]) +def test_memmapping_pool_disabled_by_mmap_mode_to_None(factory, tmpdir): + """Check that memmapping is in effect disabled by setting + mmap_mode to None while setting max_nbytes to an explicit + value.""" + # Set mmap_mode to None to disable memmapping feature + p = factory(3, mmap_mode=None, max_nbytes=40, temp_folder=tmpdir.strpath) + try: + + # Check that the tempfolder is empty + assert os.listdir(tmpdir.strpath) == [] + + # Try with a file largish than the memmap threshold of 40 bytes + large = np.ones(100, dtype=np.float64) + assert large.nbytes == 800 + p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])]) + + # Check that the tempfolder is still empty + assert os.listdir(tmpdir.strpath) == [] + + finally: + # Cleanup open file descriptors + p.terminate() + del p + + @with_numpy @with_multiprocessing @with_dev_shm