From 1e32cbffea338433dbbb1c8b17ff895e403afbe6 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 14 Sep 2022 22:45:45 +0200 Subject: [PATCH 01/10] Attempt at making test_child_raises_parent_exits_cleanly more reliable on macos --- joblib/test/test_memmapping.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/joblib/test/test_memmapping.py b/joblib/test/test_memmapping.py index 5488ac8a7..06c315780 100644 --- a/joblib/test/test_memmapping.py +++ b/joblib/test/test_memmapping.py @@ -813,6 +813,7 @@ def test_child_raises_parent_exits_cleanly(backend): # - the resource_tracker does not emit any warnings. cmd = """if 1: import os + from pathlib import Path from time import sleep import numpy as np @@ -824,9 +825,9 @@ def test_child_raises_parent_exits_cleanly(backend): def get_temp_folder(parallel_obj, backend): if "{b}" == "loky": - return p._backend._workers._temp_folder + return Path(p._backend._workers._temp_folder) else: - return p._backend._pool._temp_folder + return Path(p._backend._pool._temp_folder) if __name__ == "__main__": @@ -839,13 +840,22 @@ def get_temp_folder(parallel_obj, backend): # the temporary folder should be deleted by the end of this # call but apparently on some file systems, this takes # some time to be visible. + # + # We attempt to write into the temporary folder to test for + # its existence and we wait for a maximum of 10 seconds. for i in range(100): - if not os.path.exists(temp_folder): + try: + with open(temp_folder / "some_file.txt", "w") as f: + f.write("some content") + except FileNotFoundError: + # temp_folder has been deleted, all is fine break + + # ... else, wait a bit and try again sleep(.1) else: raise AssertionError( - temp_folder + " was not deleted" + str(temp_folder) + " was not deleted" ) from e """.format(b=backend) env = os.environ.copy() From 5dcd177e0aab7a76a387e685480b927e2e1ba518 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 14 Sep 2022 22:52:08 +0200 Subject: [PATCH 02/10] Trigger CI From b1ae2a1c224f5cb53b9f8e36cc588d266f6559c8 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 14 Sep 2022 23:24:13 +0200 Subject: [PATCH 03/10] Trigger CI From d230264115c3a250848fffe9470f4cbf59dda158 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 06:59:38 +0200 Subject: [PATCH 04/10] Trigger CI From 6fa730ee4347da8748b789ae124dc3e09d2c2abd Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 07:22:25 +0200 Subject: [PATCH 05/10] Trigger CI From f08a52ab3514badd6b7af9ffd3f8b69acc772df9 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 07:44:09 +0200 Subject: [PATCH 06/10] Trigger CI From 0c39006b0a595727fc053b4b471467a0df9752e8 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 08:06:47 +0200 Subject: [PATCH 07/10] Trigger CI From f739f4d994b5dadb03e6a47399504a1fe6120d78 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 08:39:57 +0200 Subject: [PATCH 08/10] Trigger CI From d76ad73b713f3caf8bb8808747ad6978522cd822 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 10:06:50 +0200 Subject: [PATCH 09/10] Trigger CI From 8daadbc273384a8527d0e73ca59d415ec0590938 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Thu, 15 Sep 2022 10:39:59 +0200 Subject: [PATCH 10/10] Move test_nested_loop_error_in_grandchild_resource_tracker_silent upstream in joblib --- joblib/test/test_memmapping.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/joblib/test/test_memmapping.py b/joblib/test/test_memmapping.py index 06c315780..a666c5aa0 100644 --- a/joblib/test/test_memmapping.py +++ b/joblib/test/test_memmapping.py @@ -586,39 +586,6 @@ def parallel_raise(array, temp_dirs): assert b"resource_tracker" not in err, err.decode() -@with_numpy -@with_multiprocessing -def test_nested_loop_error_in_grandchild_resource_tracker_silent(): - # Safety smoke test: test that nested parallel calls using the loky backend - # don't yield noisy resource_tracker outputs when the grandchild errors - # out. - cmd = '''if 1: - from joblib import Parallel, delayed - - - def raise_error(i): - raise ValueError - - - def nested_loop(f): - Parallel(backend="loky", n_jobs=2)( - delayed(f)(i) for i in range(10) - ) - - - if __name__ == "__main__": - Parallel(backend="loky", n_jobs=2)( - delayed(nested_loop)(func) for func in [raise_error] - ) - ''' - p = subprocess.Popen([sys.executable, '-c', cmd], - stderr=subprocess.PIPE, stdout=subprocess.PIPE) - p.wait() - out, err = p.communicate() - assert p.returncode == 1, out.decode() - assert b"resource_tracker" not in err, err.decode() - - @with_numpy @with_multiprocessing @parametrize("backend", ["multiprocessing", "loky"])