From ea06d9eb42a4a92692afa3a863500f3bf9a8526f Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Sun, 1 Aug 2021 12:05:34 +0200 Subject: [PATCH 01/42] Trigger downstream CI to fix it From 272aa62ea1e6a1200d23c55b2012bca6204b22cf Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 19:31:35 +0100 Subject: [PATCH 02/42] CI trigger From 42adc3c3052d0e425f88cc6e8317beaf19aeb60c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 19:46:06 +0100 Subject: [PATCH 03/42] FIX Install cloudpickle master in CI? --- .github/workflows/testing.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6566800e..161cf794 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -204,9 +204,14 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} - - name: Install project and dependencies + - name: Install downstream project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh + - name: Install cloudpickle + shell: bash + run: | + pushd ../cloudpickle + python -m pip install -e . - name: Test the downstream project run: | bash ./.github/scripts/test_downstream_project.sh From dd7e45b91be159bd525e612e019d8f49379cd58d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 19:54:06 +0100 Subject: [PATCH 04/42] isolate faulty tests --- .github/scripts/test_downstream_project.sh | 2 +- .github/workflows/testing.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index f0a92f1f..dde2a192 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -python -m pytest -vl +python -m pytest -vl $ADDITIONAL_PYTEST_ARGS TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 161cf794..48fdbde5 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -195,6 +195,7 @@ jobs: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" PROJECT_URL: https://github.com/tomMoral/loky.git + ADDITIONAL_PYTEST_ARGS: "-k 'test_serialization'" strategy: matrix: python_version: [3.7] @@ -210,7 +211,6 @@ jobs: - name: Install cloudpickle shell: bash run: | - pushd ../cloudpickle python -m pip install -e . - name: Test the downstream project run: | From 4acc15b33543e44d6389c07f5e175ca0568926bb Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 19:55:46 +0100 Subject: [PATCH 05/42] fight with quotes --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 48fdbde5..c28e1e44 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -195,7 +195,7 @@ jobs: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" PROJECT_URL: https://github.com/tomMoral/loky.git - ADDITIONAL_PYTEST_ARGS: "-k 'test_serialization'" + ADDITIONAL_PYTEST_ARGS: -k "test_serialization" strategy: matrix: python_version: [3.7] From 4808777effe6c8af3b0a8d8c11fc9f2db693e6bc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 19:56:44 +0100 Subject: [PATCH 06/42] fight with quotes --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c28e1e44..3001e14e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -195,7 +195,7 @@ jobs: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" PROJECT_URL: https://github.com/tomMoral/loky.git - ADDITIONAL_PYTEST_ARGS: -k "test_serialization" + ADDITIONAL_PYTEST_ARGS: -k test_serialization strategy: matrix: python_version: [3.7] From 3c4f5059eb5bbdb3d6967d33e7b22e2cd15fba3f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:01:03 +0100 Subject: [PATCH 07/42] try different python version --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 3001e14e..5cfcaa3a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -198,10 +198,10 @@ jobs: ADDITIONAL_PYTEST_ARGS: -k test_serialization strategy: matrix: - python_version: [3.7] + python_version: [3.9] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} From fe1db95d0e852f0e07d1fbc729c4713a57bc8ac2 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:02:58 +0100 Subject: [PATCH 08/42] install the downstream project last relase and not master --- .github/scripts/install_downstream_project.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index cd9f0fba..51a1631e 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,5 +1,4 @@ python -m pip install $TEST_REQUIREMENTS pushd .. -git clone $PROJECT_URL; -python -m pip install ./$PROJECT; +python -m pip install $PROJECT; popd; From 0e8c7a44b27313f7891975076c0dfac5e0b48840 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:05:39 +0100 Subject: [PATCH 09/42] install last relase and not master --- .github/scripts/install_downstream_project.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index 51a1631e..d11dd4b1 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,4 +1,6 @@ python -m pip install $TEST_REQUIREMENTS pushd .. -python -m pip install $PROJECT; +git clone $PROJECT_URL; +git checkout 53fcdbbcd9d3ba1bf96fb2075f92fb1925e2584d +python -m pip install ./$PROJECT; popd; From bd57733a4852727dd2437abc500afab457a6c992 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:08:01 +0100 Subject: [PATCH 10/42] cd in the right place --- .github/scripts/install_downstream_project.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index d11dd4b1..44f380ea 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,6 +1,8 @@ python -m pip install $TEST_REQUIREMENTS pushd .. -git clone $PROJECT_URL; +git clone $PROJECT_URL +pushd $PROJECT git checkout 53fcdbbcd9d3ba1bf96fb2075f92fb1925e2584d +popd python -m pip install ./$PROJECT; popd; From 93018ce76c21a1f3ed7dc400b5f7294bd724dfe4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:12:21 +0100 Subject: [PATCH 11/42] bisect manually --- .github/scripts/install_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index 44f380ea..840ab562 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -2,7 +2,7 @@ python -m pip install $TEST_REQUIREMENTS pushd .. git clone $PROJECT_URL pushd $PROJECT -git checkout 53fcdbbcd9d3ba1bf96fb2075f92fb1925e2584d +git checkout 047b6eff2240c918cbc15282aad2241b6a0b612f popd python -m pip install ./$PROJECT; popd; From f928a9d8776a009237c7fb3898d8540027751227 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:16:40 +0100 Subject: [PATCH 12/42] bisect manually (bis) --- .github/scripts/install_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index 840ab562..36723889 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -2,7 +2,7 @@ python -m pip install $TEST_REQUIREMENTS pushd .. git clone $PROJECT_URL pushd $PROJECT -git checkout 047b6eff2240c918cbc15282aad2241b6a0b612f +git checkout HEAD~3 popd python -m pip install ./$PROJECT; popd; From 31c95866c6fc08a51da86cf1eb4a237a2cae0d09 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:18:56 +0100 Subject: [PATCH 13/42] use new loky (!!!) --- .github/scripts/install_downstream_project.sh | 5 +---- .github/workflows/testing.yml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index 36723889..cd9f0fba 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,8 +1,5 @@ python -m pip install $TEST_REQUIREMENTS pushd .. -git clone $PROJECT_URL -pushd $PROJECT -git checkout HEAD~3 -popd +git clone $PROJECT_URL; python -m pip install ./$PROJECT; popd; diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5cfcaa3a..f6fcda2b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -194,7 +194,7 @@ jobs: env: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" - PROJECT_URL: https://github.com/tomMoral/loky.git + PROJECT_URL: https://github.com/joblib/loky.git ADDITIONAL_PYTEST_ARGS: -k test_serialization strategy: matrix: From 2cdbeae14b224e87261c6118001adf2552d29195 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:20:08 +0100 Subject: [PATCH 14/42] try running full loky test suite --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f6fcda2b..31509c60 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -195,7 +195,6 @@ jobs: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" PROJECT_URL: https://github.com/joblib/loky.git - ADDITIONAL_PYTEST_ARGS: -k test_serialization strategy: matrix: python_version: [3.9] From eeab7a9007ab9549e7d679cf7629eb6071fcb585 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:29:49 +0100 Subject: [PATCH 15/42] try upgrading ray version --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 31509c60..2ca55b51 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -232,7 +232,7 @@ jobs: - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt - python -m pip install setproctitle psutil ray==0.8.6 + python -m pip install setproctitle psutil ray==1.6.0 PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py git clone https://github.com/ray-project/ray.git ../ray From e850ef307b75cb0623a7d499ec8973ce39fc48ec Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:45:06 +0100 Subject: [PATCH 16/42] try to set up a ray's development env --- .github/workflows/testing.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2ca55b51..83c60851 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -232,7 +232,15 @@ jobs: - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt - python -m pip install setproctitle psutil ray==1.6.0 + python -m pip install setproctitle psutil + # from https://docs.ray.io/en/master/development.html#building-ray + pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl + pushd .. + git clone https://github.com/ray-project/ray.git + pushd ray + python python/ray/setup-dev.py + popd + popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py git clone https://github.com/ray-project/ray.git ../ray From 4bffec1ce8ad4fa3606d4b602d15a08c9c7e0ca3 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:48:12 +0100 Subject: [PATCH 17/42] move to python 3.8 --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 83c60851..1a1b97e7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -222,10 +222,10 @@ jobs: PROJECT: ray strategy: matrix: - python_version: [3.7] + python_version: [3.8] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} From 567b5a507b1f3b4d5ce34327088337d65afd1f98 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:50:59 +0100 Subject: [PATCH 18/42] dont ask for confirmation when setup ray dev --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1a1b97e7..cbc43fa6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -238,7 +238,7 @@ jobs: pushd .. git clone https://github.com/ray-project/ray.git pushd ray - python python/ray/setup-dev.py + python python/ray/setup-dev.py --yes popd popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") From 57e8bb82e693f7b7d26090710fe3bf6b6bab380c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:56:26 +0100 Subject: [PATCH 19/42] more dir fixes --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cbc43fa6..208f0071 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -243,7 +243,6 @@ jobs: popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py - git clone https://github.com/ray-project/ray.git ../ray cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py From 4e0d172fe1e4df0ecdc0d0c818574fba2b04d21d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 20:59:30 +0100 Subject: [PATCH 20/42] remove probably duplicate setup line --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 208f0071..d4b0d614 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -243,7 +243,7 @@ jobs: popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py - cp -R ../ray/python/ray/tests $PROJECT_DIR/tests + #cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py From f7d6c5119976c695b02ef53941d22969b80313b0 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 21:05:19 +0100 Subject: [PATCH 21/42] try to disable some problematic unrelated tests --- .github/workflows/testing.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d4b0d614..537780f4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -156,8 +156,15 @@ jobs: - name: Install project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh + - name: Install cloudpickle + shell: bash + run: | + python -m pip install -e . - name: Test the downstream project run: | + # FIXME ipv6-related failures on Ubuntu github actions CI + # https://github.com/dask/distributed/issues/4514 + export DISABLE_IPV6=1 bash ./.github/scripts/test_downstream_project.sh joblib-downstream-build: From ae1fa9d4b770f0418ad67f637f599678dc0d97bf Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 21:07:11 +0100 Subject: [PATCH 22/42] add missing distributed test dependency --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 537780f4..e5ada9da 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -141,7 +141,7 @@ jobs: if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')" env: PROJECT: distributed - TEST_REQUIREMENTS: pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 + TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 PROJECT_URL: https://github.com/dask/distributed.git PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" strategy: From 5e27b1100e42c800ea92f510beb14355e21d77ef Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 21:17:54 +0100 Subject: [PATCH 23/42] isolate problematic test --- .github/scripts/test_downstream_project.sh | 2 +- .github/workflows/testing.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index dde2a192..3fd19f20 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -python -m pytest -vl $ADDITIONAL_PYTEST_ARGS +python -m pytest -vl $ADDITIONAL_PYTEST_ARGS -k "test_pickle_empty" TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e5ada9da..7f852532 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -146,10 +146,10 @@ jobs: PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" strategy: matrix: - python_version: [3.7] + python_version: [3.8] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} From 3db9509c0296adaf251e6973ed5211094ee23c6f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 8 Sep 2021 21:20:16 +0100 Subject: [PATCH 24/42] bisect faulty python version? --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 7f852532..e5ada9da 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -146,10 +146,10 @@ jobs: PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" strategy: matrix: - python_version: [3.8] + python_version: [3.7] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.8 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} From f8380e3152014bab0bdd002eb51f81ad4c036cd6 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 18:10:31 +0100 Subject: [PATCH 25/42] try jakirkham's PR --- .github/scripts/install_downstream_project.sh | 3 +++ .github/workflows/testing.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index cd9f0fba..85345b12 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,5 +1,8 @@ python -m pip install $TEST_REQUIREMENTS pushd .. git clone $PROJECT_URL; +pushd $PROJECT +git checkout fixup_tst_pickle +popd python -m pip install ./$PROJECT; popd; diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e5ada9da..50e2bb52 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -142,7 +142,7 @@ jobs: env: PROJECT: distributed TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 - PROJECT_URL: https://github.com/dask/distributed.git + PROJECT_URL: https://github.com/jakirkham/distributed.git PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" strategy: matrix: From 632a01de6961f8fd8201a5e667f0b6e470773f36 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 18:12:17 +0100 Subject: [PATCH 26/42] test pickle related tests --- .github/scripts/test_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 3fd19f20..0f69089c 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -python -m pytest -vl $ADDITIONAL_PYTEST_ARGS -k "test_pickle_empty" +python -m pytest -vl $ADDITIONAL_PYTEST_ARGS -k "pickle or serializ" TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then From 4891d74f9bfb179945d672d6e830f753bf1ab443 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 18:18:08 +0100 Subject: [PATCH 27/42] test full distributed test suite --- .github/scripts/test_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 0f69089c..f0a92f1f 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -python -m pytest -vl $ADDITIONAL_PYTEST_ARGS -k "pickle or serializ" +python -m pytest -vl TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then From aec1a2e1eaba2e57f1e856fbe9d9529c4e8e26c5 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 19:23:06 +0100 Subject: [PATCH 28/42] temporarily exclude faulty distributed test --- .github/scripts/install_downstream_project.sh | 3 --- .github/scripts/test_downstream_project.sh | 2 +- .github/workflows/testing.yml | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/scripts/install_downstream_project.sh b/.github/scripts/install_downstream_project.sh index 85345b12..cd9f0fba 100644 --- a/.github/scripts/install_downstream_project.sh +++ b/.github/scripts/install_downstream_project.sh @@ -1,8 +1,5 @@ python -m pip install $TEST_REQUIREMENTS pushd .. git clone $PROJECT_URL; -pushd $PROJECT -git checkout fixup_tst_pickle -popd python -m pip install ./$PROJECT; popd; diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index f0a92f1f..439114ec 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -python -m pytest -vl +python -m pytest -vl $PYTEST_ADDOPTS TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 50e2bb52..40ff43d5 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -142,8 +142,8 @@ jobs: env: PROJECT: distributed TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 - PROJECT_URL: https://github.com/jakirkham/distributed.git - PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" + PROJECT_URL: https://github.com/dask/distributed.git + PYTEST_ADDOPTS: " -k \"not test_pickle_empty\"" strategy: matrix: python_version: [3.7] From 9d058879a473a2cd8ccf85c5eb475695df033e52 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 19:32:41 +0100 Subject: [PATCH 29/42] debug --- .github/scripts/test_downstream_project.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 439114ec..9584c66e 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,4 +1,5 @@ pushd ../$PROJECT +echo $PYTEST_ADDOPTS python -m pytest -vl $PYTEST_ADDOPTS TEST_RETURN_CODE=$? popd From 2ca7223c8e18779d0f87a246825072fdef196a85 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 19:34:20 +0100 Subject: [PATCH 30/42] debug --- .github/scripts/test_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 9584c66e..274e736e 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,6 +1,6 @@ pushd ../$PROJECT echo $PYTEST_ADDOPTS -python -m pytest -vl $PYTEST_ADDOPTS +python -m pytest -vl "${PYTEST_ADDOPTS}" TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then From e064cd4738be969414456e41fb5c2d214daac8d4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 19:42:41 +0100 Subject: [PATCH 31/42] debug --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 40ff43d5..05563f62 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -143,7 +143,7 @@ jobs: PROJECT: distributed TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 PROJECT_URL: https://github.com/dask/distributed.git - PYTEST_ADDOPTS: " -k \"not test_pickle_empty\"" + PYTEST_ADDOPTS: "-k \'not test_pickle_empty\'" strategy: matrix: python_version: [3.7] From 456231048e17e27b5d0fe8df0a7e333390a0e911 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 19:53:07 +0100 Subject: [PATCH 32/42] debug --- .github/scripts/test_downstream_project.sh | 2 +- .github/workflows/testing.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 274e736e..0ae7ccde 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,6 +1,6 @@ pushd ../$PROJECT echo $PYTEST_ADDOPTS -python -m pytest -vl "${PYTEST_ADDOPTS}" +python -m pytest -vl "${PYTEST_ADDOPTS[@]}" TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 05563f62..f40c2df8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -143,7 +143,6 @@ jobs: PROJECT: distributed TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 PROJECT_URL: https://github.com/dask/distributed.git - PYTEST_ADDOPTS: "-k \'not test_pickle_empty\'" strategy: matrix: python_version: [3.7] @@ -165,6 +164,7 @@ jobs: # FIXME ipv6-related failures on Ubuntu github actions CI # https://github.com/dask/distributed/issues/4514 export DISABLE_IPV6=1 + export PYTEST_ADDOPTS=("-k" "not test_pickle_empty") bash ./.github/scripts/test_downstream_project.sh joblib-downstream-build: From f5c07d8b7796e952704fb6acb643856b78969a3a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:04:34 +0100 Subject: [PATCH 33/42] debug --- .github/scripts/test_downstream_project.sh | 2 +- .github/workflows/testing.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 0ae7ccde..8470d6f5 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -echo $PYTEST_ADDOPTS +echo ${PYTEST_ADDOPTS[@]}" python -m pytest -vl "${PYTEST_ADDOPTS[@]}" TEST_RETURN_CODE=$? popd diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f40c2df8..e7f0a46f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -164,7 +164,7 @@ jobs: # FIXME ipv6-related failures on Ubuntu github actions CI # https://github.com/dask/distributed/issues/4514 export DISABLE_IPV6=1 - export PYTEST_ADDOPTS=("-k" "not test_pickle_empty") + export PYTEST_ADDOPTS=("-k" \"not test_pickle_empty\") bash ./.github/scripts/test_downstream_project.sh joblib-downstream-build: From 631690bb9d2c3f0063c59e33bdb40c8acf606eca Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:07:33 +0100 Subject: [PATCH 34/42] debug --- .github/scripts/test_downstream_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 8470d6f5..5071be4c 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,5 @@ pushd ../$PROJECT -echo ${PYTEST_ADDOPTS[@]}" +echo "${PYTEST_ADDOPTS[@]}" python -m pytest -vl "${PYTEST_ADDOPTS[@]}" TEST_RETURN_CODE=$? popd From 289cf0249caba7f6b5748924ecf54d107b19901c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:17:06 +0100 Subject: [PATCH 35/42] debug --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e7f0a46f..f40c2df8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -164,7 +164,7 @@ jobs: # FIXME ipv6-related failures on Ubuntu github actions CI # https://github.com/dask/distributed/issues/4514 export DISABLE_IPV6=1 - export PYTEST_ADDOPTS=("-k" \"not test_pickle_empty\") + export PYTEST_ADDOPTS=("-k" "not test_pickle_empty") bash ./.github/scripts/test_downstream_project.sh joblib-downstream-build: From daabf0a29b5bdfce1bcc19cf1bff8e18a0c34b1d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:22:05 +0100 Subject: [PATCH 36/42] debug --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f40c2df8..01da481a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -160,6 +160,7 @@ jobs: run: | python -m pip install -e . - name: Test the downstream project + shell: bash run: | # FIXME ipv6-related failures on Ubuntu github actions CI # https://github.com/dask/distributed/issues/4514 From 5b671562cdab8c2ade4568210479e3ceb82f766a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:27:03 +0100 Subject: [PATCH 37/42] debug --- .github/scripts/test_downstream_project.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 5071be4c..51e28ed1 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,8 +1,9 @@ pushd ../$PROJECT +echo "${DISABLE_IPV6}" echo "${PYTEST_ADDOPTS[@]}" -python -m pytest -vl "${PYTEST_ADDOPTS[@]}" -TEST_RETURN_CODE=$? -popd -if [[ "$TEST_RETURN_CODE" != "0" ]]; then - exit $TEST_RETURN_CODE -fi +# python -m pytest -vl "${PYTEST_ADDOPTS[@]}" +# TEST_RETURN_CODE=$? +# popd +# if [[ "$TEST_RETURN_CODE" != "0" ]]; then +# exit $TEST_RETURN_CODE +# fi From cefc2aeccf7f9c0de986202d036292e8684e1d95 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:30:35 +0100 Subject: [PATCH 38/42] debug --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 01da481a..a14ee053 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -166,7 +166,7 @@ jobs: # https://github.com/dask/distributed/issues/4514 export DISABLE_IPV6=1 export PYTEST_ADDOPTS=("-k" "not test_pickle_empty") - bash ./.github/scripts/test_downstream_project.sh + source ./.github/scripts/test_downstream_project.sh joblib-downstream-build: runs-on: ubuntu-latest From 7c34a045e1197d7bd00165684020b24734b58a25 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 20:31:52 +0100 Subject: [PATCH 39/42] debug --- .github/scripts/test_downstream_project.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index 51e28ed1..31aae20e 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,9 +1,9 @@ pushd ../$PROJECT echo "${DISABLE_IPV6}" echo "${PYTEST_ADDOPTS[@]}" -# python -m pytest -vl "${PYTEST_ADDOPTS[@]}" -# TEST_RETURN_CODE=$? -# popd -# if [[ "$TEST_RETURN_CODE" != "0" ]]; then -# exit $TEST_RETURN_CODE -# fi +python -m pytest -vl "${PYTEST_ADDOPTS[@]}" +TEST_RETURN_CODE=$? +popd +if [[ "$TEST_RETURN_CODE" != "0" ]]; then + exit $TEST_RETURN_CODE +fi From df86f2febaccc778c105fc989486a17cf38977b9 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 21:04:32 +0100 Subject: [PATCH 40/42] ci trigger From 7d6bb53637956222ae6407d592e25e478b012e8b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 21:35:17 +0100 Subject: [PATCH 41/42] skip distributed flaky test --- .github/workflows/testing.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a14ee053..9d8dc141 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -165,7 +165,11 @@ jobs: # FIXME ipv6-related failures on Ubuntu github actions CI # https://github.com/dask/distributed/issues/4514 export DISABLE_IPV6=1 - export PYTEST_ADDOPTS=("-k" "not test_pickle_empty") + # Skipping + # - test_pickle_empty until https://github.com/dask/distributed/pull/5303 gets merged, + # - test_dont_steal_unknown_function (https://github.com/dask/distributed/issues/3574) + # See https://github.com/cloudpipe/cloudpickle/pull/432 + export PYTEST_ADDOPTS=("-k" "not test_pickle_empty and not test_dont_steal_unknown_functions") source ./.github/scripts/test_downstream_project.sh joblib-downstream-build: From 6a789996c2e6e0598033e06284f36152a645b160 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 9 Sep 2021 22:08:29 +0100 Subject: [PATCH 42/42] various cleanups --- .github/workflows/testing.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9d8dc141..8baca4b1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -155,7 +155,7 @@ jobs: - name: Install project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh - - name: Install cloudpickle + - name: Install developed version of cloudpickle shell: bash run: | python -m pip install -e . @@ -209,17 +209,17 @@ jobs: PROJECT_URL: https://github.com/joblib/loky.git strategy: matrix: - python_version: [3.9] + python_version: [3.7] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.9 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} - name: Install downstream project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh - - name: Install cloudpickle + - name: Install developed version of cloudpickle shell: bash run: | python -m pip install -e . @@ -255,7 +255,6 @@ jobs: popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py - #cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py