From 19385720cdc42ec85b5e09464963469700cbe40a Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Fri, 10 Sep 2021 11:01:54 +0200 Subject: [PATCH] Fix downstream CI (#432) Co-authored-by: Pierre Glaser --- .github/scripts/test_downstream_project.sh | 4 ++- .github/workflows/testing.yml | 42 ++++++++++++++++------ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/scripts/test_downstream_project.sh b/.github/scripts/test_downstream_project.sh index f0a92f1f9..31aae20ea 100644 --- a/.github/scripts/test_downstream_project.sh +++ b/.github/scripts/test_downstream_project.sh @@ -1,5 +1,7 @@ pushd ../$PROJECT -python -m pytest -vl +echo "${DISABLE_IPV6}" +echo "${PYTEST_ADDOPTS[@]}" +python -m pytest -vl "${PYTEST_ADDOPTS[@]}" TEST_RETURN_CODE=$? popd if [[ "$TEST_RETURN_CODE" != "0" ]]; then diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6566800e8..8baca4b12 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -141,9 +141,8 @@ jobs: if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')" env: PROJECT: distributed - TEST_REQUIREMENTS: pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 + TEST_REQUIREMENTS: cryptography pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3 PROJECT_URL: https://github.com/dask/distributed.git - PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args and not test_remote_access\"" strategy: matrix: python_version: [3.7] @@ -156,9 +155,22 @@ jobs: - name: Install project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh + - name: Install developed version of cloudpickle + shell: bash + run: | + python -m pip install -e . - name: Test the downstream project + shell: bash run: | - bash ./.github/scripts/test_downstream_project.sh + # FIXME ipv6-related failures on Ubuntu github actions CI + # https://github.com/dask/distributed/issues/4514 + export DISABLE_IPV6=1 + # Skipping + # - test_pickle_empty until https://github.com/dask/distributed/pull/5303 gets merged, + # - test_dont_steal_unknown_function (https://github.com/dask/distributed/issues/3574) + # See https://github.com/cloudpipe/cloudpickle/pull/432 + export PYTEST_ADDOPTS=("-k" "not test_pickle_empty and not test_dont_steal_unknown_functions") + source ./.github/scripts/test_downstream_project.sh joblib-downstream-build: runs-on: ubuntu-latest @@ -194,7 +206,7 @@ jobs: env: PROJECT: loky TEST_REQUIREMENTS: "pytest psutil" - PROJECT_URL: https://github.com/tomMoral/loky.git + PROJECT_URL: https://github.com/joblib/loky.git strategy: matrix: python_version: [3.7] @@ -204,9 +216,13 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} - - name: Install project and dependencies + - name: Install downstream project and dependencies run: | bash ./.github/scripts/install_downstream_project.sh + - name: Install developed version of cloudpickle + shell: bash + run: | + python -m pip install -e . - name: Test the downstream project run: | bash ./.github/scripts/test_downstream_project.sh @@ -218,21 +234,27 @@ jobs: PROJECT: ray strategy: matrix: - python_version: [3.7] + python_version: [3.8] steps: - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: python-version: ${{ matrix.python_version }} - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt - python -m pip install setproctitle psutil ray==0.8.6 + python -m pip install setproctitle psutil + # from https://docs.ray.io/en/master/development.html#building-ray + pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl + pushd .. + git clone https://github.com/ray-project/ray.git + pushd ray + python python/ray/setup-dev.py --yes + popd + popd PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py - git clone https://github.com/ray-project/ray.git ../ray - cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py