testing.yml

name: Automated Tests

on:
  push:
    branches: master
  pull_request:
    branches: master

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: 3.7
    - name: Install flake8
      shell: bash
      run: |
        python -V
        python -m pip install flake8
    - name: Run flake8 on diff with upstream/master
      shell: bash
      run: |
        bash ./.github/scripts/flake8_diff.sh

  build:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python_version: [3.6, 3.7, 3.8, 3.9, "3.10-dev", "pypy-3.8"]
        exclude:
          # Do not test all minor versions on all platforms, especially if they
          # are not the oldest/newest supported versions
          - os: windows-latest
            python_version: 3.6
          - os: windows-latest
            python_version: 3.7
          - os: windows-latest
            python_version: 3.8
            # as of  4/02/2020, psutil won't build under PyPy + Windows
          - os: windows-latest
            python_version: "pypy-3.8"
          - os: macos-latest
            python_version: 3.6
          - os: macos-latest
            python_version: 3.7
          - os: macos-latest
            python_version: 3.8
          - os: macos-latest
            # numpy triggers: RuntimeError: Polyfit sanity test emitted a
            # warning
            python_version: "pypy-3.8"

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python_version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      shell: bash
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e .
        python -m pip install -r dev-requirements.txt
        python ci/install_coverage_subprocess_pth.py
        export
    - name: Install supported dependencies (only test in Python 3.6)
      shell: bash
      run: python -m pip install typing-extensions
      if: matrix.python_version == '3.6'
    - name: Display Python version
      shell: bash
      run: python -c "import sys; print(sys.version)"
    - name: Look for syntax errors/undefined names
      shell: bash
      run: |
        python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \
          --show-source --statistics
    - name: Test with pytest
      shell: bash
      run: |
        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
          PYTHONPATH='.:tests' python -m pytest -r s
        coverage combine --append
        coverage xml -i
    - name: Publish coverage results
      uses: codecov/codecov-action@v1
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  python-nightly:
    runs-on: ubuntu-18.04
    # This entry is made optional for now, see https://github.com/cloudpipe/cloudpickle/pull/420
    if: "contains(github.event.pull_request.labels.*.name, 'ci python-nightly')"
    steps:
    - uses: actions/checkout@v1
    - name: Install Python from ppa:deadsnakes/nightly
      run: |
        sudo add-apt-repository ppa:deadsnakes/nightly
        sudo apt update
        sudo apt install python3.11 python3.11-venv python3.11-dev
        python3.11 -m venv nightly-venv
        echo "$PWD/nightly-venv/bin" >> $GITHUB_PATH
    - name: Display Python version
      run: python -c "import sys; print(sys.version)"
    - name: Install project and dependencies
      run: |
        set -e
        python -m pip install --upgrade pip
        python -m pip install -r dev-requirements.txt
        python -m pip install -e .
        python ci/install_coverage_subprocess_pth.py
    - name: Generate old pickles (backward compat)
      shell: bash
      run: |
        git_head=$(git rev-parse HEAD)
        cp tests/generate_old_pickles.py tests/_generate_old_pickles.py
        git checkout v1.4.1
        python tests/_generate_old_pickles.py
        git checkout ${git_head}
    - name: Test with pytest
      run: |
        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
          PYTHONPATH='.:tests' python -m pytest -r s
        coverage combine --append
        coverage xml -i
    - name: Publish coverage results
      uses: codecov/codecov-action@v1
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  distributed-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: distributed
      TEST_REQUIREMENTS: cryptography pytest pytest-asyncio<0.14.0 pytest-timeout pytest-rerunfailures numpy pandas mock bokeh fsspec>=0.3.3
      PROJECT_URL: https://github.com/dask/distributed.git
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Install developed version of cloudpickle
      shell: bash
      run: |
        python -m pip install -e .
    - name: Test the downstream project
      shell: bash
      run: |
        # FIXME ipv6-related failures on Ubuntu github actions CI
        # https://github.com/dask/distributed/issues/4514
        export DISABLE_IPV6=1
        export PYTEST_ADDOPTS=("-m" "not avoid_ci")
        source ./.github/scripts/test_downstream_project.sh

  joblib-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci joblib') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: joblib
      TEST_REQUIREMENTS: "threadpoolctl pytest numpy distributed"
      PROJECT_URL: https://github.com/joblib/joblib.git
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Vendor current cloudpickle inside joblib
      run: |
        pushd ../joblib/joblib/externals
        source vendor_cloudpickle.sh ../../../cloudpickle
        popd
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  loky-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci loky') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: loky
      TEST_REQUIREMENTS: "pytest psutil"
      PROJECT_URL: https://github.com/joblib/loky.git
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install downstream project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Install developed version of cloudpickle
      shell: bash
      run: |
        python -m pip install -e .
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  ray-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci ray') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: ray
    strategy:
      matrix:
        python_version: [3.8]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.8
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        python -m pip install --upgrade -r dev-requirements.txt
        python -m pip install setproctitle psutil
        # from https://docs.ray.io/en/master/development.html#building-ray
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
        pushd ..
        git clone https://github.com/ray-project/ray.git
        pushd ray
        python python/ray/setup-dev.py --yes
        popd
        popd
        PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")
        rm $PROJECT_DIR/cloudpickle/cloudpickle.py
        cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py
        cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py
        cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py
    - name: Test the downstream project
      run: |
        PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")"
        COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_simple_serialization
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_complex_serialization
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_ray_recursive_objects
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_serialization_final_fallback
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_nested_functions