testing.yml

name: Automated Tests

on:
  push:
    branches: master
  pull_request:
    branches: master

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: 3.7
    - name: Install flake8
      shell: bash
      run: |
        python -V
        python -m pip install flake8
    - name: Run flake8 on diff with upstream/master
      shell: bash
      run: |
        bash ./.github/scripts/flake8_diff.sh

  build:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python_version: [3.5, 3.6, 3.7, 3.8, 3.9, "pypy3"]
        exclude:
          # Do not test all minor versions on all platforms, especially if they
          # are not the oldest/newest supported versions
          - os: windows-latest
            python_version: 3.6
          - os: windows-latest
            python_version: 3.7
          - os: windows-latest
            python_version: 3.8
            # as of  4/02/2020, psutil won't build under PyPy + Windows
          - os: windows-latest
            python_version: "pypy3"
          - os: macos-latest
            python_version: 3.6
          - os: macos-latest
            python_version: 3.7
          - os: macos-latest
            python_version: 3.8
          - os: macos-latest
            # numpy triggers: RuntimeError: Polyfit sanity test emitted a
            # warning
            python_version: "pypy3"

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v1
    - name: Set up Python ${{ matrix.python_version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      shell: bash
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e .
        python -m pip install -r dev-requirements.txt
        python ci/install_coverage_subprocess_pth.py
        export
    - name: Install supported dependencies (only test in Python 3.6)
      shell: bash
      run: python -m pip install typing-extensions
      if: matrix.python_version == '3.6'
    - name: Display Python version
      shell: bash
      run: python -c "import sys; print(sys.version)"
    - name: Look for syntax errors/undefined names
      shell: bash
      run: |
        python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \
          --show-source --statistics
    - name: Test with pytest
      shell: bash
      run: |
        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
          PYTHONPATH='.:tests' python -m pytest -r s
        coverage combine --append
        coverage xml -i
    - name: Publish coverage results
      uses: codecov/codecov-action@v1
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  python-nightly:
    runs-on: ubuntu-18.04
    steps:
    - uses: actions/checkout@v1
    - name: Install Python from ppa:deadsnakes/nightly
      run: |
        sudo add-apt-repository ppa:deadsnakes/nightly
        sudo apt update
        sudo apt install python3.10 python3.10-venv python3.10-dev
        python3.10 -m venv nightly-venv
        echo "$PWD/nightly-venv/bin" >> $GITHUB_PATH
    - name: Display Python version
      run: python -c "import sys; print(sys.version)"
    - name: Install project and dependencies
      run: |
        set -e
        python -m pip install --upgrade pip
        python -m pip install -r dev-requirements.txt
        python -m pip install -e .
        python ci/install_coverage_subprocess_pth.py
    - name: Generate old pickles (backward compat)
      shell: bash
      run: |
        git_head=$(git rev-parse HEAD)
        cp tests/generate_old_pickles.py tests/_generate_old_pickles.py
        git checkout v1.4.1
        python tests/_generate_old_pickles.py
        git checkout ${git_head}
    - name: Test with pytest
      run: |
        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
          PYTHONPATH='.:tests' python -m pytest -r s
        coverage combine --append
        coverage xml -i
    - name: Publish coverage results
      uses: codecov/codecov-action@v1
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml

  distributed-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: distributed
      TEST_REQUIREMENTS: pytest pytest-timeout numpy pandas mock bokeh fsspec>=0.3.3
      PROJECT_URL: https://github.com/dask/distributed.git
      PYTEST_ADDOPTS: "--timeout-method=thread --timeout=300 -m \"not avoid_travis\" -k \"not test_dask_scheduler and not test_workspace_concurrency and not test_defaults and not test_service_hosts and not test_logging_file_config and not test_hostport and not test_workdir_simple and not test_two_workspaces_in_same_directory and not test_recompute_released_results and not test_connection_args and not test_listen_args  and not test_remote_access\""
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  joblib-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci joblib') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: joblib
      TEST_REQUIREMENTS: "threadpoolctl pytest numpy distributed"
      PROJECT_URL: https://github.com/joblib/joblib.git
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Vendor current cloudpickle inside joblib
      run: |
        pushd ../joblib/joblib/externals
        source vendor_cloudpickle.sh ../../../cloudpickle
        popd
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  loky-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci loky') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: loky
      TEST_REQUIREMENTS: "pytest psutil"
      PROJECT_URL: https://github.com/tomMoral/loky.git
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        bash ./.github/scripts/install_downstream_project.sh
    - name: Test the downstream project
      run: |
        bash ./.github/scripts/test_downstream_project.sh

  ray-downstream-build:
    runs-on: ubuntu-latest
    if: "contains(github.event.pull_request.labels.*.name, 'ci ray') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"
    env:
      PROJECT: ray
    strategy:
      matrix:
        python_version: [3.7]
    steps:
    - uses: actions/checkout@v1
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python_version }}
    - name: Install project and dependencies
      run: |
        python -m pip install --upgrade -r dev-requirements.txt
        python -m pip install setproctitle psutil ray==0.8.6
        PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")
        rm $PROJECT_DIR/cloudpickle/cloudpickle.py
        git clone https://github.com/ray-project/ray.git ../ray
        cp -R ../ray/python/ray/tests $PROJECT_DIR/tests
        cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py
        cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py
        cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py
    - name: Test the downstream project
      run: |
        PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")"
        COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_simple_serialization
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_complex_serialization
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_ray_recursive_objects
        pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_serialization_final_fallback
        pytest -vl $PROJECT_DIR/tests/test_basic.py::test_nested_functions