Upload code coverage reports from different jobs, other CI improvements

.github/workflows/ci.yml

@@ -31,45 +31,41 @@ jobs:
         # file with notes on your contribution.
         git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!"
 
-  gpu_checks:
-    name: GPU Checks
-    if: github.repository == 'allenai/allennlp'  # self-hosted runner only available on main repo
-    timeout-minutes: 15
-    runs-on: [self-hosted, GPU]
+  style:
+    name: Style
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set Docker tag
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+
+    - name: Install requirements
       run: |
-        if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
-            echo "DOCKER_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV;
-        else
-            echo "DOCKER_TAG=$GITHUB_SHA" >> $GITHUB_ENV;
-        fi
+        grep -E '^black' dev-requirements.txt | xargs pip install
 
-    - name: Build test image
+    - name: Debug info
       run: |
-        make docker-test-image DOCKER_TAG=$DOCKER_TAG
+        pip freeze
 
-    - name: Run GPU tests
+    - name: Run black
       run: |
-        make docker-test-run DOCKER_TAG=$DOCKER_TAG ARGS='gpu-test'
+        black --check .
 
-  check_core:
-    name: Check Core
+  lint:
+    name: Lint
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ['3.7', '3.8']
 
     steps:
     - uses: actions/checkout@v2
 
     - name: Setup Python
       uses: actions/setup-python@v1
       with:
-        python-version: ${{ matrix.python }}
+        python-version: 3.8
 
     - uses: actions/cache@v2
       with:
@@ -84,36 +80,64 @@ jobs:
       run: |
         pip freeze
 
-    - name: Format
-      if: '! cancelled()'
+    - name: Run flake8
       run: |
-        make format
+        flake8 .
 
-    - name: Lint
-      if: '! cancelled()'
+    - name: Run mypy
+      if: '! cancelled()'  # run this step even if the last step failed.
       run: |
-        make lint
+        mypy . --cache-dir=/dev/null
 
-    - name: Type check
-      if: '! cancelled()'
+  gpu_tests:
+    name: GPU Tests
+    if: github.repository == 'allenai/allennlp'  # self-hosted runner only available on main repo
+    timeout-minutes: 15
+    runs-on: [self-hosted, GPU]
+    env:
+      # Required to use the setup-python action.
+      AGENT_TOOLSDIRECTORY: '/opt/hostedtoolcache'
+      # Our self-hosted runner currently is currently compatible with CUDA 11.*.
+      TORCH_VERSION: 'torch==1.8.1+cu111 torchvision==0.9.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      env:
+        # Log useful debugging information.
+        ACTIONS_STEP_DEBUG: 'true'
+      with:
+        python-version: 3.8
+
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ env.TORCH_VERSION }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
+
+    - name: Install requirements
       run: |
-        make typecheck
+        make install
+
+    - name: Debug info
+      run: |
+        which python
+        python --version
+        pip freeze
 
     - name: Run tests
-      if: '! cancelled()'
       run: |
-        make test-with-cov
+        make gpu-test
+        mkdir coverage
+        mv coverage.xml coverage/
 
-    - name: Upload coverage to Codecov
-      if: matrix.python == '3.7' && github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
-      uses: codecov/codecov-action@v1
+    - name: Save coverage report
+      if: github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
+      uses: actions/upload-artifact@v1
       with:
-        file: ./coverage.xml
-        # Ignore codecov failures as the codecov server is not
-        # very reliable but we don't want to report a failure
-        # in the github UI just because the coverage report failed to
-        # be published.
-        fail_ci_if_error: false
+        name: gpu-tests-coverage
+        path: ./coverage
 
     - name: Clean up
       if: always()
@@ -122,8 +146,8 @@ jobs:
         # See https://github.com/pypa/pip/issues/4537.
         pip uninstall --yes allennlp
 
-  check_models:
-    name: Check Models
+  cpu_tests:
+    name: CPU Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -150,6 +174,52 @@ jobs:
       run: |
         pip freeze
 
+    - name: Run tests
+      if: '! cancelled()'
+      run: |
+        make test
+        mkdir coverage
+        mv coverage.xml coverage/
+
+    - name: Save coverage report
+      if: matrix.python == '3.7' && github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
+      uses: actions/upload-artifact@v1
+      with:
+        name: cpu-tests-coverage
+        path: ./coverage
+
+    - name: Clean up
+      if: always()
+      run: |
+        # Could run into issues with the cache if we don't uninstall the editable.
+        # See https://github.com/pypa/pip/issues/4537.
+        pip uninstall --yes allennlp
+
+  model_tests:
+    name: Model Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+
+    - uses: actions/cache@v2
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
+
+    - name: Install requirements
+      run: |
+        make install
+
+    - name: Debug info
+      run: |
+        pip freeze
+
     - name: Pull and install models repo
       env:
         ALLENNLP_VERSION_OVERRIDE: ""  # Don't replace the core library.
@@ -158,15 +228,63 @@ jobs:
         cd allennlp-models
         pip install --upgrade --upgrade-strategy eager -e . -r dev-requirements.txt
 
-    - name: Run models tests
+    - name: Run tests
       run: |
-        cd allennlp-models && make test
+        cd allennlp-models
+        make test-with-cov COV=allennlp
+        mkdir coverage
+        mv coverage.xml coverage/
+
+    - name: Save coverage report
+      if: github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
+      uses: actions/upload-artifact@v1
+      with:
+        name: model-tests-coverage
+        path: allennlp-models/coverage
 
     - name: Clean up
       if: always()
       run: |
         pip uninstall --yes allennlp allennlp-models
 
+  upload_coverage:
+    name: Upload Coverage Report
+    if: github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
+    runs-on: ubuntu-latest
+    needs: [cpu_tests, gpu_tests, model_tests]
+
+    steps:
+      # Need to checkout code to get the coverage config.
+    - uses: actions/checkout@v2
+
+    - name: Download coverage report from CPU tests
+      uses: actions/download-artifact@v1
+      with:
+        name: cpu-tests-coverage
+        path: coverage/cpu_tests
+
+    - name: Download coverage report from GPU Tests
+      uses: actions/download-artifact@v1
+      with:
+        name: gpu-tests-coverage
+        path: coverage/gpu_tests
+
+    - name: Download coverage report from model tests
+      uses: actions/download-artifact@v1
+      with:
+        name: model-tests-coverage
+        path: coverage/model_tests
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        files: coverage/cpu_tests/coverage.xml,coverage/gpu_tests/coverage.xml,coverage/model_tests/coverage.xml
+        # Ignore codecov failures as the codecov server is not
+        # very reliable but we don't want to report a failure
+        # in the github UI just because the coverage report failed to
+        # be published.
+        fail_ci_if_error: false
+
   # Builds package distribution files for PyPI.
   build_package:
     name: Build Package
@@ -481,7 +599,7 @@ jobs:
   # Publish the core distribution files to PyPI.
   publish:
     name: PyPI
-    needs: [check_core, check_models, gpu_checks, build_package, test_package, docker, docs]
+    needs: [style, lint, cpu_tests, gpu_tests, model_tests, build_package, test_package, docker, docs]
     # Only publish to PyPI on releases and nightly builds to "allenai/allennlp" (not forks).
     if: github.repository == 'allenai/allennlp' && (github.event_name == 'release' || github.event_name == 'schedule')
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -124,7 +124,7 @@ When you're ready to contribute code to address an open issue, please follow the
 
     Our CI also uses [`flake8`](https://github.com/allenai/allennlp/tree/main/tests) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with
 
-        make lint
+        flake8 .
 
     and