Add code coverage (#4670)

* Add automated code coverage

* Add pytest-cov

* Temporarily reduce number of tests

* Run all tests again

* Update .gitignore

* Only run one test

* Specify source instead of input

* Run all tests again

* Trigger CI

* Exclude code coverage reports in test_upload_directory

* Decrease coverage target

* Update .gitignore

* Lint

.coveragerc

@@ -1,6 +1,6 @@
 [run]
-include = 
-    distributed/*
+source =
+    distributed
 omit =
     distributed/tests/test*
     distributed/hdfs.py

.github/workflows/tests.yaml

@@ -116,12 +116,15 @@ jobs:
           source continuous_integration/scripts/set_ulimit.sh
 
           pytest distributed -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
-            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
+            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID --cov=distributed --cov-report=xml
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
 
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
       - name: Upload test artifacts
         # ensure this runs even if pytest fails
         if: >

.gitignore

@@ -15,6 +15,8 @@ continuous_integration/hdfs-initialized
 .DS_Store
 .cache
 .coverage
+.coverage.*
+coverage.xml
 *.lock
 .#*
 .eggs/

codecov.yml

@@ -0,0 +1,17 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "85...100"
+
+  status:
+    project:
+      default:
+        target: 85%
+        threshold: 1%
+    patch: no
+    changes: no
+
+comment: off

continuous_integration/environment-3.7.yaml

@@ -9,6 +9,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec
@@ -27,6 +28,7 @@ dependencies:
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

continuous_integration/environment-3.8.yaml

@@ -9,6 +9,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec
   - h5py
@@ -25,6 +26,7 @@ dependencies:
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

continuous_integration/environment-3.9.yaml

@@ -10,6 +10,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec  # overridden by git tip below
   - h5py
@@ -28,6 +29,7 @@ dependencies:
   - pynvml  # Only tested here
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures

distributed/tests/test_client.py

@@ -6861,7 +6861,8 @@ async def test_computation_object_code_client_compute(c, s, a, b):
 async def test_upload_directory(c, s, a, b, tmp_path):
     from dask.distributed import UploadDirectory
 
-    files = set(os.listdir())
+    # Be sure to exclude code coverage reports
+    files_start = {f for f in os.listdir() if not f.startswith(".coverage")}
 
     with open(tmp_path / "foo.py", "w") as f:
         f.write("x = 123")
@@ -6887,7 +6888,8 @@ def f():
         results = await c.run(f)
         assert results[n.worker_address] == 123
 
-    assert files == set(os.listdir())  # no change
+    files_end = {f for f in os.listdir() if not f.startswith(".coverage")}
+    assert files_start == files_end  # no change
 
 
 @gen_cluster(client=True)