Generate coverage reports using only GitHub Actions, without codecov.…

…io (#447)

* Initial coverage report.

* Enable for windows.

* Try to support windows paths for coverage.

* Try the branch.

* Update all-success to newer version that as no set-output warnings.

* Add a comment.

* Pass comment via a file to handle newlines.

* Use nodejs like a pro.

* Apply suggestions from code review

Co-authored-by: Kyle Altendorf <sda@fstab.net>

* Include diff. Wrap in markdown.

* Enable test report.

* Fix after re-review.

* Remove run_id from coverage artifact.

* Add a name for the comment script.

* Remove comment about single artifact as this is what we have by default for a workflow.

* Add restricted permissions for token.

* Use markdown report for diff-cover.

Co-authored-by: Kyle Altendorf <sda@fstab.net>

.github/scripts/pr_comment.js

@@ -0,0 +1,80 @@
+/*
+Have a single comment on a PR, identified by a comment marker.
+
+Create a new comment if no comment already exists.
+Update the content of the existing comment.
+
+
+https://octokit.github.io/rest.js/v19
+*/
+module.exports = async ({github, context, process}) => {
+    var octokit_rest = github
+    if (context.eventName != "pull_request") {
+        // Only PR are supported.
+        return
+    }
+
+    var sleep = function(second) {
+        return new Promise(resolve => setTimeout(resolve, second * 1000))
+    }
+
+    /*
+    Perform the actual logic.
+
+    This is wrapped so that we can retry on errors.
+    */
+    var doAction = async function() {
+
+        console.log(context)
+
+        fs = require('fs');
+
+        const body = fs.readFileSync(
+            process.env.GITHUB_WORKSPACE + "/" + process.env.COMMENT_BODY, 'utf8');
+        var comment_id = null
+        var comment_marker = '\n' + process.env.COMMENT_MARKER
+        var comment_body = body + comment_marker
+
+        var comments = await octokit_rest.issues.listComments({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.payload.number,
+          })
+
+        console.log(comments)
+
+        comments.data.forEach(comment => {
+            if (comment.body.endsWith(comment_marker)) {
+                comment_id = comment.id
+            }
+        })
+
+        if (comment_id) {
+            // We have an existing comment.
+            // update the content.
+            await octokit_rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: comment_id,
+                body: comment_body,
+            })
+            return
+        }
+
+        // Create a new comment.
+        await octokit_rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.payload.number,
+            body: comment_body,
+        })
+
+    }
+
+    try {
+        await doAction()
+    } catch (e) {
+        await sleep(5)
+        await doAction()
+    }
+}

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
     tags: [ "**" ]
   pull_request:
 
+permissions:
+  contents: read
+
 defaults:
   run:
     shell: bash
@@ -96,8 +99,18 @@ jobs:
 
     - run: nox --python ${{ matrix.python.action }} -e ${{ matrix.task.nox }} -- --use-wheel dist/*.whl
 
+    - name: Store coverage file
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage
+        # It is important to keep the unique names for the coverage files
+        # so that when uploaded to the same artifact, they don't overlap.
+        path: .coverage*
+
     - name: Codecov
       run: |
+        ls -al .coverage*
+        coverage combine
         codecov -n "GitHub Actions - ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }}"
 
 
@@ -137,8 +150,18 @@ jobs:
 
     - run: nox --python ${{ matrix.python.action }} -e ${{ matrix.task.nox }} -- --use-wheel dist/*.whl
 
+    - name: Store coverage file
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage
+        # It is important to keep the unique names for the coverage files
+        # so that when uploaded to the same artifact, they don't overlap.
+        path: .coverage*
+
     - name: Codecov
       run: |
+        ls -al .coverage*
+        coverage combine
         codecov -n "GitHub Actions - ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }}"
 
   check:
@@ -232,6 +255,65 @@ jobs:
         password: ${{ secrets.PYPI_TOKEN }}
         verbose: true
 
+  coverage-report:
+    name: Coverage report
+    runs-on: ubuntu-latest
+    permissions:
+      # Even we send a comment to a PR, we use the issues API.
+      # Issues and PR share the same comment API.
+      issues: write
+    needs:
+      # We are waiting only for test jobs.
+      - test-linux
+      - test-windows
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade coverage[toml] diff_cover
+
+    - name: Download coverage reports
+      uses: actions/download-artifact@v3
+      with:
+        name: coverage
+        path: .
+
+    - name: Combine coverage
+      run: coverage combine
+
+    - name: Report coverage
+      run: |
+          coverage xml
+          coverage report --no-skip-covered --show-missing
+          # Wrap output in markdown verbatim text.
+          echo '```' > coverage-report.txt
+          coverage report --show-missing >> coverage-report.txt
+          echo '```' >> coverage-report.txt
+          diff-cover --markdown-report diff-cover.md  --compare-branch origin/trunk coverage.xml
+          cat diff-cover.md >> coverage-report.txt
+
+    # Use the generic JS script to call our custom script
+    # for sending a comment to a PR.
+    - name: Send coverage comment to PR
+      uses: actions/github-script@v3
+      env:
+        COMMENT_MARKER: "<!--- automatic-coverage-report -->"
+        COMMENT_BODY: coverage-report.txt
+      with:
+        script: |
+          const script = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/pr_comment.js`)
+          // Only pass top level objects as GHA does dependecy injection.
+          await script({github, context, process})
+
+    - name: Enforce diff coverage
+      run: |
+          diff-cover --fail-under=100 --compare-branch origin/trunk coverage.xml
+
+
   # This is a meta-job to simplify PR CI enforcement configuration in GitHub.
   # Inside the GitHub config UI you only configure this job as required.
   # All the extra requirements are defined "as code" as part of the `needs`
@@ -250,8 +332,9 @@ jobs:
       - test-windows
       - check
       - pypi-publish
+      - coverage-report
     steps:
       - name: Require all successes
-        uses: re-actors/alls-green@3a2de129f0713010a71314c74e33c0e3ef90e696
+        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
         with:
           jobs: ${{ toJSON(needs) }}

.gitignore

@@ -11,6 +11,7 @@ dist/
 venv/
 htmlcov/
 .coverage
+coverage.xml
 *~
 *.lock
 apidocs/

noxfile.py

@@ -36,9 +36,8 @@ def tests(session: nox.Session) -> None:
     session.run("coverage", "run", "--module", "twisted.trial", *posargs)
 
     if os.environ.get("CI") != "true":
+        # When running the test locally, show the coverage report.
         session.notify("coverage_report")
-    else:
-        session.run("coverage", "combine")
 
 
 @nox.session

pyproject.toml

@@ -91,7 +91,12 @@ branch = true
 source = ["towncrier"]
 
 [tool.coverage.paths]
-source = ["src", ".nox/*/site-packages"]
+source = [
+    "src",
+    "*.nox/*/site-packages",
+    # required until coverage 6.6.0 is used: https://github.com/nedbat/coveragepy/issues/991
+    "*.nox\\*\\*\\site-packages"
+]
 
 [tool.coverage.report]
 show_missing = true
@@ -102,5 +107,4 @@ exclude_lines = [
 ]
 omit = [
     "src/towncrier/__main__.py",
-    "src/towncrier/test/*",
 ]