diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
index ccea59d5540..97c43e60eac 100644
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -63,69 +63,21 @@ jobs:
 
       - name: Run tests
         id: run_tests
+        env:
+          XTRATESTARGS: "--report-log output-log.jsonl"
         run: source continuous_integration/scripts/run_tests.sh
 
+      - name: Open or update issue on failure
+        if: |
+          failure()
+          && github.event_name != 'pull_request'
+          && github.repository == 'dask/dask'
+          && steps.run_tests.outcome == 'failure'
+        uses: xarray-contrib/issue-from-pytest-log@v1.1
+        with:
+          log-path: output-log.jsonl
+          issue-title: ⚠️ Upstream CI failed ⚠️
+          issue-label: upstream
+
       - name: Coverage
         uses: codecov/codecov-action@v3
-
-  report:
-    name: report
-    needs: build
-    if: |
-      always()
-      && github.event_name != 'pull_request'
-      && github.repository == 'dask/dask'
-      && needs.build.result == 'failure'
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/checkout@v3.1.0
-      - name: Report failures
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const title = "⚠️ Upstream CI failed ⚠️"
-            const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
-            const issue_body = `[Workflow Run URL](${workflow_url})`
-            // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures
-            const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){
-              repository(owner: $owner, name: $name) {
-                issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) {
-                  edges {
-                    node {
-                      body
-                      id
-                      number
-                    }
-                  }
-                }
-              }
-            }`;
-            const variables = {
-                owner: context.repo.owner,
-                name: context.repo.repo,
-                label: 'upstream',
-                creator: "github-actions[bot]"
-            }
-            const result = await github.graphql(query, variables)
-            // If no issue is open, create a new issue,
-            // else update the body of the existing issue.
-            if (result.repository.issues.edges.length === 0) {
-                github.rest.issues.create({
-                    owner: variables.owner,
-                    repo: variables.name,
-                    body: issue_body,
-                    title: title,
-                    labels: [variables.label]
-                })
-            } else {
-                github.rest.issues.update({
-                    owner: variables.owner,
-                    repo: variables.name,
-                    issue_number: result.repository.issues.edges[0].node.number,
-                    body: issue_body
-                })
-            }
diff --git a/continuous_integration/scripts/install.sh b/continuous_integration/scripts/install.sh
index 60ac5f3d404..7a128e5c1a5 100644
--- a/continuous_integration/scripts/install.sh
+++ b/continuous_integration/scripts/install.sh
@@ -35,6 +35,9 @@ if [[ ${UPSTREAM_DEV} ]]; then
         git+https://github.com/dask/distributed \
         git+https://github.com/dask/fastparquet \
         git+https://github.com/zarr-developers/zarr-python
+
+    # Used when automatically opening an issue when the `upstream` CI build fails
+    mamba install pytest-reportlog
 fi
 
 # Install dask