Revamp testing and coverage via tox

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
       tox-source: "./core[dev]"
 
   package:
-    name: Python Package
+    name: Python package
     uses: beeware/.github/.github/workflows/python-package-create.yml@main
     with:
       tox-source: "./core[dev]"
@@ -60,182 +60,200 @@ jobs:
         - "winforms"
 
   core:
+    name: Test core
     runs-on: ${{ matrix.platform }}
-    needs: [pre-commit, towncrier, package]
+    needs: [ pre-commit, towncrier, package ]
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
         platform: [ "macos-12", "macos-14", "ubuntu-latest", "windows-latest" ]
         python-version: [ "3.8", "3.12", "3.13-dev" ]
         include:
-          - experimental: false
-          # Test Python 3.9-3.11 on Ubuntu only
-          - platform: "ubuntu-latest"
-            python-version: "3.9"
-            experimental: false
-          - platform: "ubuntu-latest"
-            python-version: "3.10"
-            experimental: false
-          - platform: "ubuntu-latest"
-            python-version: "3.11"
-            experimental: false
-          # Allow development Python to fail without failing entire job.
-          - python-version: "3.13-dev"
-            experimental: true
+        - experimental: false
+        # Test Python 3.9-3.11 on Ubuntu only
+        - platform: "ubuntu-latest"
+          python-version: "3.9"
+          experimental: false
+        - platform: "ubuntu-latest"
+          python-version: "3.10"
+          experimental: false
+        - platform: "ubuntu-latest"
+          python-version: "3.11"
+          experimental: false
+        # Allow development Python to fail without failing entire job.
+        - python-version: "3.13-dev"
+          experimental: true
         exclude:
-          # macos-14 (i.e. arm64) does not support Python 3.8
-          - platform: "macos-14"
-            python-version: "3.8"
-
-          # Pillow isn't available for Python 3.13 on Windows
-          - platform: "windows-latest"
-            python-version: "3.13-dev"
+        # macos-14 (i.e. arm64) does not support Python 3.8
+        - platform: "macos-14"
+          python-version: "3.8"
+        # Pillow isn't available for Python 3.13 on Windows
+        - platform: "windows-latest"
+          python-version: "3.13-dev"
     steps:
-    - uses: actions/checkout@v4.1.1
+    - name: Checkout
+      uses: actions/checkout@v4.1.1
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dev dependencies
+
+    - name: Install dev Dependencies
       run: |
         # We don't actually want to install toga-core;
         # we just want the dev extras so we have a known version of tox and coverage
         python -m pip install ./core[dev]
-    - name: Get packages
+
+    - name: Get Packages
       uses: actions/download-artifact@v4.1.4
       with:
         pattern: ${{ needs.package.outputs.artifact-name }}-*
         merge-multiple: true
+
     - name: Test
       run: |
         # The $(ls ...) shell expansion is done in the Github environment;
-        # the value of TOGA_INSTALL_COMMAND will be a literal string,
-        # without any shell expansions to perform
-        TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" tox -e py
+        # the value of TOGA_INSTALL_COMMAND will be a literal string without any shell expansions to perform
+        TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" \
+          tox -e py-cov
+        tox -qe coverage$(tr -dc "0-9" <<< "${{ matrix.python-version }}")
         mv core/.coverage core/.coverage.${{ matrix.platform }}.${{ matrix.python-version }}
-    - name: Store coverage data
+
+    - name: Store Coverage Data
       uses: actions/upload-artifact@v4.3.1
       with:
         name: core-coverage-data-${{ matrix.platform }}-${{ matrix.python-version }}
         path: "core/.coverage.*"
         if-no-files-found: error
 
   core-coverage:
-    name: Combine & check core coverage.
-    runs-on: ubuntu-latest
+    name: Coverage
     needs: core
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.1
+    - name: Checkout
+      uses: actions/checkout@v4.1.1
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@v5.0.0
+
+    - name: Set up Python ${{ env.min_python_version }}
+      uses: actions/setup-python@v5.0.0
       with:
-        # Use latest, so it understands all syntax.
-        python-version: ${{ env.max_python_version }}
+        # Use minimum version of python for coverage to avoid phantom branches
+        # https://github.com/nedbat/coveragepy/issues/1572#issuecomment-1522546425
+        python-version: ${{ env.min_python_version }}
+
     - name: Install dev dependencies
       run: |
         # We don't actually want to install toga-core;
         # we just want the dev extras so we have a known version of coverage
         python -m pip install ./core[dev]
-    - name: Retrieve coverage data
+
+    - name: Retrieve Coverage Data
       uses: actions/download-artifact@v4.1.4
       with:
         pattern: core-coverage-data-*
         path: core
         merge-multiple: true
-    - name: Generate coverage report
-      run: |
-        cd core
-        python -m coverage combine
-        python -m coverage html --skip-covered --skip-empty
-        python -m coverage report --rcfile ../pyproject.toml --fail-under=100
-    - name: Upload HTML report if check failed.
+
+    - name: Generate Coverage Report
+      run: tox -e coverage-html-fail
+
+    - name: Upload HTML Coverage Report
       uses: actions/upload-artifact@v4.3.1
       if: failure()
       with:
         name: html-coverage-report
         path: core/htmlcov
 
   testbed:
-    runs-on: ${{ matrix.runs-on }}
+    name: Testbed
     needs: core
+    runs-on: ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
       matrix:
         backend: [ "macOS-x86_64", "macOS-arm64", "windows", "linux", "android", "iOS" ]
         include:
-          - pre-command:
-            briefcase-run-prefix:
-            briefcase-run-args:
-            setup-python: true
-
-          - backend: "macOS-x86_64"
-            platform: "macOS"
-            runs-on: "macos-12"
-            app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed
-
-          - backend: "macOS-arm64"
-            platform: "macOS"
-            runs-on: "macos-14"
-            app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed
-
-          # We use a fixed Ubuntu version rather than `-latest` because at some point,
-          # `-latest` will be updated, but it will be a soft changeover, which would cause
-          # the system Python version to become inconsistent from run to run.
-          - backend: "linux"
-            platform: "linux"
-            runs-on: "ubuntu-22.04"
-            # The package list should be the same as in tutorial-0.rst, and the BeeWare
-            # tutorial, plus blackbox to provide a window manager. We need a window
-            # manager that is reasonably lightweight, honors full screen mode, and
-            # treats the window position as the top-left corner of the *window*, not the
-            # top-left corner of the window *content*. The default GNOME window managers of
-            # most distros meet these requirements, but they're heavyweight; flwm doesn't
-            # work either. Blackbox is the lightest WM we've found that works.
-            pre-command: |
-              sudo apt update -y
-              sudo apt install -y blackbox pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-webkit2-4.0
-
-              # Start Virtual X server
-              echo "Start X server..."
-              Xvfb :99 -screen 0 2048x1536x24 &
-              sleep 1
-
-              # Start Window manager
-              echo "Start window manager..."
-              DISPLAY=:99 blackbox &
-              sleep 1
-
-            briefcase-run-prefix: 'DISPLAY=:99'
-            setup-python: false  # Use the system Python packages.
-            app-user-data-path: $HOME/.local/share/testbed
-
-          - backend: "windows"
-            platform: "windows"
-            runs-on: "windows-latest"
-            app-user-data-path: $HOME\AppData\Local\Tiberius Yak\Toga Testbed\Data
-
-          - backend: "iOS"
-            platform: "iOS"
-            runs-on: "macos-14"
-            briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
-            app-user-data-path: $(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents
-
-          - backend: "android"
-            platform: "android"
-            runs-on: "ubuntu-latest"
-            briefcase-run-args: " -d '{\"avd\":\"beePhone\",\"skin\":\"pixel_3a\"}' --Xemulator=-no-window --Xemulator=-no-snapshot --Xemulator=-no-audio --Xemulator=-no-boot-anim --shutdown-on-exit"
-            pre-command: |
-              # check if virtualization is supported...
-              sudo apt install -qq --no-install-recommends cpu-checker coreutils && echo "CPUs=$(nproc --all)" && kvm-ok
-              # allow access to KVM to run the emulator
-              echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
-                | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-              sudo udevadm control --reload-rules
-              sudo udevadm trigger --name-match=kvm
+        - pre-command: ""
+          briefcase-run-prefix: ""
+          briefcase-run-args: ""
+          setup-python: true
+
+        - backend: "macOS-x86_64"
+          platform: "macOS"
+          runs-on: "macos-12"
+          app-user-data-path: "$HOME/Library/Application Support/org.beeware.toga.testbed"
+
+        - backend: "macOS-arm64"
+          platform: "macOS"
+          runs-on: "macos-14"
+          app-user-data-path: "$HOME/Library/Application Support/org.beeware.toga.testbed"
+
+        # We use a fixed Ubuntu version rather than `-latest` because at some point,
+        # `-latest` will be updated, but it will be a soft changeover, which would cause
+        # the system Python version to become inconsistent from run to run.
+        - backend: "linux"
+          platform: "linux"
+          runs-on: "ubuntu-22.04"
+          # The package list should be the same as in tutorial-0.rst, and the BeeWare
+          # tutorial, plus blackbox to provide a window manager. We need a window
+          # manager that is reasonably lightweight, honors full screen mode, and
+          # treats the window position as the top-left corner of the *window*, not the
+          # top-left corner of the window *content*. The default GNOME window managers of
+          # most distros meet these requirements, but they're heavyweight; flwm doesn't
+          # work either. Blackbox is the lightest WM we've found that works.
+          pre-command: |
+            sudo apt update -y
+            sudo apt install -y --no-install-recommends \
+              blackbox pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-webkit2-4.0
+
+            # Start Virtual X server
+            echo "Start X server..."
+            Xvfb :99 -screen 0 2048x1536x24 &
+            sleep 1
+
+            # Start Window manager
+            echo "Start window manager..."
+            DISPLAY=:99 blackbox &
+            sleep 1
+          briefcase-run-prefix: 'DISPLAY=:99'
+          setup-python: false  # Use the system Python packages
+          app-user-data-path: "$HOME/.local/share/testbed"
+
+        - backend: "windows"
+          platform: "windows"
+          runs-on: "windows-latest"
+          app-user-data-path: '$HOME\AppData\Local\Tiberius Yak\Toga Testbed\Data'
+
+        - backend: "iOS"
+          platform: "iOS"
+          runs-on: "macos-14"
+          briefcase-run-args: "--device 'iPhone SE (3rd generation)'"
+          app-user-data-path: "$(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents"
+
+        - backend: "android"
+          platform: "android"
+          runs-on: "ubuntu-latest"
+          briefcase-run-prefix: JAVA_HOME=${JAVA_HOME_17_X64}
+          briefcase-run-args: >
+            --device '{"avd":"beePhone","skin":"pixel_3a"}'
+            --Xemulator=-no-window
+            --Xemulator=-no-snapshot
+            --Xemulator=-no-audio
+            --Xemulator=-no-boot-anim
+            --shutdown-on-exit
+          pre-command: |
+            # allow access to KVM to run the emulator
+            echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+              | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+            sudo udevadm control --reload-rules
+            sudo udevadm trigger --name-match=kvm
     steps:
-    - uses: actions/checkout@v4.1.1
+    - name: Checkout
+      uses: actions/checkout@v4.1.1
       with:
         fetch-depth: 0
 
@@ -249,7 +267,7 @@ jobs:
         # * It doesn't have an Android build of Pillow yet.
         python-version: "3.10"
 
-    - name: Install dependencies
+    - name: Install Dependencies
       run: |
         ${{ matrix.pre-command }}
         # Use the development version of Briefcase
@@ -259,22 +277,24 @@ jobs:
     - name: Test App
       working-directory: testbed
       timeout-minutes: 15
-      run: ${{ matrix.briefcase-run-prefix }} briefcase run ${{ matrix.platform }} --test ${{ matrix.briefcase-run-args }}
+      run: |
+        ${{ matrix.briefcase-run-prefix }} \
+          briefcase run ${{ matrix.platform }} --test ${{ matrix.briefcase-run-args }}
 
-    - name: Upload logs
+    - name: Upload Logs
       uses: actions/upload-artifact@v4.3.1
       if: failure()
       with:
         name: testbed-failure-logs-${{ matrix.backend }}
         path: testbed/logs/*
 
-    - name: Copy app generated user data
+    - name: Copy App Generated User Data
       if: failure() && matrix.backend != 'android'
       run: |
         mkdir -p testbed/app_data
         cp -r "${{ matrix.app-user-data-path }}" testbed/app_data/testbed-app_data-${{ matrix.backend }}
 
-    - name: Upload app data
+    - name: Upload App Data
       uses: actions/upload-artifact@v4.3.1
       if: failure() && matrix.backend != 'android'
       with:

changes/2440.misc.rst

@@ -0,0 +1 @@
+The tox environments for tests and coverage were revamped for easier and more effective use.

core/pyproject.toml

@@ -63,7 +63,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-# Extras used by developers *of* briefcase are pinned to specific versions to
+# Extras used by developers *of* Toga are pinned to specific versions to
 # ensure environment consistency.
 dev = [
     "coverage[toml] == 7.4.3",
@@ -104,6 +104,9 @@ Documentation = "https://toga.readthedocs.io/en/latest/"
 Tracker = "https://github.com/beeware/toga/issues"
 Source = "https://github.com/beeware/toga"
 
+[project.entry-points."toga.image_formats"]
+pil = "toga.plugins.image_formats.PILConverter"
+
 [tool.setuptools_scm]
 root = ".."
 
@@ -127,6 +130,3 @@ asyncio_mode = "auto"
 filterwarnings = [
     "error",
 ]
-
-[project.entry-points."toga.image_formats"]
-pil = "toga.plugins.image_formats.PILConverter"