CI/CD #633
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: CI/CD | |
| on: | |
| merge_group: | |
| push: | |
| branches: | |
| - master | |
| - >- | |
| [0-9].[0-9]+ | |
| tags: | |
| - v* | |
| pull_request: | |
| branches: | |
| - master | |
| - >- | |
| [0-9].[0-9]+ | |
| schedule: | |
| - cron: 0 6 * * * # Daily 6AM UTC build | |
| env: | |
| COLOR: >- # Supposedly, pytest or coveragepy use this | |
| yes | |
| FORCE_COLOR: 1 # Request colored output from CLI tools supporting it | |
| MYPY_FORCE_COLOR: 1 # MyPy's color enforcement | |
| PIP_DISABLE_PIP_VERSION_CHECK: 1 | |
| PIP_NO_PYTHON_VERSION_WARNING: 1 | |
| PIP_NO_WARN_SCRIPT_LOCATION: 1 | |
| PRE_COMMIT_COLOR: always | |
| PROJECT_NAME: multidict | |
| PY_COLORS: 1 # Recognized by the `py` package, dependency of `pytest` | |
| PYTHONIOENCODING: utf-8 | |
| PYTHONUTF8: 1 | |
| PYTHON_LATEST: 3.x | |
| jobs: | |
| pre-setup: | |
| name: ⚙️ Pre-set global build settings | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| defaults: | |
| run: | |
| shell: python | |
| outputs: | |
| # NOTE: These aren't env vars because the `${{ env }}` context is | |
| # NOTE: inaccessible when passing inputs to reusable workflows. | |
| dists-artifact-name: python-package-distributions | |
| sdist-name: ${{ env.PROJECT_NAME }}-*.tar.gz | |
| wheel-name: ${{ env.PROJECT_NAME }}-*.whl | |
| steps: | |
| - run: >- | |
| print('No-op') | |
| build-pure-python-dists: | |
| name: 📦 Build distribution packages | |
| needs: | |
| - pre-setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| outputs: | |
| sdist-filename: >- | |
| ${{ steps.dist-filenames-detection.outputs.sdist-filename }} | |
| wheel-filename: >- | |
| ${{ steps.dist-filenames-detection.outputs.wheel-filename }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ env.PYTHON_LATEST }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_LATEST }} | |
| - name: Install core libraries for build | |
| run: python -Im pip install build | |
| - name: Build sdists and pure-python wheel | |
| env: | |
| MULTIDICT_NO_EXTENSIONS: Y | |
| run: python -Im build | |
| - name: Determine actual created filenames | |
| id: dist-filenames-detection | |
| run: >- | |
| { | |
| echo -n sdist-filename= | |
| ; | |
| basename "$(ls -1 dist/${{ needs.pre-setup.outputs.sdist-name }})" | |
| ; | |
| echo -n wheel-filename= | |
| ; | |
| basename "$(ls -1 dist/${{ needs.pre-setup.outputs.wheel-name }})" | |
| ; | |
| } | |
| >> "${GITHUB_OUTPUT}" | |
| - name: Upload built artifacts for testing | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| # NOTE: Exact expected file names are specified here | |
| # NOTE: as a safety measure — if anything weird ends | |
| # NOTE: up being in this dir or not all dists will be | |
| # NOTE: produced, this will fail the workflow. | |
| path: | | |
| dist/${{ steps.dist-filenames-detection.outputs.sdist-filename }} | |
| dist/${{ steps.dist-filenames-detection.outputs.wheel-filename }} | |
| retention-days: 15 | |
| lint: | |
| uses: ./.github/workflows/reusable-linters.yml | |
| secrets: | |
| codecov-token: ${{ secrets.CODECOV_TOKEN }} | |
| build-wheels-for-tested-arches: | |
| name: Build wheels on ${{ matrix.os }} | |
| needs: | |
| - build-pure-python-dists | |
| - pre-setup # transitive, for accessing settings | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu | |
| - windows | |
| - macos | |
| uses: ./.github/workflows/reusable-build-wheel.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| wheel-tags-to-skip: >- | |
| ${{ | |
| (github.event_name != 'push' || !contains(github.ref, 'refs/tags/')) | |
| && '*_i686 | |
| *-macosx_universal2 | |
| *-musllinux_* | |
| *-win32 | |
| *_arm64 | |
| pp*' | |
| || '' | |
| }} | |
| source-tarball-name: >- | |
| ${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| dists-artifact-name: ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| test: | |
| name: Test | |
| needs: | |
| - build-pure-python-dists # transitive, for accessing settings | |
| - build-wheels-for-tested-arches | |
| - pre-setup # transitive, for accessing settings | |
| strategy: | |
| matrix: | |
| pyver: | |
| - 3.13-dev | |
| - 3.12 | |
| - 3.7 | |
| - 3.11 | |
| - >- | |
| 3.10 | |
| - 3.9 | |
| - 3.8 | |
| no-extensions: ['', 'Y'] | |
| os: [ubuntu, macos, windows] | |
| exclude: | |
| - os: macos | |
| no-extensions: Y | |
| - os: windows | |
| no-extensions: Y | |
| include: | |
| - pyver: pypy-3.8 | |
| no-extensions: Y | |
| os: ubuntu | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }}-latest | |
| timeout-minutes: 15 | |
| continue-on-error: >- | |
| ${{ | |
| endsWith(matrix.pyver, '-dev') | |
| && true | |
| || false | |
| }} | |
| steps: | |
| - name: Retrieve the project source from an sdist inside the GHA artifact | |
| uses: re-actors/checkout-python-sdist@release/v1 | |
| with: | |
| source-tarball-name: >- | |
| ${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| workflow-artifact-name: >- | |
| ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| - name: Download distributions | |
| if: ${{ !endsWith(matrix.pyver, '-dev') && true || false }} | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| path: dist | |
| - name: Setup Python ${{ matrix.pyver }} | |
| id: python-install | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.pyver }} | |
| allow-prereleases: >- | |
| ${{ | |
| endsWith(matrix.pyver, '-dev') | |
| && true | |
| || false | |
| }} | |
| - name: Get pip cache dir | |
| id: pip-cache | |
| run: | | |
| echo "dir=$(pip cache dir)" >> "${GITHUB_OUTPUT}" | |
| shell: bash | |
| - name: Cache PyPI | |
| uses: actions/cache@v4 | |
| with: | |
| key: >- | |
| pip-ci-${{ | |
| runner.os | |
| }}-${{ | |
| matrix.pyver | |
| }}-${{ | |
| matrix.no-extensions | |
| }}-${{ | |
| hashFiles('requirements/*.txt') | |
| }} | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| restore-keys: >- | |
| pip-ci-${{ | |
| runner.os | |
| }}-${{ | |
| matrix.pyver | |
| }}-${{ | |
| matrix.no-extensions | |
| }}- | |
| - name: Install dependencies | |
| uses: py-actions/py-dependency-install@v4 | |
| with: | |
| path: requirements/pytest.txt | |
| - name: Determine pre-compiled compatible wheel | |
| if: ${{ !endsWith(matrix.pyver, '-dev') && true || false }} | |
| env: | |
| # NOTE: When `pip` is forced to colorize output piped into `jq`, | |
| # NOTE: the latter can't parse it. So we're overriding the color | |
| # NOTE: preference here via https://no-color.org. | |
| # NOTE: Setting `FORCE_COLOR` to any value (including 0, an empty | |
| # NOTE: string, or a "YAML null" `~`) doesn't have any effect and | |
| # NOTE: `pip` (through its verndored copy of `rich`) treats the | |
| # NOTE: presence of the variable as "force-color" regardless. | |
| # | |
| # NOTE: This doesn't actually work either, so we'll resort to unsetting | |
| # NOTE: in the Bash script. | |
| # NOTE: Ref: https://github.com/Textualize/rich/issues/2622 | |
| NO_COLOR: 1 | |
| id: wheel-file | |
| run: > | |
| echo -n path= | tee -a "${GITHUB_OUTPUT}" | |
| unset FORCE_COLOR | |
| python | |
| -X utf8 | |
| -u -I | |
| -m pip install | |
| --find-links=./dist | |
| --no-index | |
| '${{ env.PROJECT_NAME }}' | |
| --force-reinstall | |
| --no-color | |
| --no-deps | |
| --only-binary=:all: | |
| --dry-run | |
| --report=- | |
| --quiet | |
| | jq --raw-output .install[].download_info.url | |
| | tee -a "${GITHUB_OUTPUT}" | |
| shell: bash | |
| - name: >- | |
| Self-install (from ${{ | |
| endsWith(matrix.pyver, '-dev') | |
| && 'source' | |
| || 'wheel' | |
| }}) | |
| env: | |
| MULTIDICT_NO_EXTENSIONS: ${{ matrix.no-extensions }} | |
| run: >- | |
| python -Im | |
| pip install '${{ | |
| endsWith(matrix.pyver, '-dev') | |
| && '.' | |
| || steps.wheel-file.outputs.path | |
| }}' | |
| - name: Run unittests | |
| run: >- | |
| python -Im pytest tests -v | |
| --cov-report xml | |
| --junitxml=.test-results/pytest/test.xml | |
| --${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions | |
| - name: Produce markdown test summary from JUnit | |
| if: >- | |
| !cancelled() | |
| uses: test-summary/action@v2.3 | |
| with: | |
| paths: .test-results/pytest/test.xml | |
| - name: Append coverage results to Job Summary | |
| if: >- | |
| !cancelled() | |
| run: >- | |
| python -Im coverage report --format=markdown | |
| >> "${GITHUB_STEP_SUMMARY}" | |
| shell: bash | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: >- | |
| !cancelled() | |
| && failure() | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| python -Im | |
| pytest --no-cov -vvvvv --lf -rA | |
| --${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions | |
| && exit 1 | |
| shell: bash | |
| - name: Prepare coverage artifact | |
| # combining Linux and Windows paths is tricky, left this exercise for | |
| # others multidict has no Windows or macOS specific code paths anyway | |
| if: ${{ matrix.os == 'ubuntu' }} | |
| uses: aio-libs/prepare-coverage@v22.1.2 | |
| test-summary: | |
| name: Tests status | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| needs: | |
| - build-pure-python-dists # transitive, for accessing settings | |
| - lint | |
| - pre-setup # transitive, for accessing settings | |
| - test | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| - name: Retrieve the project source from an sdist inside the GHA artifact | |
| uses: re-actors/checkout-python-sdist@release/v1 | |
| with: | |
| source-tarball-name: >- | |
| ${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| workflow-artifact-name: >- | |
| ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage | |
| path: ${{ runner.temp }}/coverage | |
| - name: Install coverage | |
| run: | | |
| pip install coverage | |
| shell: bash | |
| - name: Combine coverage reports | |
| run: | | |
| coverage combine '${{ runner.temp }}/coverage' | |
| shell: bash | |
| - name: Generate XML coverage report | |
| run: | | |
| coverage xml | |
| shell: bash | |
| - name: Send coverage data to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| files: coverage.xml | |
| flags: >- | |
| CI-GHA, | |
| unit | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| pre-deploy: | |
| name: Pre-Deploy | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| needs: test-summary | |
| # Run only on pushing a tag | |
| if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') | |
| steps: | |
| - name: Dummy | |
| run: | | |
| echo "Predeploy step" | |
| build-wheels-for-odd-archs: | |
| name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} | |
| needs: | |
| - build-pure-python-dists | |
| - pre-deploy | |
| - pre-setup # transitive, for accessing settings | |
| strategy: | |
| matrix: | |
| qemu: | |
| - aarch64 | |
| - ppc64le | |
| - s390x | |
| uses: ./.github/workflows/reusable-build-wheel.yml | |
| with: | |
| qemu: ${{ matrix.qemu }} | |
| source-tarball-name: >- | |
| ${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| dists-artifact-name: ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| deploy: | |
| name: Deploy | |
| needs: | |
| - build-pure-python-dists | |
| - build-wheels-for-odd-archs | |
| - build-wheels-for-tested-arches | |
| - pre-setup # transitive, for accessing settings | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: write # IMPORTANT: mandatory for making GitHub Releases | |
| id-token: write # IMPORTANT: mandatory for trusted publishing & sigstore | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/${{ env.PROJECT_NAME }} | |
| steps: | |
| - name: Retrieve the project source from an sdist inside the GHA artifact | |
| uses: re-actors/checkout-python-sdist@release/v1 | |
| with: | |
| source-tarball-name: >- | |
| ${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| workflow-artifact-name: >- | |
| ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| - name: Download distributions | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| path: dist | |
| - run: | | |
| tree | |
| - name: Login | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token | |
| - name: Make Release | |
| uses: aio-libs/create-release@v1.6.6 | |
| with: | |
| changes_file: CHANGES.rst | |
| name: ${{ env.PROJECT_NAME }} | |
| version_file: ${{ env.PROJECT_NAME }}/__init__.py | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact: >- | |
| ${{ needs.pre-setup.outputs.dists-artifact-name }} | |
| fix_issue_regex: >- | |
| :issue:`(\d+)` | |
| fix_issue_repl: >- | |
| #\1 | |
| - name: >- | |
| Publish 🐍📦 to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| - name: Sign the dists with Sigstore | |
| uses: sigstore/gh-action-sigstore-python@v2.1.1 | |
| with: | |
| inputs: >- | |
| ./dist/${{ needs.build-pure-python-dists.outputs.sdist-filename }} | |
| ./dist/*.whl | |
| - name: Upload artifact signatures to GitHub Release | |
| # Confusingly, this action also supports updating releases, not | |
| # just creating them. This is what we want here, since we've manually | |
| # created the release above. | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| # dist/ contains the built packages, which smoketest-artifacts/ | |
| # contains the signatures and certificates. | |
| files: dist/** | |
| ... |