Run tests #4250
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run tests | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: ["main", "rc-*"] | |
| pull_request: | |
| merge_group: | |
| release: | |
| types: [published] | |
| schedule: | |
| - cron: "0 8 * * *" | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # "3.10" must be a string; otherwise it is interpreted as 3.1. | |
| python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"] | |
| os: [ubuntu-latest, windows-latest, macOS-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| id: install | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Run unit tests | |
| if: steps.install.outcome == 'success' && (success() || failure()) | |
| run: | | |
| make check-tests | |
| - name: Type check | |
| if: steps.install.outcome == 'success' && (success() || failure()) | |
| run: | | |
| make check-types | |
| - name: Lint code | |
| if: steps.install.outcome == 'success' && (success() || failure()) | |
| run: | | |
| make check-lint | |
| - name: Verify code formatting | |
| if: steps.install.outcome == 'success' && (success() || failure()) | |
| run: | | |
| make check-format | |
| playwright-shiny: | |
| runs-on: ${{ matrix.os }} | |
| if: github.event_name != 'release' | |
| strategy: | |
| matrix: | |
| python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Determine browsers for testing | |
| uses: ./.github/py-shiny/pytest-browsers | |
| id: browsers | |
| with: | |
| all-browsers: ${{ startsWith(github.head_ref, 'playwright') }} | |
| - name: Display browser | |
| shell: bash | |
| run: echo '${{ steps.browsers.outputs.browsers }}' | |
| - name: Run End-to-End tests | |
| timeout-minutes: 20 | |
| run: | | |
| make playwright-shiny SUB_FILE=". -vv" ${{ steps.browsers.outputs.browsers }} | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: "playright-shiny-${{ matrix.os }}-${{ matrix.python-version }}-results" | |
| path: test-results/ | |
| retention-days: 5 | |
| playwright-examples: | |
| runs-on: ${{ matrix.os }} | |
| if: github.event_name != 'release' | |
| strategy: | |
| matrix: | |
| python-version: ["3.12", "3.11", "3.10", "3.9", "3.8"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "18" | |
| cache: npm | |
| cache-dependency-path: examples/brownian/shinymediapipe/package-lock.json | |
| - name: Install node.js package | |
| working-directory: examples/brownian/shinymediapipe | |
| run: | | |
| npm ci | |
| - name: Determine browsers for testing | |
| uses: ./.github/py-shiny/pytest-browsers | |
| id: browsers | |
| with: | |
| all-browsers: ${{ startsWith(github.head_ref, 'playwright') }} | |
| - name: Run example app tests | |
| timeout-minutes: 20 | |
| run: | | |
| make playwright-examples SUB_FILE=". -vv" ${{ steps.browsers.outputs.browsers }} | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: "playright-examples-${{ matrix.os }}-${{ matrix.python-version }}-results" | |
| path: test-results/ | |
| retention-days: 5 | |
| playwright-deploys-precheck: | |
| if: github.event_name != 'release' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| # Matches deploy server python version | |
| python-version: ["3.10"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Test that deployable example apps work | |
| timeout-minutes: 5 # ~10s locally | |
| env: | |
| DEPLOY_APPS: "false" | |
| run: | | |
| make playwright-deploys SUB_FILE=". -vv" | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: "playright-examples-${{ matrix.os }}-${{ matrix.python-version }}-results" | |
| path: test-results/ | |
| retention-days: 5 | |
| playwright-deploys: | |
| needs: [playwright-deploys-precheck] | |
| if: github.event_name != 'release' && (github.event_name == 'push' || startsWith(github.head_ref, 'deploy')) | |
| # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR) | |
| # Only one is allowed to run at a time because it is deploying to the same server location. | |
| concurrency: playwright-deploys | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| # Matches deploy server python version | |
| python-version: ["3.10"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Test that deployable example apps work | |
| timeout-minutes: 5 # ~10s locally | |
| env: | |
| DEPLOY_APPS: "false" | |
| run: | | |
| make playwright-deploys SUB_FILE=". -vv" | |
| - name: Deploy apps and run tests (on `push` or `deploy**` branches) | |
| env: | |
| DEPLOY_APPS: "true" | |
| DEPLOY_CONNECT_SERVER_URL: "https://rsc.radixu.com/" | |
| DEPLOY_CONNECT_SERVER_API_KEY: "${{ secrets.DEPLOY_CONNECT_SERVER_API_KEY }}" | |
| DEPLOY_SHINYAPPS_NAME: "${{ secrets.DEPLOY_SHINYAPPS_NAME }}" | |
| DEPLOY_SHINYAPPS_TOKEN: "${{ secrets.DEPLOY_SHINYAPPS_TOKEN }}" | |
| DEPLOY_SHINYAPPS_SECRET: "${{ secrets.DEPLOY_SHINYAPPS_SECRET }}" | |
| timeout-minutes: 30 | |
| # Given we are waiting for external servers to finish, | |
| # we can have many local processes waiting for deployment to finish | |
| run: | | |
| make playwright-deploys SUB_FILE=". -vv --numprocesses 12" | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: "playright-deploys-${{ matrix.os }}-${{ matrix.python-version }}-results" | |
| path: test-results/ | |
| retention-days: 5 | |
| pypi: | |
| name: "Deploy to PyPI" | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| needs: [check] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Set up Python 3.10" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install https://github.com/rstudio/py-htmltools/tarball/main | |
| make install-deps | |
| make install | |
| - name: "Build Package" | |
| run: | | |
| make dist | |
| # test deploy ---- | |
| - name: "Test Deploy to PyPI" | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| if: startsWith(github.event.release.name, 'TEST') | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TEST_API_TOKEN }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| ## prod deploy ---- | |
| - name: "Deploy to PyPI" | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| if: startsWith(github.event.release.name, 'shiny') | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| testrail-reporting-nightly: | |
| runs-on: ${{ matrix.os }} | |
| if: ${{ github.event_name == 'schedule' || startsWith(github.head_ref, 'testrail') }} | |
| strategy: | |
| matrix: | |
| python-version: | |
| - "3.12" | |
| - "3.11" | |
| - "3.10" | |
| - "3.9" | |
| # - "3.8" # Testrail has errors when running on python 3.8; Skipping | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup py-shiny | |
| uses: ./.github/py-shiny/setup | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "18" | |
| cache: npm | |
| cache-dependency-path: examples/brownian/shinymediapipe/package-lock.json | |
| - name: Install node.js package | |
| working-directory: examples/brownian/shinymediapipe | |
| run: | | |
| npm ci | |
| - name: Run End-to-End tests | |
| timeout-minutes: 20 | |
| run: | | |
| make testrail-junit | |
| - name: Report results to Testrail | |
| env: | |
| TESTRAIL_URL: "https://posit.testrail.io" | |
| TESTRAIL_PROJECT: "Shiny-Python" | |
| TESTRAIL_USER: "${{ secrets.TESTRAIL_USER }}" | |
| TESTRAIL_PASSWORD: "${{ secrets.TESTRAIL_PASSWORD }}" | |
| run: | | |
| CURRENT_DATE="$(date +'%Y-%m-%d %H:%M:%S') Nightly tests" | |
| trcli \ | |
| --yes \ | |
| --host "$TESTRAIL_URL" \ | |
| --project "$TESTRAIL_PROJECT" \ | |
| --username "$TESTRAIL_USER" \ | |
| --password "$TESTRAIL_PASSWORD" \ | |
| parse_junit \ | |
| --file "report.xml" \ | |
| --title "$CURRENT_DATE" \ | |
| --close-run |