Long-running generative tests #550
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Long-running generative tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "26 0 * * *" | |
| jobs: | |
| gentests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: opensafely-core/setup-action@v1 | |
| with: | |
| install-just: true | |
| python-version: "3.11" | |
| - name: Set up dev environment | |
| run: just devenv | |
| # To work around memory leak issues in our database containers we run the | |
| # tests in multiple smaller batches, recreating the containers between | |
| # each batch | |
| - name: Run `just test tests/generative/test_query_model.py::test_query_model` in batches | |
| run: | | |
| set -euo pipefail | |
| start_time=$(date +%s) | |
| export \ | |
| GENTEST_EXAMPLES=150 \ | |
| GENTEST_RANDOMIZE=t \ | |
| GENTEST_MAX_DEPTH=25 | |
| # We get a maximum of 6 hours runtime with Github Actions so breaking | |
| # the loop after 4 hours feels like a reasonably balance of getting a | |
| # decent amount of generative testing done while leaving plenty of | |
| # headroom. | |
| end_time=$((start_time + 4 * 3600)) | |
| i=1 | |
| while true; do | |
| echo | |
| echo "==> Running test batch $i with $GENTEST_EXAMPLES examples" | |
| echo | |
| # Do the actual testing | |
| set +e | |
| just test tests/generative/test_query_model.py::test_query_model | |
| rc=$? | |
| set -e | |
| # 6 == dead database error, continue to remove db containers and start new batch | |
| if [[ $rc -ne 0 && $rc -ne 6 ]]; then | |
| exit $rc | |
| fi | |
| if [[ $(date +%s) -ge $end_time ]]; then | |
| break | |
| fi | |
| just remove-database-containers | |
| ((i++)) | |
| done | |
| - name: "Notify Slack on Failure" | |
| if: failure() && github.ref_name == 'main' | |
| uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e # v1.6.0 | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| with: | |
| channel_id: C069YDR4NCA | |
| status: "Generative Test Failure" | |
| color: danger |