Daily image cache build for autotests #1012
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily image cache build for autotests | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '0 3 * * 1-5' | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| - name: Identify branches | |
| id: branches | |
| run: | | |
| BRANCHES="$(echo "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/branches{/branch}" | \ | |
| ./tools/github/workflow/grabrelease.py --all --json --add master)" | |
| echo "Will build image cache for branches: ${BRANCHES}" | |
| echo "branches=${BRANCHES}" >> $GITHUB_OUTPUT | |
| outputs: | |
| branches: ${{ steps.branches.outputs.branches }} | |
| build_autotests: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| branch: ${{ fromJson(needs.setup.outputs.branches) }} | |
| steps: | |
| - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| with: | |
| ref: ${{ matrix.branch }} | |
| - name: Check and get files | |
| id: files | |
| shell: bash | |
| run: | | |
| MATRIX_PARSER_PY="${{ github.workspace }}/tools/test/matrix_parser.py" | |
| MATRIX_CONF="${{ github.workspace }}/etc/docker/test/matrix.yml" | |
| BUILD_IMAGES_PY="${{ github.workspace }}/tools/test/build_images.py" | |
| echo "requirements_met=$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT | |
| echo "matrix_parser=$MATRIX_PARSER_PY" >> $GITHUB_OUTPUT | |
| echo "matrix_configuration=$MATRIX_CONF" >> $GITHUB_OUTPUT | |
| echo "build_images=$BUILD_IMAGES_PY" >> $GITHUB_OUTPUT | |
| - name: Update pip, install python requirements for matrix parser | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| run: | | |
| python3 -m pip install -U pip setuptools | |
| python3 -m pip install -U PyYAML | |
| - name: Identify Matrix | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| id: matrix | |
| run: echo "matrix=$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})" >> $GITHUB_OUTPUT | |
| - name: Build and upload images | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| id: images | |
| shell: bash | |
| run: | | |
| BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache") | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| # only build without cache on schedule (to update distro packages) | |
| BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache") | |
| fi | |
| # the {} input for build_images.py is necessary until version 2 | |
| if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then | |
| # new --branch option not available in older build_images.py revisions | |
| BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}") | |
| fi | |
| if [[ "${{ matrix.branch }}" != "master" ]]; then | |
| # The image tag is the same for the different rucio versions. This if ensures, that only the master gets builded. rucio#5572 | |
| echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master" | |
| exit 0 | |
| fi | |
| docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} | |
| i=0; until [ "$i" -ge 3 ]; do | |
| IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} ./etc/docker/test || echo "")" | |
| if [[ -n "$IMAGES" ]]; then break; | |
| else | |
| i=$((i+1)); sleep 5; | |
| echo "::warning::Building images failed, retrying…" | |
| fi | |
| done | |
| docker logout https://ghcr.io | |
| if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi | |
| build_integration_tests: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - setup | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| branch: ${{ fromJson(needs.setup.outputs.branches) }} | |
| steps: | |
| - name: Checkout rucio containers repository | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| with: | |
| repository: rucio/containers | |
| - name: Checkout rucio source | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 | |
| with: | |
| path: dev/rucio | |
| ref: ${{ matrix.branch }} | |
| - name: Check and get files | |
| id: files | |
| shell: bash | |
| run: | | |
| MATRIX_PARSER_PY="${{ github.workspace }}/dev/rucio/tools/test/matrix_parser.py" | |
| MATRIX_CONF="${{ github.workspace }}/dev/rucio/etc/docker/test/matrix_integration_tests.yml" | |
| BUILD_IMAGES_PY="${{ github.workspace }}/dev/rucio/tools/test/build_images.py" | |
| echo "requirements_met=$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT | |
| echo "matrix_parser=$MATRIX_PARSER_PY" >> $GITHUB_OUTPUT | |
| echo "matrix_configuration=$MATRIX_CONF" >> $GITHUB_OUTPUT | |
| echo "build_images=$BUILD_IMAGES_PY" >> $GITHUB_OUTPUT | |
| - name: Update pip, install python requirements for matrix parser | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| run: | | |
| python3 -m pip install -U pip setuptools | |
| python3 -m pip install -U PyYAML | |
| - name: Identify Matrix | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| id: matrix | |
| run: echo "matrix=$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})" >> $GITHUB_OUTPUT | |
| - name: Build and upload images | |
| if: ${{ steps.files.outputs.requirements_met == 'true' }} | |
| id: images | |
| shell: bash | |
| run: | | |
| BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache") | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| # only build without cache on schedule (to update distro packages) | |
| BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache") | |
| fi | |
| # the {} input for build_images.py is necessary until version 2 | |
| if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then | |
| # new --branch option not available in older build_images.py revisions | |
| BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}") | |
| fi | |
| if [[ "${{ matrix.branch }}" != "master" ]]; then | |
| # The image tag is the same for the different rucio versions. This if ensures, that only the master gets builded. rucio#5572 | |
| echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master" | |
| exit 0 | |
| fi | |
| docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} | |
| i=0; until [ "$i" -ge 3 ]; do | |
| IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} \ | |
| ${{ github.workspace }}/dev || echo "")" | |
| if [[ -n "$IMAGES" ]]; then break; | |
| else | |
| i=$((i+1)); sleep 5; | |
| echo "::warning::Building images failed, retrying…" | |
| fi | |
| done | |
| docker logout https://ghcr.io | |
| if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi |