[QT-589] Use the go module cache between CI and build #3
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | ||
| workflow_call: | ||
| inputs: | ||
| go-arch: | ||
| description: The execution architecture (arm, amd64, etc.) | ||
| required: true | ||
| type: string | ||
| enterprise: | ||
| description: A flag indicating if this workflow is executing for the enterprise repository. | ||
| required: true | ||
| type: string | ||
| total-runners: | ||
| description: Number of runners to use for executing the tests on. | ||
| required: true | ||
| type: string | ||
| env-vars: | ||
| description: A map of environment variables as JSON. | ||
| required: false | ||
| type: string | ||
| default: '{}' | ||
| extra-flags: | ||
| description: A space-separated list of additional build flags. | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| runs-on: | ||
| description: An expression indicating which kind of runners to use. | ||
| required: false | ||
| type: string | ||
| default: ubuntu-latest | ||
| go-tags: | ||
| description: A comma-separated list of additional build tags to consider satisfied during the build. | ||
| required: false | ||
| type: string | ||
| name: | ||
| description: A suffix to append to archived test results | ||
| required: false | ||
| default: '' | ||
| type: string | ||
| go-test-parallelism: | ||
| description: The parallelism parameter for Go tests | ||
| required: false | ||
| default: 20 | ||
| type: number | ||
| timeout-minutes: | ||
| description: The maximum number of minutes that this workflow should run | ||
| required: false | ||
| default: 60 | ||
| type: number | ||
| env: ${{ fromJSON(inputs.env-vars) }} | ||
| jobs: | ||
| runner-indexes: | ||
| runs-on: ${{ fromJSON(inputs.runs-on) }} | ||
| steps: | ||
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
| - uses: ./.github/actions/set-up-go | ||
| with: | ||
| no-restore: true # We don't need the vault Go modules when generating indices | ||
| - name: Authenticate to Vault | ||
| id: vault-auth | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| run: vault-auth | ||
| - name: Fetch Secrets | ||
| id: secrets | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e | ||
| with: | ||
| url: ${{ steps.vault-auth.outputs.addr }} | ||
| caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} | ||
| token: ${{ steps.vault-auth.outputs.token }} | ||
| secrets: | | ||
| kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY; | ||
| kv/data/github/${{ github.repository }}/github-token username-and-token | github-token; | ||
| kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI; | ||
| kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID; | ||
| - id: setup-git-private | ||
| name: Setup Git configuration (private) | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| run: | | ||
| git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com | ||
| - id: setup-git-public | ||
| name: Setup Git configuration (public) | ||
| if: github.repository != 'hashicorp/vault-enterprise' | ||
| run: | | ||
| git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com | ||
| - run: go install gotest.tools/gotestsum@v1.9.0 | ||
| - run: mkdir -p test-results/go-test | ||
| # We use a unique "read-" prefix to guarantee that we're not scribbling on | ||
| # the aggregated test data in the event of test failure. This key is | ||
| # unique for every test run and just used to restore the previous | ||
| # aggregated data. We persist all test data after a successful run and | ||
| # store that in the go-test-reports- cache. | ||
| - id: restore-from-cache | ||
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 | ||
| with: | ||
| path: test-results/go-test | ||
| key: read-go-test-reports-${{ github.run_number }} | ||
| restore-keys: go-test-reports- | ||
| - name: List cached results | ||
| id: list-cached-results | ||
| run: ls -lhR test-results/go-test | ||
| - name: Build matrix excluding binary tests | ||
| id: build-non-binary | ||
| env: | ||
| GOPRIVATE: github.com/hashicorp/* | ||
| run: | | ||
| ( | ||
| go list ./... | grep -v "_binary" | gotestsum tool ci-matrix --debug \ | ||
| --partitions 16 \ | ||
| --timing-files 'test-results/go-test/*.json' > matrix.json | ||
| ) | ||
| - name: Capture list of binary tests | ||
| id: list-binary-tests | ||
| run: | | ||
| LIST="$(go list ./... | grep "_binary" | xargs)" | ||
| echo "list=$LIST" >> "$GITHUB_OUTPUT" | ||
| - name: Build complete matrix | ||
| id: build | ||
| run: | | ||
| set -exo pipefail | ||
| export BINARY_TESTS="${{ steps.list-binary-tests.outputs.list }}" | ||
| ( | ||
| echo -n "matrix=" | ||
| jq -c --arg BINARY "${BINARY_TESTS}" \ | ||
| '.include += [{ | ||
| "id": 16, | ||
| "estimatedRuntime": "N/A", | ||
| "packages": $BINARY, | ||
| "description": "partition 16 - binary test packages" | ||
| }]' matrix.json | ||
| ) >> "$GITHUB_OUTPUT" | ||
| outputs: | ||
| runner-indexes: ${{ steps.generate-index-list.outputs.indexes }} | ||
| steps: | ||
| - id: generate-index-list | ||
| run: | | ||
| INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})" | ||
| INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")" | ||
| echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}" | ||
| test-go: | ||
| needs: test-matrix | ||
| permissions: | ||
| id-token: write # Note: this permission is explicitly required for Vault auth | ||
| contents: read | ||
| name: "${{ matrix.runner-index }}" | ||
| needs: | ||
| - runner-indexes | ||
| runs-on: ${{ fromJSON(inputs.runs-on) }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJSON(needs.test-matrix.outputs.matrix) }} | ||
| env: | ||
| GOPRIVATE: github.com/hashicorp/* | ||
| TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }} | ||
| steps: | ||
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 | ||
| - uses: ./.github/actions/set-up-go | ||
| - name: Authenticate to Vault | ||
| id: vault-auth | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| run: vault-auth | ||
| - name: Fetch Secrets | ||
| id: secrets | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e | ||
| with: | ||
| url: ${{ steps.vault-auth.outputs.addr }} | ||
| caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} | ||
| token: ${{ steps.vault-auth.outputs.token }} | ||
| secrets: | | ||
| kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY; | ||
| kv/data/github/${{ github.repository }}/github-token username-and-token | github-token; | ||
| kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI; | ||
| kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET; | ||
| kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID; | ||
| - id: setup-git-private | ||
| name: Setup Git configuration (private) | ||
| if: github.repository == 'hashicorp/vault-enterprise' | ||
| run: | | ||
| git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com | ||
| - id: setup-git-public | ||
| name: Setup Git configuration (public) | ||
| if: github.repository != 'hashicorp/vault-enterprise' | ||
| run: | | ||
| git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com | ||
| - id: build | ||
| if: contains(matrix.packages, '_binary') | ||
| env: | ||
| GOPRIVATE: github.com/hashicorp/* | ||
| run: time make ci-bootstrap dev | ||
| - id: run-go-tests | ||
| name: Run Go tests | ||
| timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }} | ||
| env: | ||
| COMMIT_SHA: ${{ github.sha }} | ||
| run: | | ||
| set -exo pipefail | ||
| # Build the dynamically generated source files. | ||
| make prep | ||
| # We don't want VAULT_LICENSE set when running Go tests, because that's | ||
| # not what developers have in their environments and it could break some | ||
| # tests; it would be like setting VAULT_TOKEN. However some non-Go | ||
| # CI commands, like the UI tests, shouldn't have to worry about licensing. | ||
| # So we provide the tests which want an externally supplied license with licenses | ||
| # via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it. | ||
| # shellcheck disable=SC2034 | ||
| VAULT_LICENSE= | ||
| # Assign test licenses to relevant variables if they aren't already | ||
| if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then | ||
| export VAULT_LICENSE_CI=${{ secrets.ci_license }} | ||
| export VAULT_LICENSE_2=${{ secrets.ci_license_2 }} | ||
| export HCP_API_ADDRESS=${{ secrets.HCP_API_ADDRESS }} | ||
| export HCP_AUTH_URL=${{ secrets.HCP_AUTH_URL }} | ||
| export HCP_CLIENT_ID=${{ secrets.HCP_CLIENT_ID }} | ||
| export HCP_CLIENT_SECRET=${{ secrets.HCP_CLIENT_SECRET }} | ||
| export HCP_RESOURCE_ID=${{ secrets.HCP_RESOURCE_ID }} | ||
| # Temporarily removing this variable to cause HCP Link tests | ||
| # to be skipped. | ||
| #export HCP_SCADA_ADDRESS=${{ secrets.HCP_SCADA_ADDRESS }} | ||
| fi | ||
| if [ -f bin/vault ]; then | ||
| VAULT_BINARY="$(pwd)/bin/vault" | ||
| export VAULT_BINARY | ||
| fi | ||
| # shellcheck disable=SC2086 # can't quote package list | ||
| GOARCH=${{ inputs.go-arch }} \ | ||
| go run gotest.tools/gotestsum --format=short-verbose \ | ||
| --junitfile test-results/go-test/results-${{ matrix.id }}.xml \ | ||
| --jsonfile test-results/go-test/results-${{ matrix.id }}.json \ | ||
| --jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \ | ||
| -- \ | ||
| -tags "${{ inputs.go-tags }}" \ | ||
| -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ | ||
| -parallel=${{ inputs.go-test-parallelism }} \ | ||
| ${{ inputs.extra-flags }} \ | ||
| ${{ matrix.packages }} | ||
| - name: Prepare datadog-ci | ||
| if: github.repository == 'hashicorp/vault' && (success() || failure()) | ||
| continue-on-error: true | ||
| run: | | ||
| curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" | ||
| chmod +x /usr/local/bin/datadog-ci | ||
| - name: Upload test results to DataDog | ||
| continue-on-error: true | ||
| env: | ||
| DD_ENV: ci | ||
| run: | | ||
| if [[ ${{ github.repository }} == 'hashicorp/vault' ]]; then | ||
| export DATADOG_API_KEY=${{ secrets.DATADOG_API_KEY }} | ||
| fi | ||
| datadog-ci junit upload --service "$GITHUB_REPOSITORY" test-results/go-test/results-${{ matrix.id }}.xml | ||
| if: success() || failure() | ||
| - name: Archive test results | ||
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | ||
| with: | ||
| name: test-results${{ inputs.name }} | ||
| path: test-results/go-test | ||
| if: success() || failure() | ||
| - name: Upload failure summary | ||
| uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 | ||
| if: success() || failure() | ||
| with: | ||
| name: failure-summary | ||
| path: failure-summary-${{ matrix.id }}${{inputs.name}}.json | ||
| test-collect-reports: | ||
| needs: test-go | ||
| runs-on: ${{ fromJSON(inputs.runs-on) }} | ||
| steps: | ||
| - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 | ||
| with: | ||
| path: test-results/go-test | ||
| key: go-test-reports-${{ github.run_number }} | ||
| restore-keys: go-test-reports- | ||
| - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 | ||
| with: | ||
| name: test-results | ||
| path: test-results/go-test | ||
| - run: | | ||
| ls -lhR test-results/go-test | ||
| find test-results/go-test -mindepth 1 -mtime +3 -delete | ||
| ls -lhR test-results/go-test | ||
GitHub Actions