diff --git a/.github/actions/set-up-go/action.yml b/.github/actions/set-up-go/action.yml new file mode 100644 index 0000000000000..c89ccba723ffc --- /dev/null +++ b/.github/actions/set-up-go/action.yml @@ -0,0 +1,70 @@ +--- +name: Set up Go with a shared module cache +description: Set up Go with a shared module cache + +inputs: + no-restore: + description: "Whether or not to restore the Go module cache on a cache hit" + type: boolean + default: false + +outputs: + cache-key: + description: "The Go modules cache key" + value: ${{ steps.metadata.outputs.cache-key }} + cache-path: + description: "The GOMODCACHE path" + value: ${{ steps.metadata.outputs.cache-path }} + go-version: + description: "The version of Go in the .go-version file" + value: ${{ steps.go-version.outputs.go-version }} + +runs: + using: composite + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - id: go-version + shell: bash + run: echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version: ${{ steps.go-version.outputs.go-version }} + cache: false # We use our own caching strategy + - id: metadata + shell: bash + run: | + echo "cache-path=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + echo "cache-key=go-modules-${{ hashFiles('**/go.sum') }}" >> "$GITHUB_OUTPUT" + - id: cache-modules + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + enableCrossOsArchive: true + lookup-only: ${{ inputs.no-restore }} + # We need to be very considerate of our caching strategy because Github only allows 10gb + # of caches per repository before it starts to evict older caches. This is usually fine + # if you only use the actions cache for cache, but we also use it for Go test time results. + # These results are used to balance our Go test groups, without which we could have + # painfully unbalanced Go test execution times. We have to ensure current caches for all + # active release branches and main do not exceed 10gb. Ideally we'd cache Go modules + # and Go build cache on a per version/platform/architecture/tag/module basis, but that + # would result in several hungred gb over all of our build workflows and release branches. + # Instead, we've chosen a middle ground approach where were share Go modules between build + # workflows but lose the Go build cache. + # We intentionally do not use partial restore keys. If we get dont get an exact cache hit + # we only want to download the latest modules, not append them to a prior cache. This + # keeps cache upload time, download time, and storage size to a minimum. + path: ${{ steps.metadata.outputs.cache-path }} + key: ${{ steps.metadata.outputs.cache-key }} + - if: steps.cache-modules.outputs.cache-hit != 'true' + name: Download go modules + shell: bash + run: | + git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com".insteadOf https://github.com + # go list ./... forces downloading some additional versions of modules that 'go mod + # download' misses. We need this because we make use of go list itself during + # code generation in later builds that rely on this module cache. + go list ./... + go list -test ./... + go mod download + ( cd sdk && go mod download ) + ( cd api && go mod download ) diff --git a/.github/workflows/build-vault-oss.yml b/.github/workflows/build-vault-oss.yml index a9e9f0ec9ac30..6d27d2ebf0884 100644 --- a/.github/workflows/build-vault-oss.yml +++ b/.github/workflows/build-vault-oss.yml @@ -24,16 +24,8 @@ on: goarch: required: true type: string - go-cache: - required: true - type: string - go-mod-cache: - required: true - type: string go-tags: type: string - go-version: - type: string package-name: type: string default: vault @@ -50,33 +42,14 @@ jobs: name: Vault ${{ inputs.goos }} ${{ inputs.goarch }} v${{ inputs.vault-version }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version: ${{ inputs.go-version }} - cache: false # Use our own caching strategy for better cross platform support - - name: Set up Go cache key tags - id: cache-key-tags - run: echo "gotags=$(echo ${{ inputs.go-tags }} | tr ' ' '-')" >> "$GITHUB_ENV" - - name: Set up Go cache - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - path: | - ${{ inputs.go-cache }} - ${{ inputs.go-mod-cache }} - # Manage the Go cache for each build workflow individually. This ensures that only relevant - # module and build cache for that specific combination kept. This helps reduce our cache - # download and speeds up compiling because the build cache is always preserved. - key: go-${{ inputs.go-version }}-${{ inputs.goos }}-${{ inputs.goarch }}-${{ env.gotags }}-${{ hashFiles('**/go.sum') }} - # We intentionally omit partial restore keys to ensure that we always create a new cache - # if we don't get a hit. That ensures that we only keep up-to-date modules and build cache. + - uses: ./github/actions/set-up-go - name: Restore UI from cache uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: + # Restore the UI asset from the UI build workflow. Never use a partial restore key. enableCrossOsArchive: true fail-on-cache-miss: true path: http/web_ui - # Only restore the UI asset cache if we haven't modified anything in the ui directory. - # Never do a partial restore of the web_ui if we don't get a cache hit. key: ${{ inputs.web-ui-cache-key }} - name: Build Vault env: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8252e60094db7..51516a82664bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,14 +8,16 @@ on: # This is insufficient for our needs, since we're skipping stuff on PRs in # draft mode. By adding the ready_for_review type, when a draft pr is marked # ready, we run everything, including the stuff we'd have skipped up until now. - types: [ opened, synchronize, reopened, ready_for_review ] + types: [opened, synchronize, reopened, ready_for_review] push: branches: - main - release/** + concurrency: group: ${{ github.head_ref || github.run_id }}-build cancel-in-progress: true + jobs: # verify-changes determines if the changes are only for docs (website) verify-changes: @@ -31,24 +33,18 @@ jobs: outputs: build-date: ${{ steps.get-metadata.outputs.build-date }} filepath: ${{ steps.generate-metadata-file.outputs.filepath }} - go-cache: ${{ steps.get-metadata.outputs.go-cache }} - go-mod-cache: ${{ steps.get-metadata.outputs.go-mod-cache }} - go-version: ${{ steps.go-version.outputs.go-version }} matrix-test-group: ${{ steps.get-metadata.outputs.matrix-test-group }} package-name: ${{ steps.get-metadata.outputs.package-name }} vault-revision: ${{ steps.get-metadata.outputs.vault-revision }} vault-version: ${{ steps.get-metadata.outputs.vault-version }} vault-base-version: ${{ steps.get-metadata.outputs.vault-base-version }} - web-ui-cache-key: ui-${{ steps.get-metadata.outputs.web-ui-cache-key }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Determine Go version - id: go-version - run: echo "go-version=$(cat ./.go-version)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + - name: Ensure Go modules are cached + uses: ./.github/actions/set-up-go + id: set-up-go with: - go-version: ${{ steps.go-version.outputs.go-version }} - cache: false + no-restore: true # don't download them on a cache hit - name: Get metadata id: get-metadata env: @@ -59,13 +55,10 @@ jobs: run: | # shellcheck disable=SC2129 echo "build-date=$(make ci-get-date)" >> "$GITHUB_OUTPUT" - echo "go-cache=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" - echo "go-mod-cache=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" echo "matrix-test-group=$(make ci-get-matrix-group-id)" >> "$GITHUB_OUTPUT" echo "package-name=vault" >> "$GITHUB_OUTPUT" echo "vault-base-version=$(make ci-get-version-base)" >> "$GITHUB_OUTPUT" echo "vault-revision=$(make ci-get-revision)" >> "$GITHUB_OUTPUT" - echo "web-ui-cache-key=$(git ls-tree HEAD ui --object-only)" >> "$GITHUB_OUTPUT" echo "vault-version=$(make ci-get-version)" >> "$GITHUB_OUTPUT" - uses: hashicorp/actions-generate-metadata@v1 id: generate-metadata-file @@ -131,10 +124,7 @@ jobs: create-packages: false goarch: ${{ matrix.goarch }} goos: ${{ matrix.goos }} - go-cache: ${{ needs.product-metadata.outputs.go-cache }} - go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }} go-tags: ui - go-version: ${{ needs.product-metadata.outputs.go-version }} package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} @@ -154,10 +144,7 @@ jobs: with: goarch: ${{ matrix.goarch }} goos: ${{ matrix.goos }} - go-cache: ${{ needs.product-metadata.outputs.go-cache }} - go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }} go-tags: ui - go-version: ${{ needs.product-metadata.outputs.go-version }} package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} @@ -178,10 +165,7 @@ jobs: create-packages: false goarch: ${{ matrix.goarch }} goos: ${{ matrix.goos }} - go-cache: ${{ needs.product-metadata.outputs.go-cache }} - go-mod-cache: ${{ needs.product-metadata.outputs.go-mod-cache }} go-tags: ui - go-version: ${{ needs.product-metadata.outputs.go-version }} package-name: ${{ needs.product-metadata.outputs.package-name }} web-ui-cache-key: ${{ needs.build-ui.outputs.cache-key }} vault-version: ${{ needs.product-metadata.outputs.vault-version }} @@ -302,7 +286,7 @@ jobs: - run: | echo "Some of the required build and test workflows have failed!" exit 1 - + notify-completed-successfully-failures-oss: if: ${{ always() && github.repository == 'hashicorp/vault' && needs.completed-successfully.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} runs-on: ubuntu-latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d05d00a4cae4..8786015c4e545 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,11 @@ on: - main - release/** workflow_dispatch: + concurrency: group: ${{ github.head_ref || github.run_id }}-ci cancel-in-progress: true + jobs: setup: name: Setup @@ -24,8 +26,9 @@ jobs: compute-larger: ${{ steps.setup-outputs.outputs.compute-larger }} compute-huge: ${{ steps.setup-outputs.outputs.compute-huge }} enterprise: ${{ steps.setup-outputs.outputs.enterprise }} - go-build-tags: ${{ steps.setup-outputs.outputs.go-build-tags }} + go-tags: ${{ steps.setup-outputs.outputs.go-tags }} steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - id: setup-outputs name: Setup outputs run: | @@ -38,7 +41,7 @@ jobs: echo 'compute-larger=["self-hosted","ondemand","linux","type=m5.2xlarge"]' >> "$GITHUB_OUTPUT" echo 'compute-huge=["self-hosted","ondemand","linux","type=m5.4xlarge"]' >> "$GITHUB_OUTPUT" echo 'enterprise=1' >> "$GITHUB_OUTPUT" - echo 'go-build-tags=ent,enterprise' >> "$GITHUB_OUTPUT" + echo 'go-tags=ent,enterprise' >> "$GITHUB_OUTPUT" else # shellcheck disable=SC2129 echo 'compute-tiny="ubuntu-latest"' >> "$GITHUB_OUTPUT" # 2 cores, 7 GB RAM, 14 GB SSD @@ -46,16 +49,13 @@ jobs: echo 'compute-larger="custom-linux-medium-vault-latest"' >> "$GITHUB_OUTPUT" # 16 cores, 64 GB RAM, 600 GB SSD echo 'compute-huge="custom-linux-xl-vault-latest"' >> "$GITHUB_OUTPUT" # 32-cores, 128 GB RAM, 1200 GB SSD echo 'enterprise=' >> "$GITHUB_OUTPUT" - echo 'go-build-tags=' >> "$GITHUB_OUTPUT" + echo 'go-tags=' >> "$GITHUB_OUTPUT" fi - setup-go-cache: - name: Go Caches - needs: - - setup - uses: ./.github/workflows/setup-go-cache.yml - with: - runs-on: ${{ needs.setup.outputs.compute-standard }} - secrets: inherit + - name: Ensure Go modules are cached + uses: ./.github/actions/set-up-go + with: + no-restore: true # don't download them on a cache hit + diff-oss-ci: name: Diff OSS needs: @@ -88,14 +88,15 @@ jobs: - id: diff run: | ./.github/scripts/oss-diff.sh ${{ steps.determine-branch.outputs.BRANCH }} HEAD + verify-changes: name: Verify doc-ui only PRs uses: ./.github/workflows/verify_changes.yml + test-go: name: Run Go tests needs: - setup - - setup-go-cache - verify-changes # Don't run this job for docs/ui only PRs if: | @@ -109,15 +110,15 @@ jobs: # other tests aren't slowed down waiting for a binary build. total-runners: 17 go-arch: amd64 - go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock' + go-tags: '${{ needs.setup.outputs.go-tags }},deadlock' runs-on: ${{ needs.setup.outputs.compute-larger }} enterprise: ${{ needs.setup.outputs.enterprise }} secrets: inherit + test-go-race: name: Run Go tests with data race detection needs: - setup - - setup-go-cache - verify-changes # Don't run this job for docs/ui only PRs if: | @@ -133,11 +134,12 @@ jobs: } extra-flags: '-race' go-arch: amd64 - go-build-tags: ${{ needs.setup.outputs.go-build-tags }} + go-tags: ${{ needs.setup.outputs.go-tags }} runs-on: ${{ needs.setup.outputs.compute-huge }} enterprise: ${{ needs.setup.outputs.enterprise }} name: "-race" secrets: inherit + test-go-fips: name: Run Go tests with FIPS configuration # Only run this job for the enterprise repo if the PR is not docs/ui only @@ -148,7 +150,6 @@ jobs: needs.verify-changes.outputs.is_ui_change == 'false' needs: - setup - - setup-go-cache - verify-changes uses: ./.github/workflows/test-go.yml with: @@ -158,11 +159,12 @@ jobs: "GOEXPERIMENT": "boringcrypto" } go-arch: amd64 - go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock,cgo,fips,fips_140_2' + go-tags: '${{ needs.setup.outputs.go-tags }},deadlock,cgo,fips,fips_140_2' runs-on: ${{ needs.setup.outputs.compute-larger }} enterprise: ${{ needs.setup.outputs.enterprise }} name: "-fips" secrets: inherit + test-ui: name: Test UI # The test-ui job is only run on: @@ -184,10 +186,7 @@ jobs: runs-on: ${{ fromJSON(needs.setup.outputs.compute-larger) }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true + - uses: ./github/actions/set-up-go # Setup node.js without caching to allow running npm install -g yarn (next step) - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 with: @@ -261,17 +260,18 @@ jobs: paths: "ui/test-results/qunit/results.xml" show: "fail" if: always() + tests-completed: needs: - setup - - setup-go-cache - test-go - test-ui - if: always() + if: always() runs-on: ${{ fromJSON(needs.setup.outputs.compute-tiny) }} steps: - run: | tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)' + notify-tests-completed-failures-oss: if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }} runs-on: ubuntu-latest @@ -324,7 +324,7 @@ jobs: slack-bot-token: ${{ steps.secrets.outputs.SLACK_BOT_TOKEN }} payload: | {"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]} - + test-summary: name: Go test failures runs-on: ubuntu-latest diff --git a/.github/workflows/code-checker.yml b/.github/workflows/code-checker.yml index 584f707740a18..87b78627401ce 100644 --- a/.github/workflows/code-checker.yml +++ b/.github/workflows/code-checker.yml @@ -7,6 +7,7 @@ on: branches: - main - release/** + concurrency: group: ${{ github.head_ref || github.run_id }}-lint cancel-in-progress: true @@ -20,38 +21,31 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: fetch-depth: 0 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true + - uses: ./.github/actions/set-up-go - run: make ci-deprecations name: Check deprecations + codechecker: - name: Code checks + name: Code checks runs-on: ubuntu-latest if: github.base_ref == 'main' steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: fetch-depth: 0 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true + - uses: ./.github/actions/set-up-go # Note: if there is a function we want to ignore the nilnil check for, # You can add 'ignore-nil-nil-function-check' somewhere in the # godoc for the function. - run: make ci-vet-codechecker name: Check custom linters + format: name: Format runs-on: ubuntu-latest steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true + - uses: ./.github/actions/set-up-go - name: Go format run: | make ci-bootstrap @@ -61,6 +55,7 @@ jobs: echo "Code has formatting errors. Run 'make fmt' to fix" exit 1 fi + semgrep: name: Semgrep runs-on: ubuntu-latest diff --git a/.github/workflows/plugin-update-check.yml b/.github/workflows/plugin-update-check.yml index 96588a9e377b8..fd02739a728de 100644 --- a/.github/workflows/plugin-update-check.yml +++ b/.github/workflows/plugin-update-check.yml @@ -28,6 +28,7 @@ jobs: - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: + cache: false # save cache space for vault builds: https://github.com/hashicorp/vault/pull/21764 go-version-file: .go-version - name: update plugin diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 336505aab1d27..5cdd37436045b 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -18,6 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: + cache: false # save cache space for vault builds: https://github.com/hashicorp/vault/pull/21764 go-version: 1.18 - name: Set up Python @@ -42,19 +43,19 @@ jobs: cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-semgrep" go build -o scan-plugin-semgrep . mv scan-plugin-semgrep "$HOME/.bin" - + cd "$GITHUB_WORKSPACE/security-scanner/pkg/sdk/examples/scan-plugin-codeql" go build -o scan-plugin-codeql . mv scan-plugin-codeql "$HOME/.bin" - + # Semgrep python3 -m pip install semgrep - + # CodeQL LATEST=$(gh release list --repo https://github.com/github/codeql-action | cut -f 3 | sort --version-sort | tail -n1) gh release download --repo https://github.com/github/codeql-action --pattern codeql-bundle-linux64.tar.gz "$LATEST" tar xf codeql-bundle-linux64.tar.gz -C "$HOME/.bin" - + # Add to PATH echo "$HOME/.bin" >> "$GITHUB_PATH" echo "$HOME/.bin/codeql" >> "$GITHUB_PATH" diff --git a/.github/workflows/setup-go-cache.yml b/.github/workflows/setup-go-cache.yml deleted file mode 100644 index 8d6927f0534a9..0000000000000 --- a/.github/workflows/setup-go-cache.yml +++ /dev/null @@ -1,33 +0,0 @@ -on: - workflow_call: - inputs: - runs-on: - required: true - type: string -jobs: - setup-go-cache: - runs-on: ${{ fromJSON(inputs.runs-on) }} - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - id: setup-go - name: Setup go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true - - id: setup-git - name: Setup Git configuration - run: | - git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com".insteadOf https://github.com - - id: download-modules - name: Download go modules - run: | - # go list ./... forces downloading some additional versions of modules that 'go mod - # download' misses. We need this because we make use of go list itself during - # code generation in later builds that rely on this module cache. - go list ./... - go list -test ./... - - go mod download - ( cd sdk && go mod download ) - ( cd api && go mod download ) diff --git a/.github/workflows/test-enos-scenario-ui.yml b/.github/workflows/test-enos-scenario-ui.yml index 36218a3236e4c..aeb7360ca7043 100644 --- a/.github/workflows/test-enos-scenario-ui.yml +++ b/.github/workflows/test-enos-scenario-ui.yml @@ -68,10 +68,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Set Up Go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version + - uses: ./.github/actions/set-up-go - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 370b878e07483..f5ff30336b84d 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -28,7 +28,7 @@ on: required: false type: string default: ubuntu-latest - go-build-tags: + go-tags: description: A comma-separated list of additional build tags to consider satisfied during the build. required: false type: string @@ -43,26 +43,21 @@ on: default: 20 type: number timeout-minutes: - description: The maximum number of minutes that this workflow should run + description: The maximum number of minutes that this workflow should run required: false - default: 60 + default: 60 type: number - env: ${{ fromJSON(inputs.env-vars) }} jobs: - test-matrix: - permissions: - id-token: write # Note: this permission is explicitly required for Vault auth - contents: read + runner-indexes: runs-on: ${{ fromJSON(inputs.runs-on) }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + - uses: ./.github/actions/set-up-go with: - go-version-file: ./.go-version - cache: true + no-restore: true # We don't need the vault Go modules when generating indices - name: Authenticate to Vault id: vault-auth if: github.repository == 'hashicorp/vault-enterprise' @@ -144,26 +139,32 @@ jobs: }]' matrix.json ) >> "$GITHUB_OUTPUT" outputs: - matrix: ${{ steps.build.outputs.matrix }} + runner-indexes: ${{ steps.generate-index-list.outputs.indexes }} + steps: + - id: generate-index-list + run: | + INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})" + INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")" + echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}" test-go: needs: test-matrix permissions: id-token: write # Note: this permission is explicitly required for Vault auth contents: read + name: "${{ matrix.runner-index }}" + needs: + - runner-indexes runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.test-matrix.outputs.matrix) }} env: GOPRIVATE: github.com/hashicorp/* - TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }} + TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version - cache: true + - uses: ./.github/actions/set-up-go - name: Authenticate to Vault id: vault-auth if: github.repository == 'hashicorp/vault-enterprise' @@ -216,7 +217,7 @@ jobs: # not what developers have in their environments and it could break some # tests; it would be like setting VAULT_TOKEN. However some non-Go # CI commands, like the UI tests, shouldn't have to worry about licensing. - # So we provide the tests which want an externally supplied license with licenses + # So we provide the tests which want an externally supplied license with licenses # via the VAULT_LICENSE_CI and VAULT_LICENSE_2 environment variables, and here we unset it. # shellcheck disable=SC2034 VAULT_LICENSE= @@ -239,7 +240,7 @@ jobs: VAULT_BINARY="$(pwd)/bin/vault" export VAULT_BINARY fi - + # shellcheck disable=SC2086 # can't quote package list GOARCH=${{ inputs.go-arch }} \ go run gotest.tools/gotestsum --format=short-verbose \ @@ -247,13 +248,13 @@ jobs: --jsonfile test-results/go-test/results-${{ matrix.id }}.json \ --jsonfile-timing-events failure-summary-${{ matrix.id }}${{inputs.name}}.json \ -- \ - -tags "${{ inputs.go-build-tags }}" \ + -tags "${{ inputs.go-tags }}" \ -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \ -parallel=${{ inputs.go-test-parallelism }} \ ${{ inputs.extra-flags }} \ ${{ matrix.packages }} - name: Prepare datadog-ci - if: github.repository == 'hashicorp/vault' && (success() || failure()) + if: github.repository == 'hashicorp/vault' && (success() || failure()) continue-on-error: true run: | curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" @@ -297,4 +298,4 @@ jobs: - run: | ls -lhR test-results/go-test find test-results/go-test -mindepth 1 -mtime +3 -delete - ls -lhR test-results/go-test \ No newline at end of file + ls -lhR test-results/go-test diff --git a/.github/workflows/test-run-acc-tests-for-path.yml b/.github/workflows/test-run-acc-tests-for-path.yml index 35f177ea11ca7..cbd066f6c8ac8 100644 --- a/.github/workflows/test-run-acc-tests-for-path.yml +++ b/.github/workflows/test-run-acc-tests-for-path.yml @@ -21,10 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Set Up Go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: ./.go-version + - uses: ./.github/actions/set-up-go - run: go test -v ./${{ inputs.path }}/... 2>&1 | tee ${{ inputs.name }}.txt - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: