From 79c0bb8cf9bd2e68a0aae0d88ed8123dd36699ff Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Fri, 16 Dec 2022 19:17:26 +0100 Subject: [PATCH 01/14] ci: make go workflow faster --- .github/workflows/go.yml | 76 +++++++++++++++++++++++++++------------- Makefile | 6 +++- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 462669ace52..12bfb4cc561 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,15 +2,19 @@ name: Go on: push: + paths-ignore: + - packaging/** branches: - 'master' pull_request: + paths-ignore: + - packaging/** branches: - '**' jobs: build: - name: Build + name: Build and Test runs-on: ${{ matrix.os }} strategy: matrix: @@ -18,16 +22,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Setup Go - if: matrix.os == 'ubuntu-latest' - uses: actions/setup-go@v3 - with: - cache: false - go-version-file: go.mod - name: Setup Go - if: matrix.os == 'macos-latest' || matrix.os == 'windows-latest' uses: actions/setup-go@v3 with: cache: true @@ -39,35 +34,68 @@ jobs: - name: Commit linting if: matrix.os == 'ubuntu-latest' && github.ref != 'refs/heads/master' uses: wagoid/commitlint-github-action@v5 + - name: Build + run: make build + - name: Test with race detector (Ubuntu and MacOS) + if: matrix.os != 'windows-latest' + run: make test-race + - name: Test without race detector (Windows) + if: matrix.os == 'windows-latest' + run: make test + golangci-lint: + name: GolangCI-Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + cache: false + go-version-file: go.mod - name: GolangCI-Lint if: matrix.os == 'ubuntu-latest' uses: golangci/golangci-lint-action@v3 with: skip-cache: false version: v1.49 - - name: Build - run: make build - - name: Test whitespaces, race detector & code coverage - if: matrix.os == 'ubuntu-latest' - # only generate code coverage profile on ubuntu + coverage: + name: Coverage Report + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + cache: true + go-version-file: go.mod + - name: Test whitespaces and code coverage run: | make check-whitespace - make cover=1 test-race - - name: Test with race detector (MacOS) - if: matrix.os == 'macos-latest' - run: make test-race - - name: Test without race detector (Windows) - if: matrix.os == 'windows-latest' - run: make test + make cover=1 test - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true files: ./cover.out + trigger-beekeeper: + name: Trigger Beekeeper + runs-on: ubuntu-latest + needs: [build, golangci-lint, coverage] + if: github.ref == 'refs/heads/master' && matrix.os == 'ubuntu-latest' && success() + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v3 + with: + cache: false + go-version-file: go.mod - name: Trigger Beekeeper - if: github.ref == 'refs/heads/master' && matrix.os == 'ubuntu-latest' && success() uses: peter-evans/repository-dispatch@v2 with: token: ${{ secrets.GHA_PAT_BASIC }} diff --git a/Makefile b/Makefile index 717e9b5b79b..c3cd1511dfa 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,11 @@ test-integration: .PHONY: test test: - $(GO) test -v -failfast ./... +ifdef cover + $(GO) test -failfast -coverprofile=cover.out -v ./... +else + $(GO) test -failfast -v ./... +endif .PHONY: build build: CGO_ENABLED=0 From 966c0931f455bfd633454415e4ba4b10654ccb32 Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Fri, 16 Dec 2022 19:19:15 +0100 Subject: [PATCH 02/14] ci: check only if master --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 12bfb4cc561..588705ab666 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -84,7 +84,7 @@ jobs: name: Trigger Beekeeper runs-on: ubuntu-latest needs: [build, golangci-lint, coverage] - if: github.ref == 'refs/heads/master' && matrix.os == 'ubuntu-latest' && success() + if: github.ref == 'refs/heads/master' steps: - name: Checkout uses: actions/checkout@v3 From 1f59cea561631796528f2d824e1400b48e231188 Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Fri, 16 Dec 2022 19:20:34 +0100 Subject: [PATCH 03/14] ci: run golangci-lint always --- .github/workflows/go.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 588705ab666..c87c4acd64a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -54,7 +54,6 @@ jobs: cache: false go-version-file: go.mod - name: GolangCI-Lint - if: matrix.os == 'ubuntu-latest' uses: golangci/golangci-lint-action@v3 with: skip-cache: false From 757af2e235bd2ec67a1313a6f92c0e2c3ea7e753 Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Fri, 16 Dec 2022 19:22:49 +0100 Subject: [PATCH 04/14] ci: move commit lint to golangci-lint job --- .github/workflows/go.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c87c4acd64a..e56c41b7364 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -31,9 +31,6 @@ jobs: # make sure that line endings are not converted on windows # as gofmt linter will report that they need to be changed run: git config --global core.autocrlf false - - name: Commit linting - if: matrix.os == 'ubuntu-latest' && github.ref != 'refs/heads/master' - uses: wagoid/commitlint-github-action@v5 - name: Build run: make build - name: Test with race detector (Ubuntu and MacOS) @@ -48,11 +45,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Setup Go uses: actions/setup-go@v3 with: cache: false go-version-file: go.mod + - name: Commit linting + if: github.ref != 'refs/heads/master' + uses: wagoid/commitlint-github-action@v5 - name: GolangCI-Lint uses: golangci/golangci-lint-action@v3 with: From 41c1c6ec137ab169992e7fb715fa470e8521e51f Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Fri, 16 Dec 2022 19:45:55 +0100 Subject: [PATCH 05/14] ci: remove go version check --- Makefile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Makefile b/Makefile index c3cd1511dfa..b026011bce5 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,6 @@ BEEKEEPER_BRANCH ?= master REACHABILITY_OVERRIDE_PUBLIC ?= false BATCHFACTOR_OVERRIDE_PUBLIC ?= 5 -GO_MIN_VERSION ?= "1.17" -GO_BUILD_VERSION ?= "1.17.2" -GO_MOD_ENABLED_VERSION ?= "1.12" -GO_MOD_VERSION ?= "$(shell go mod edit -print | awk '/^go[ \t]+[0-9]+\.[0-9]+(\.[0-9]+)?[ \t]*$$/{print $$2}')" -GO_SYSTEM_VERSION ?= "$(shell go version | awk '{ gsub(/go/, "", $$3); print $$3 }')" - BEE_API_VERSION ?= "$(shell grep '^ version:' openapi/Swarm.yaml | awk '{print $$2}')" BEE_DEBUG_API_VERSION ?= "$(shell grep '^ version:' openapi/SwarmDebug.yaml | awk '{print $$2}')" @@ -126,16 +120,9 @@ endif .PHONY: build build: CGO_ENABLED=0 -build: check-version build: $(GO) build -trimpath -ldflags "$(LDFLAGS)" ./... -.PHONY: check-version -check-version: - [ ${GO_SYSTEM_VERSION} \< ${GO_MOD_ENABLED_VERSION} ] && echo "The version of Golang on the system (${GO_SYSTEM_VERSION}) is too old and does not support go modules. Please use at least ${GO_MIN_VERSION}." && exit 1; exit 0 - [ ${GO_SYSTEM_VERSION} \< ${GO_MIN_VERSION} ] && echo "The version of Golang on the system (${GO_SYSTEM_VERSION}) is below the minimum required version (${GO_MIN_VERSION}) and therefore will not build correctly." && exit 1; exit 0 - if ! expr ${GO_BUILD_VERSION} : ^${GO_MOD_VERSION} 1>/dev/null; then echo "The version of Golang mod (${GO_MOD_VERSION}) does not match required version (${GO_BUILD_VERSION})." && exit 1; fi - .PHONY: githooks githooks: ln -f -s ../../.githooks/pre-push.bash .git/hooks/pre-push From 8ee19ba2182aba39c000f3dd3a5146772197b1af Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 16:19:49 +0100 Subject: [PATCH 06/14] ci: debug workflow --- .gitattributes | 1 + .github/workflows/go.yml | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..a0717e4b3b9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go text eol=lf \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e56c41b7364..2711fc7963b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,12 +27,11 @@ jobs: with: cache: true go-version-file: go.mod - - name: Set git to use LF - # make sure that line endings are not converted on windows - # as gofmt linter will report that they need to be changed - run: git config --global core.autocrlf false - name: Build run: make build + - name: Setup tmate session + if: matrix.os == 'ubuntu-latest' + uses: mxschmitt/action-tmate@v3 - name: Test with race detector (Ubuntu and MacOS) if: matrix.os != 'windows-latest' run: make test-race From 3989c86948826dd1d99a354c5a9ed2b85a04b94e Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 18:02:05 +0100 Subject: [PATCH 07/14] ci: remove failfast because its not cacheable --- .github/workflows/go.yml | 4 ++-- Makefile | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2711fc7963b..ef75a09f87c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -34,10 +34,10 @@ jobs: uses: mxschmitt/action-tmate@v3 - name: Test with race detector (Ubuntu and MacOS) if: matrix.os != 'windows-latest' - run: make test-race + run: make test-race-ci - name: Test without race detector (Windows) if: matrix.os == 'windows-latest' - run: make test + run: make test-ci golangci-lint: name: GolangCI-Lint runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index b026011bce5..310b2184484 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,14 @@ else $(GO) test -race -failfast -v ./... endif +.PHONY: test-race-ci +test-race-ci: +ifdef cover + $(GO) test -race -coverprofile=cover.out -v ./... +else + $(GO) test -race -v ./... +endif + .PHONY: test-integration test-integration: $(GO) test -tags=integration -v ./... @@ -118,6 +126,14 @@ else $(GO) test -failfast -v ./... endif +.PHONY: test-ci +test-ci: +ifdef cover + $(GO) test -coverprofile=cover.out -v ./... +else + $(GO) test -v ./... +endif + .PHONY: build build: CGO_ENABLED=0 build: From 8391cfbfd099a38b0fcdfa5858eaa3a205542485 Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 18:07:05 +0100 Subject: [PATCH 08/14] ci: export CGO_ENABLED --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 310b2184484..93e13c9e6d2 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ else endif .PHONY: build -build: CGO_ENABLED=0 +build: export CGO_ENABLED=0 build: $(GO) build -trimpath -ldflags "$(LDFLAGS)" ./... From e187f64a0a51de12db13a30d2d0c3c0425caaddb Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 18:07:20 +0100 Subject: [PATCH 09/14] ci: add newline --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index a0717e4b3b9..d207b1802b2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -*.go text eol=lf \ No newline at end of file +*.go text eol=lf From be2e533e7a0fbd505127073ff95c72e430587acd Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 18:11:58 +0100 Subject: [PATCH 10/14] ci: remove tmate debug --- .github/workflows/go.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ef75a09f87c..339016b8734 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,9 +29,6 @@ jobs: go-version-file: go.mod - name: Build run: make build - - name: Setup tmate session - if: matrix.os == 'ubuntu-latest' - uses: mxschmitt/action-tmate@v3 - name: Test with race detector (Ubuntu and MacOS) if: matrix.os != 'windows-latest' run: make test-race-ci From 6d520061ac354d828741f7793bf297ee30b3dabd Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 20:41:06 +0100 Subject: [PATCH 11/14] ci: disable cache for coverage report --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 339016b8734..33f3084427c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -65,7 +65,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - cache: true + cache: false go-version-file: go.mod - name: Test whitespaces and code coverage run: | From bfe2aaa7327606996d0b1957789e5b0fa713b09c Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 21:09:30 +0100 Subject: [PATCH 12/14] ci: resolve cache conflicts --- .github/workflows/beekeeper.yml | 11 ++++++++++- .github/workflows/go.yml | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index e17f368a653..c5928e21be2 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -45,8 +45,17 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - cache: true + cache: false go-version-file: go.mod + - name: Cache Go Modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - name: Get Commit Message id: commit run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 33f3084427c..9b8d0108b4b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -70,7 +70,7 @@ jobs: - name: Test whitespaces and code coverage run: | make check-whitespace - make cover=1 test + make cover=1 test-ci - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From 4df0752bf9644834eeda2002bd7f2251eca1717d Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 21:22:30 +0100 Subject: [PATCH 13/14] ci: move ci-stake after ci-pingping and ci-fullconnectivity --- .github/workflows/beekeeper.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index c5928e21be2..8aee6d879bb 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -128,15 +128,15 @@ jobs: - name: Set local cluster run: | timeout ${TIMEOUT} make deploylocal BEEKEEPER_CLUSTER=local-dns - - name: Test staking - id: stake - run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake - name: Test pingpong id: pingpong run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-pingpong; do echo "waiting for pingpong..."; sleep .3; done' - name: Test fullconnectivity id: fullconnectivity run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks=ci-full-connectivity; do echo "waiting for full connectivity..."; sleep .3; done' + - name: Test staking + id: stake + run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-stake - name: Test settlements id: settlements run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-settlements From 4d5a43f40b66bf2e0e36fea63802098fb29466b2 Mon Sep 17 00:00:00 2001 From: Ivan Vandot Date: Sat, 17 Dec 2022 21:31:43 +0100 Subject: [PATCH 14/14] ci: add cache for coverage test --- .github/workflows/go.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9b8d0108b4b..0ebd288b6dc 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -67,6 +67,13 @@ jobs: with: cache: false go-version-file: go.mod + - name: Cache Go Modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-coverage-${{ hashFiles('**/go.sum') }} - name: Test whitespaces and code coverage run: | make check-whitespace