Fix all the dynamically generated code (#547)

Lots of fixes to stuff that is dynamically generated along with a few extra bits here and there in related code to make things better.

- [x] Fixes the `go generate` calls
- [x] Changes protobuf/grpc stuff to use buf for both dependency management and building/calling protoc
- [x] Changes from goimports to gofumpt as we want this anyway
- [x] Build the tools in tools.go via the Makefile
- [x] Dropped protoc-gen-doc support as that seemed unused. (easy to bring back when we have a use for it)
- [x] Add a make target to check that the generated files are up to date
- [x] Prettier-ify the GHA yaml files too (not just *.yml files)
- [x] Update README/docs
- [x] Setup CI to ensure files are up to date

<!--- Please describe what this PR is going to change -->

Mostly fixes for dynamically generated files.

<!--- Link to issue you have raised -->

I noticed that #541 had some protobuf format commits which I was not expecting being done separately. I went looking into it and noticed the mess we're in.

## How Has This Been Tested?

CI

## How are existing users impacted? What migration steps/scripts do we need?

Will have better automation for messing with generated files.

## Checklist:

I have:

- [x] updated the documentation and/or roadmap (if required)
- [x] added unit or e2e tests
- [ ] provided instructions on how to upgrade

.github/workflows/ci.yaml

@@ -6,7 +6,7 @@ env:
   CGO_ENABLED: 0
 
 jobs:
-  lint:
+  verify:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -15,8 +15,8 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: "1.17.0"
-      - name: make lint
-        run: make lint
+      - name: make verify
+        run: make verify
   test:
     runs-on: ubuntu-latest
     steps:
@@ -43,8 +43,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - ci-checks
-      - lint
       - test
+      - verify
     steps:
       - name: fake
         run: echo ":+1:"

CONTRIBUTING.md

@@ -177,8 +177,11 @@ It has a base server that implements the API.
 ### protos
 
 The `protos` package contains all the protobuf files used by the gRPC server.
-Handling of protobuf deps and generating the go files are both handled by the [protoc.sh](./protos/protoc.sh) script.
-Also, both `go`, and `protoc` are required by `protoc.sh`.
+Handling of protobuf deps and generating the go files are both handled [buf] via `make pbfiles`.
+The protubuf/grpc files are not generated as part of the build pipeline to keep builds fast.
+CI will ensure generated files are up to date.
+
+[buf]: https://buf.build/
 
 ```
 .

Makefile

@@ -1,6 +1,10 @@
+help: ## Print this help
+	@grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*##/·/' | sort | column -ts '·' -c 120
+
 all: cli server worker ## Build all binaries for host OS and CPU
 
 -include rules.mk
+-include lint.mk
 
 crosscompile: $(crossbinaries) ## Build all binaries for Linux and all supported CPU arches
 images: tink-cli-image tink-server-image tink-worker-image ## Build all docker images
@@ -10,53 +14,12 @@ test: ## Run tests
 	go clean -testcache
 	go test ./... -v
 
-verify: lint
+verify: lint check-generated # Verify code style, is lint free, freshness ...
+	gofumpt -s -d .
 
-help: ## Print this help
-	@grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sed 's/:.*##/·/' | sort | column -ts '·' -c 120
+generated: pbfiles protomocks ## Generate dynamically created files
+check-generated: check-pbfiles check-protomocks ## Check if generated files are up to date
+
+tools: ${toolsBins} ## Build Go based build tools
 
-# BEGIN: lint-install --dockerfile=warn .
-# http://github.com/tinkerbell/lint-install
-
-GOLINT_VERSION ?= v1.42.0
-HADOLINT_VERSION ?= v2.7.0
-SHELLCHECK_VERSION ?= v0.7.2
-LINT_OS := $(shell uname)
-LINT_ARCH := $(shell uname -m)
-LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
-
-# shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation
-ifeq ($(LINT_OS),Darwin)
-	ifeq ($(LINT_ARCH),arm64)
-		LINT_ARCH=x86_64
-	endif
-endif
-
-LINT_LOWER_OS  = $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
-GOLINT_CONFIG:=$(LINT_ROOT)/.golangci.yml
-
-lint: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-	out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run
-	out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) --no-fail $(shell find . -name "*Dockerfile")
-	out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh")
-
-fix: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-	out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run --fix
-	out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh") -f diff | git apply -p2 -
-
-out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck:
-	mkdir -p out/linters
-	curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_LOWER_OS).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
-	mv out/linters/shellcheck-$(SHELLCHECK_VERSION) out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
-
-out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH):
-	mkdir -p out/linters
-	curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
-	chmod u+x out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
-
-out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH):
-	mkdir -p out/linters
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLINT_VERSION)
-	mv out/linters/golangci-lint out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
-
-# END: lint-install --dockerfile=warn .
+.PHONY: all check-generated crosscompile generated help images run test tools verify

buf.gen.yaml

@@ -0,0 +1,13 @@
+version: v1
+plugins:
+  - name: go
+    out: .
+    opt: paths=source_relative
+  - name: go-grpc
+    out: .
+    opt:
+      - paths=source_relative
+      - require_unimplemented_servers=false
+  - name: grpc-gateway
+    out: .
+    opt: paths=source_relative

buf.lock

@@ -0,0 +1,10 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    branch: main
+    commit: 389e82394fcc4e18b2463c26674170b3
+    digest: b1-j-IWUPcGJWMkqbee6hR7IJCXlLF9Y5P4ARue8gLwLP0=
+    create_time: 2021-10-01T15:05:49.20888Z

buf.yaml

@@ -0,0 +1,9 @@
+version: v1
+lint:
+  use:
+    - DEFAULT
+deps:
+  - buf.build/googleapis/googleapis
+breaking:
+  use:
+    - FILE

ci-checks.sh

@@ -12,7 +12,7 @@ if ! git ls-files '*.sh' '*.go' '*.md' | xargs codespell -q 3 -I .codespell-whit
 fi
 
 # --check doesn't show what line number fails, so write the result to disk for the diff to catch
-if ! git ls-files '*.yml' '*.json' '*.md' | xargs prettier --list-different --write; then
+if ! git ls-files '*.yaml' '*.yml' '*.json' '*.md' | xargs prettier --list-different --write; then
 	failed=1
 fi
 

go.mod

@@ -3,6 +3,7 @@ module github.com/tinkerbell/tink
 go 1.13
 
 require (
+	github.com/bufbuild/buf v1.0.0-rc2
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/equinix-labs/otel-init-go v0.0.1
@@ -16,6 +17,7 @@ require (
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/ktr0731/evans v0.10.0
 	github.com/lib/pq v1.10.1
+	github.com/matryer/moq v0.2.3
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550
 	github.com/pkg/errors v0.9.1
@@ -24,15 +26,16 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
-	github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816
 	github.com/stretchr/testify v1.7.0
 	github.com/testcontainers/testcontainers-go v0.11.1
 	go.mongodb.org/mongo-driver v1.1.2 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.22.0
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
-	google.golang.org/grpc v1.39.0
+	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6
+	google.golang.org/grpc v1.41.0-dev.0.20210907181116-2f3355d2244e
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v2 v2.4.0
+	mvdan.cc/gofumpt v0.1.1
 )
 
 replace github.com/stormcat24/protodep => github.com/ackintosh/protodep v0.0.0-20200728152107-abf8eb579d6c