From 49d0d0f0da0f53c5ee7ee1a638b9afb225f493e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Fri, 29 Nov 2019 22:18:40 +0100 Subject: [PATCH] Makefile changes to allow easy builds with or without vendoring. Also fixes version bug for both cases. (#1095) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Extracted -mod=vendor into a variable * Use ?= to set variable only if not set yet. This enables "MOD_VENDOR= make all" use from command line * Enable/disable vendoring with VENDOR=true or false. Enabled by default * Make go generate use -mod=vendor depending on VENDOR flag * Use GOMOD to specify module version to use. GOMOD can be set to empty string, "vendor", "readonly" and in Go 1.14 to "mod". If not defined, defaults to "vendor". Non empty values are passed to -mod parameter to Go tools, and --modules-download-mode parameter for golangci-cli linting tool. Signed-off-by: Peter Štibraný * Fix versioning info when not using vendoring. Signed-off-by: Peter Štibraný * Updated build image to use Golintci-cli v1.21.0 and Go 1.13.4. Signed-off-by: Peter Štibraný * Updated build image to version 0.8.0. Changs: Golintci-cli v1.21.0, Go 1.13.4. Signed-off-by: Peter Štibraný * Renamed MOD_VENDOR to MOD_FLAG to better describe its function. Signed-off-by: Peter Štibraný * Fix version info in all command line tools. Signed-off-by: Peter Štibraný * Use loki-build-image:0.8.0 Signed-off-by: Peter Štibraný --- .circleci/config.yml | 2 +- .drone/drone.yml | 10 ++++---- .golangci.yml | 3 --- Makefile | 42 +++++++++++++++++++++++--------- cmd/docker-driver/Dockerfile | 2 +- cmd/docker-driver/main.go | 1 + cmd/fluent-bit/out_loki.go | 3 ++- cmd/logcli/main.go | 1 + cmd/loki-canary/Dockerfile.cross | 2 +- cmd/loki-canary/main.go | 3 ++- cmd/loki/Dockerfile.cross | 2 +- cmd/loki/main.go | 3 ++- cmd/promtail/Dockerfile.cross | 2 +- cmd/promtail/main.go | 1 + loki-build-image/Dockerfile | 4 +-- pkg/build/build.go | 23 +++++++++++++++++ pkg/promtail/server/ui/doc.go | 2 +- 17 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 pkg/build/build.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 138b9aec12e6..f937312f3dac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ workflows: # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ .defaults: &defaults docker: - - image: grafana/loki-build-image:0.7.4 + - image: grafana/loki-build-image:0.8.0 working_directory: /src/loki jobs: diff --git a/.drone/drone.yml b/.drone/drone.yml index a85e4995e636..c0b5920fc457 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -12,28 +12,28 @@ workspace: steps: - name: test - image: grafana/loki-build-image:0.7.4 + image: grafana/loki-build-image:0.8.0 commands: - make BUILD_IN_CONTAINER=false test depends_on: - clone - name: lint - image: grafana/loki-build-image:0.7.4 + image: grafana/loki-build-image:0.8.0 commands: - make BUILD_IN_CONTAINER=false lint depends_on: - clone - name: check-generated-files - image: grafana/loki-build-image:0.7.4 + image: grafana/loki-build-image:0.8.0 commands: - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone - name: check-mod - image: grafana/loki-build-image:0.7.4 + image: grafana/loki-build-image:0.8.0 commands: - make BUILD_IN_CONTAINER=false check-mod depends_on: @@ -526,7 +526,7 @@ platform: steps: - name: trigger - image: grafana/loki-build-image:0.7.4 + image: grafana/loki-build-image:0.8.0 commands: - ./tools/deploy.sh environment: diff --git a/.golangci.yml b/.golangci.yml index 6d51a26ea0ae..c58d807253a6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,9 +3,6 @@ # options for analysis running run: - # trust vendored dependencies - modules-download-mode: vendor - # default concurrency is a available CPU number concurrency: 16 diff --git a/Makefile b/Makefile index 643b5fc38ec1..78da63c9915f 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,24 @@ .PHONY: benchmark-store, drone, check-mod SHELL = /usr/bin/env bash + +# Empty value = no -mod parameter is used. +# If not empty, GOMOD is passed to -mod= parameter. +# In Go 1.13, "readonly" and "vendor" are accepted. +# In Go 1.14, "readonly", "vendor" and "mod" values are accepted. +# If no value is specified, defaults to "vendor". +# +# Can be used from command line by using "GOMOD= make" (empty = no -mod parameter), or "GOMOD=vendor make" (default). + +GOMOD?=vendor +ifeq ($(strip $(GOMOD)),) # Is empty? + MOD_FLAG= + GOLANGCI_ARG= +else + MOD_FLAG=-mod=$(GOMOD) + GOLANGCI_ARG=--modules-download-mode=$(GOMOD) +endif + ############# # Variables # ############# @@ -20,7 +38,7 @@ IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%, # make BUILD_IN_CONTAINER=false target # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -BUILD_IMAGE_VERSION := 0.7.4 +BUILD_IMAGE_VERSION := 0.8.0 # Docker image info IMAGE_PREFIX ?= grafana @@ -40,15 +58,15 @@ DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o - APP_GO_FILES := $(shell find . $(DONT_FIND) -name .y.go -prune -o -name .pb.go -prune -o -name cmd -prune -o -type f -name '*.go' -print) # Build flags -VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version +VPREFIX := github.com/grafana/loki/pkg/build GO_LDFLAGS := -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION) -X $(VPREFIX).BuildUser=$(shell whoami)@$(shell hostname) -X $(VPREFIX).BuildDate=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -GO_FLAGS := -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -mod vendor -DYN_GO_FLAGS := -ldflags "$(GO_LDFLAGS)" -tags netgo -mod vendor +GO_FLAGS := -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG) +DYN_GO_FLAGS := -ldflags "$(GO_LDFLAGS)" -tags netgo $(MOD_FLAG) # Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented # the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. # Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. -DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo -mod vendor -DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo -mod vendor +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" $(GO_LDFLAGS)" -tags netgo $(MOD_FLAG) +DYN_DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "$(GO_LDFLAGS)" -tags netgo $(MOD_FLAG) NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -186,7 +204,7 @@ promtail-clean-assets: # Rule to generate promtail static assets file $(PROMTAIL_GENERATED_FILE): $(PROMTAIL_UI_FILES) @echo ">> writing assets" - GOFLAGS=-mod=vendor GOOS=$(shell go env GOHOSTOS) go generate -x -v ./pkg/promtail/server/ui + GOFLAGS="$(MOD_FLAG)" GOOS=$(shell go env GOHOSTOS) go generate -x -v ./pkg/promtail/server/ui cmd/promtail/promtail: $(APP_GO_FILES) $(PROMTAIL_GENERATED_FILE) cmd/promtail/main.go CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_GO_FLAGS) -o $@ ./$(@D) @@ -219,14 +237,14 @@ publish: dist ######## lint: - GO111MODULE=on GOGC=10 golangci-lint run -v + GO111MODULE=on GOGC=10 golangci-lint run -v $(GOLANGCI_ARG) ######## # Test # ######## test: all - GOGC=10 go test -mod=vendor -p=4 ./... + GOGC=10 go test $(MOD_FLAG) -p=4 ./... ######### # Clean # @@ -242,7 +260,7 @@ clean: rm -rf dist/ rm -rf cmd/fluent-bit/out_loki.h rm -rf cmd/fluent-bit/out_loki.so - go clean -mod=vendor ./... + go clean $(MOD_FLAG) ./... ######### # YACCs # @@ -482,8 +500,8 @@ build-image-push: build-image ######## benchmark-store: - go run -mod=vendor ./pkg/storage/hack/main.go - go test -mod=vendor ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out + go run $(MOD_FLAG) ./pkg/storage/hack/main.go + go test $(MOD_FLAG) ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out # regenerate drone yaml drone: diff --git a/cmd/docker-driver/Dockerfile b/cmd/docker-driver/Dockerfile index 017e6686b18a..0ab5fa0da317 100644 --- a/cmd/docker-driver/Dockerfile +++ b/cmd/docker-driver/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 +ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/loki -f cmd/loki/Dockerfile . diff --git a/cmd/docker-driver/main.go b/cmd/docker-driver/main.go index 1598998f7e8c..dbd5581929ee 100644 --- a/cmd/docker-driver/main.go +++ b/cmd/docker-driver/main.go @@ -8,6 +8,7 @@ import ( "github.com/docker/go-plugins-helpers/sdk" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + _ "github.com/grafana/loki/pkg/build" "github.com/prometheus/common/version" "github.com/weaveworks/common/logging" ) diff --git a/cmd/fluent-bit/out_loki.go b/cmd/fluent-bit/out_loki.go index 638af0d891fb..3d2cd09d680f 100644 --- a/cmd/fluent-bit/out_loki.go +++ b/cmd/fluent-bit/out_loki.go @@ -2,16 +2,17 @@ package main import ( "C" + "fmt" "time" "unsafe" "github.com/fluent/fluent-bit-go/output" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + _ "github.com/grafana/loki/pkg/build" "github.com/prometheus/common/version" "github.com/weaveworks/common/logging" ) -import "fmt" var plugin *loki var logger log.Logger diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index cae5097bceaf..017be88881ce 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -6,6 +6,7 @@ import ( "os" "time" + _ "github.com/grafana/loki/pkg/build" "github.com/grafana/loki/pkg/logcli/client" "github.com/grafana/loki/pkg/logcli/labelquery" "github.com/grafana/loki/pkg/logcli/output" diff --git a/cmd/loki-canary/Dockerfile.cross b/cmd/loki-canary/Dockerfile.cross index 9cd830a7e69a..701ddadf67fa 100644 --- a/cmd/loki-canary/Dockerfile.cross +++ b/cmd/loki-canary/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 +ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/promtail -f cmd/promtail/Dockerfile . diff --git a/cmd/loki-canary/main.go b/cmd/loki-canary/main.go index eded56a2635b..e166c9135722 100644 --- a/cmd/loki-canary/main.go +++ b/cmd/loki-canary/main.go @@ -13,6 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" + _ "github.com/grafana/loki/pkg/build" "github.com/grafana/loki/pkg/canary/comparator" "github.com/grafana/loki/pkg/canary/reader" "github.com/grafana/loki/pkg/canary/writer" @@ -39,7 +40,7 @@ func main() { flag.Parse() if *printVersion { - fmt.Print(version.Print("loki-canary")) + fmt.Println(version.Print("loki-canary")) os.Exit(0) } diff --git a/cmd/loki/Dockerfile.cross b/cmd/loki/Dockerfile.cross index f061f7c65a2b..546641c6ab89 100644 --- a/cmd/loki/Dockerfile.cross +++ b/cmd/loki/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 +ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/loki -f cmd/loki/Dockerfile . diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 73368fb43942..aa26dba3c5be 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -7,6 +7,7 @@ import ( "reflect" "github.com/go-kit/kit/log/level" + _ "github.com/grafana/loki/pkg/build" "github.com/grafana/loki/pkg/cfg" "github.com/grafana/loki/pkg/loki" "github.com/prometheus/client_golang/prometheus" @@ -31,7 +32,7 @@ func main() { os.Exit(1) } if *printVersion { - fmt.Print(version.Print("loki")) + fmt.Println(version.Print("loki")) os.Exit(0) } diff --git a/cmd/promtail/Dockerfile.cross b/cmd/promtail/Dockerfile.cross index 1962e40a2af0..b1f4e1fea187 100644 --- a/cmd/promtail/Dockerfile.cross +++ b/cmd/promtail/Dockerfile.cross @@ -1,4 +1,4 @@ -ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4 +ARG BUILD_IMAGE=grafana/loki-build-image:0.8.0 # Directories in this file are referenced from the root of the project not this folder # This file is intented to be called from the root like so: # docker build -t grafana/promtail -f cmd/promtail/Dockerfile . diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 78eb17c6012e..36264575f2e1 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/common/version" "github.com/weaveworks/common/logging" + _ "github.com/grafana/loki/pkg/build" "github.com/grafana/loki/pkg/cfg" "github.com/grafana/loki/pkg/logentry/stages" "github.com/grafana/loki/pkg/promtail" diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index 99f20a28a1c6..1412fe2a5035 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -9,12 +9,12 @@ RUN apk add --no-cache curl && \ FROM alpine as golangci RUN apk add --no-cache curl && \ cd / && \ - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.20.0 + curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.21.0 FROM alpine:edge as docker RUN apk add --no-cache docker-cli -FROM golang:1.13.1-stretch +FROM golang:1.13.4-stretch RUN apt-get update && \ apt-get install -qy \ musl \ diff --git a/pkg/build/build.go b/pkg/build/build.go new file mode 100644 index 000000000000..bab8f4d52eb1 --- /dev/null +++ b/pkg/build/build.go @@ -0,0 +1,23 @@ +package build + +import "github.com/prometheus/common/version" + +// Version information passed to Prometheus version package. +// Package path as used by linker changes based on vendoring being used or not, +// so it's easier just to use stable Loki path, and pass it to +// Prometheus in the code. +var ( + Version string + Revision string + Branch string + BuildUser string + BuildDate string +) + +func init() { + version.Version = Version + version.Revision = Revision + version.Branch = Branch + version.BuildUser = BuildUser + version.BuildDate = BuildDate +} diff --git a/pkg/promtail/server/ui/doc.go b/pkg/promtail/server/ui/doc.go index a9fa78522e96..aa28071ff027 100644 --- a/pkg/promtail/server/ui/doc.go +++ b/pkg/promtail/server/ui/doc.go @@ -7,4 +7,4 @@ import ( _ "github.com/shurcooL/vfsgen" ) -//go:generate go run -tags=dev assets_generate.go -build_flags=-mod=vendor +//go:generate go run -tags=dev assets_generate.go -build_flags="$GOFLAGS"