Go 1.20.7 build image + log/cli/scli output improvements + other deps (…

…#811)

* Go 1.20.7 build image

* use the newer cli/scli/log with clearer color output. bump up grpc too

* update for color detection + bug fix in release search and replace, also bump cli with version without extra paren after buildinfo help

* adding note about color

.circleci/config.yml

@@ -8,7 +8,7 @@ defaultEnv:
   &defaultEnv
   docker:
     # specify the version
-    - image: docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7
+    - image: docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301
   working_directory: /build/fortio
 
 jobs:

Dockerfile

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7 as build
+FROM docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301 as build
 WORKDIR /build
 COPY --chown=build:build . fortio
 ARG MODE=install

Dockerfile.build

@@ -1,5 +1,5 @@
 # Dependencies and linters for build:
-FROM golang:1.20.6@sha256:8e5a0067e6b387263a01d06b91ef1a983f90e9638564f6e25392fd2695f7ab6c
+FROM golang:1.20.7@sha256:bc5f0b5e43282627279fe5262ae275fecb3d2eae3b33977a7fd200c7a760d6f1
 # Need gcc for -race test (and some linters though those work with CGO_ENABLED=0)
 RUN apt-get -y update && \
   apt-get --no-install-recommends -y upgrade && \

Dockerfile.echosrv

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7 as build
+FROM docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301 as build
 WORKDIR /build
 COPY . fortio
 RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/echosrv

Dockerfile.fcurl

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7 as build
+FROM docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301 as build
 WORKDIR /build
 COPY . fortio
 RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/fcurl

Makefile

@@ -7,7 +7,7 @@
 IMAGES=echosrv fcurl # plus the combo image / Dockerfile without ext.
 
 DOCKER_PREFIX := docker.io/fortio/fortio
-BUILD_IMAGE_TAG := v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7
+BUILD_IMAGE_TAG := v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301
 BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
 BUILDX_POSTFIX :=
 ifeq '$(shell echo $(BUILDX_PLATFORMS) | awk -F "," "{print NF-1}")' '0'

README.md

@@ -1,4 +1,4 @@
-<!-- 1.57.4 -->
+<!-- 1.57.5 -->
 # Fortio
 
 [![Awesome Go](https://fortio.org/mentioned-badge.svg)](https://github.com/avelino/awesome-go#networking)
@@ -60,13 +60,13 @@ You can install from source:
 The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets):
 
 ```shell
-curl -L https://github.com/fortio/fortio/releases/download/v1.57.4/fortio-linux_amd64-1.57.4.tgz \
+curl -L https://github.com/fortio/fortio/releases/download/v1.57.5/fortio-linux_amd64-1.57.5.tgz \
  | sudo tar -C / -xvzpf -
 # or the debian package
-wget https://github.com/fortio/fortio/releases/download/v1.57.4/fortio_1.57.4_amd64.deb
-dpkg -i fortio_1.57.4_amd64.deb
+wget https://github.com/fortio/fortio/releases/download/v1.57.5/fortio_1.57.5_amd64.deb
+dpkg -i fortio_1.57.5_amd64.deb
 # or the rpm
-rpm -i https://github.com/fortio/fortio/releases/download/v1.57.4/fortio-1.57.4-1.x86_64.rpm
+rpm -i https://github.com/fortio/fortio/releases/download/v1.57.5/fortio-1.57.5-1.x86_64.rpm
 # and more, see assets in release page
 ```
 
@@ -76,7 +76,7 @@ On a MacOS you can also install Fortio using [Homebrew](https://brew.sh/):
 brew install fortio
 ```
 
-On Windows, download https://github.com/fortio/fortio/releases/download/v1.57.4/fortio_win_1.57.4.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
+On Windows, download https://github.com/fortio/fortio/releases/download/v1.57.5/fortio_win_1.57.5.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
 ```
 fortio.exe server
 ```
@@ -128,7 +128,7 @@ Full list of command line flags (`fortio help`):
 <!-- use release/updateFlags.sh to update this section -->
 <pre>
 <!-- USAGE_START -->
-Φορτίο 1.57.4 usage:
+Φορτίο 1.57.5 usage:
         fortio command [flags] target
 where command is one of: load (load testing), server (starts ui, rest api,
  http-echo, redirect, proxies, tcp-echo, udp-echo and grpc ping servers),
@@ -416,6 +416,8 @@ There is also the GRPC health and ping servers, as well as the http->https redir
 
 ## Example use and output
 
+Note that recent versions of fortio, when run on the console, have colorized output (and JSON when ran in server environment). What's below is plain text from older versions.
+
 ### Start the internal servers
 
 ```Shell

Webtest.sh

@@ -140,7 +140,7 @@ fi
 PPROF_URL="$BASE_URL/debug/pprof/heap?debug=1"
 $CURL "$PPROF_URL" | grep -i TotalAlloc # should find this in memory profile
 # creating dummy container to hold a volume for test certs due to remote docker bind mount limitation.
-DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7 sleep 120)
+DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301 sleep 120)
 # while we have something with actual curl binary do
 # Test for h2c upgrade (#562)
 docker exec $DOCKERSECVOLNAME /usr/bin/curl -v --http2 -m 10 -d foo42 http://localhost:8080/debug | tee >(cat 1>&2) | grep foo42

fnet/network.go

@@ -450,12 +450,12 @@ func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
 		log.Debugf("read %d from %+v: %v", nr, src, er)
 		if nr > 0 {
 			nw, ew := dst.Write(buf[0:nr])
-			log.Debugf("wrote %d (expected %d) to %+v: %v", nw, nr, dst, ew)
+			log.Debugf("wrote %d (expected %d) to %p: %v", nw, nr, dst, ew)
 			if nw > 0 {
 				written += int64(nw)
 			}
 			if ew != nil {
-				log.Errf("copy: %+v -> %+v write error: %v", src, dst, ew)
+				log.Errf("copy: %p -> %p write error: %v", src, dst, ew)
 				err = ew
 				break
 			}
@@ -468,10 +468,10 @@ func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
 			if os.IsTimeout(er) {
 				// return but not log as error (for UDPNetCat use case)
 				err = er
-				log.LogVf("copy: %+v -> %+v timeout/read error: %v", src, dst, er)
+				log.LogVf("copy: %p -> %p timeout/read error: %v", src, dst, er)
 			} else if !errors.Is(er, io.EOF) {
 				err = er
-				log.Errf("copy: %+v -> %+v read error: %v", src, dst, er)
+				log.Errf("copy: %p -> %p read error: %v", src, dst, er)
 			}
 			break
 		}

go.mod

@@ -4,16 +4,16 @@ go 1.18
 
 require (
 	fortio.org/assert v1.2.0
-	fortio.org/cli v1.2.0
+	fortio.org/cli v1.3.3
 	fortio.org/dflag v1.5.2
-	fortio.org/log v1.8.1
-	fortio.org/scli v1.9.0
+	fortio.org/log v1.9.3
+	fortio.org/scli v1.10.3
 	fortio.org/testscript v0.3.1
 	fortio.org/version v1.0.2
 	github.com/golang/protobuf v1.5.3
 	github.com/google/uuid v1.3.0
 	golang.org/x/net v0.13.0
-	google.golang.org/grpc v1.56.2
+	google.golang.org/grpc v1.57.0
 )
 
 // Local dev of dependencies changes
@@ -29,10 +29,10 @@ require (
 require (
 	fortio.org/sets v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
+	golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/tools v0.8.0 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -1,13 +1,13 @@
 fortio.org/assert v1.2.0 h1:XscfvR8yp4xW7OMCvNbCsieRFDxlwdEcb69+JZRp6LA=
 fortio.org/assert v1.2.0/go.mod h1:039mG+/iYDPO8Ibx8TrNuJCm2T2SuhwRI3uL9nHTTls=
-fortio.org/cli v1.2.0 h1:MXmbYnbNHcclf0xYLZjeCnTINby7FpooJxR01+hnBmM=
-fortio.org/cli v1.2.0/go.mod h1:zJwHjRnvVqneF7ES+REfElc33xMjhbRDJslSrqIK9Kk=
+fortio.org/cli v1.3.3 h1:b2JPQ7ekVFpF7SrEllJTbh1rdivsyXT081oQUeiuIp0=
+fortio.org/cli v1.3.3/go.mod h1:ZsevXTvMidh3xllYoAElS52XRcGx+f1hVlgB+v2Pa0Y=
 fortio.org/dflag v1.5.2 h1:F9XVRj4Qr2IbJP7BMj7XZc9wB0Q/RZ61Ool+4YPVad8=
 fortio.org/dflag v1.5.2/go.mod h1:ppb/A8u+KKg+qUUYZNYuvRnXuVb8IsdHb/XGzsmjkN8=
-fortio.org/log v1.8.1 h1:Ybtjp+qDXb124ESnZ0XVuXJ6khDrWcom/C55YPupoKg=
-fortio.org/log v1.8.1/go.mod h1:u/8/2lyczXq52aT5Nw6reD+3cR6m/EbS2jBiIYhgiTU=
-fortio.org/scli v1.9.0 h1:laTn/mACr1+TRKH3ZKaQWw58kTzQ1ntpg2sH02KUJAA=
-fortio.org/scli v1.9.0/go.mod h1:Ox3nhp3eJpF2rQsZ8fBKXqdJGQPTQHWEZpLikDnkzjY=
+fortio.org/log v1.9.3 h1:rkgDmgF7dLfNRtVOE44Av5KBlro8FkpTKnTnphHy074=
+fortio.org/log v1.9.3/go.mod h1:u/8/2lyczXq52aT5Nw6reD+3cR6m/EbS2jBiIYhgiTU=
+fortio.org/scli v1.10.3 h1:6aNn9ggOAoyp9vDs3rXN3uMZiRHcVdlPbsTHoN8PfiE=
+fortio.org/scli v1.10.3/go.mod h1:qfmKaJUtGumrCs0+CfErIFp2qjT2OV6BTBGXGgRqaY4=
 fortio.org/sets v1.0.3 h1:HzewdGjH69YmyW06yzplL35lGr+X4OcqQt0qS6jbaO4=
 fortio.org/sets v1.0.3/go.mod h1:QZVj0r6KP/ZD9ebySW9SgxVNy/NjghUfyHW9NN+WU+4=
 fortio.org/testscript v0.3.1 h1:MmRO64AsmzaU1KlYMzAbotJIMKRGxD1XXssJnBRiMGQ=
@@ -23,8 +23,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
-golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
+golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -35,10 +35,10 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
 golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I=
+google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
+google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=

release/Dockerfile.in

@@ -1,5 +1,5 @@
 # Concatenated after ../Dockerfile to create the tgz
-FROM docker.io/fortio/fortio.build:v61@sha256:98038f5f81bdc65fcc1a99abcba1ff6a2e22f60d627d26ad75ad8d952888e8f7 as stage
+FROM docker.io/fortio/fortio.build:v62@sha256:f7d431202f97b3dd7aab8bdfcf0ea9a4217740fb9ea0ffcb434d27bcbbe67301 as stage
 ARG archs="amd64 arm64 ppc64le s390x"
 ENV archs=${archs}
 # Build image defaults to build user, switch back to root for

release/bumpRelease.sh

@@ -26,7 +26,8 @@ fi
 echo "Changing $FILENAME from $CURRENT to release $RELEASE and updating usage section"
 ./release/updateFlags.sh | sed -e "s/Φορτίο dev/Φορτίο $RELEASE/" > /tmp/fortio_flags.txt
 cp README.md README.md.bak
-sed -e "s/$CURRENT/$RELEASE/g" README.md.bak | \
+SEARCH=${CURRENT//\./\\.} # escape dots so we don't go and replace 12534 in a timestamp when looking for 1.5.4
+sed -e "s/$SEARCH/$RELEASE/g" README.md.bak | \
    awk '/USAGE_START/ {print $0; skip=1; system("cat /tmp/fortio_flags.txt")} /USAGE_END/ {skip=0} {if (!skip) print $0}' \
    > README.md
 echo "DONE. Check the diff:"

release/updateFlags.sh

@@ -15,4 +15,6 @@
 #
 # Extract fortio's help and rewrap it to 90 cols (as MD doesn't wrap code blocks)
 # Also remove the /var/folders/fq/gng4z4915mb73r9js_422h4c0000gn/T/go-build179128464/b001/exe/ noise
-go run . help | expand | sed -e 's!/.*/fortio !fortio !' | fold -s -w 90 | sed -e "s/ $//" -e "s/</\&lt;/"
+# The 2>&1 isn't because the help goes to stderr (it doesn't) but to make the cli/log package
+# detect we're not on the console and thus not use color mode.
+go run . help 2>&1 | expand | sed -e 's!/.*/fortio !fortio !' | fold -s -w 90 | sed -e "s/ $//" -e "s/</\&lt;/"

stats/stats_test.go

@@ -47,7 +47,7 @@ func TestCounter(t *testing.T) {
 	log.SetFlags(0)
 	log.SetOutput(w)
 	c.Export().CalcPercentile(50)
-	expected += "E > Unexpected call to CalcPercentile(50) with no data\n"
+	expected += "[E]> Unexpected call to CalcPercentile(50) with no data\n"
 	c.Record(23.1)
 	c.Counter.Print(w, "test2")
 	expected += "test2 : count 1 avg 23.1 +/- 0 min 23.1 max 23.1 sum 23.1\n"
@@ -70,7 +70,7 @@ func TestCounter(t *testing.T) {
 	log.Config.LogFileAndLine = false
 	log.Config.LogPrefix = ""
 	c.Counter.Log("testLogC")
-	expected += "I testLogC" + finalExpected
+	expected += "[I] testLogC" + finalExpected
 	_ = w.Flush()
 	actual := b.String()
 	if actual != expected {
@@ -80,7 +80,7 @@ func TestCounter(t *testing.T) {
 	c.Log("testLogH", nil)
 	_ = w.Flush()
 	actual = b.String()
-	expected = "I testLogH" + finalExpected + `# range, mid point, percentile, count
+	expected = "[I] testLogH" + finalExpected + `# range, mid point, percentile, count
 >= -977 <= 22 , -477.5 , 16.67, 1
 > 22.8 <= 22.9 , 22.85 , 50.00, 2
 > 23 <= 23.1 , 23.05 , 83.33, 2