codeql v2 and a lot of github actions update, sha pinning in dockerfi…

…le etc (#647)

* codeql v2 - fixes #642

* also pin the build image by sha

* update echo srv standalone main for full version

* update update-build-image-tag to work with presence of SHA and update docs on what to do

* resurrect the dockhub auto build for fcurl and echosrv

* use fixed assert package with Fail, improve chance of less spurious failures in circleci with timing test, remove unused makefile/dockerfile.tests for dflag/

* stop using deprecated set-output in actions. use v3 for checkout hopefully last build warnings

* prep for 1.38.3, fix docker actions sha/pins, get fcurl/echosrv from right path

* docker qemu also need v2

.circleci/config.yml

@@ -8,7 +8,7 @@ defaultEnv:
   &defaultEnv
   docker:
     # specify the version
-    - image: docker.io/fortio/fortio.build:v50
+    - image: docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c
   working_directory: /build/fortio
 
 jobs:

.github/workflows/codeql-analysis.yml

@@ -40,22 +40,22 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # pin@v2
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@a6611b86918424d4588efe7d6dbe18fe52d42518 # pin@v1
+        uses: github/codeql-action/init@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # pin@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
           # By default, queries listed here will override any specified in a config file.
           # Prefix the list here with "+" to use these queries and those in the config file.
           # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
+          # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+          # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@a6611b86918424d4588efe7d6dbe18fe52d42518 # pin@v1
+        uses: github/codeql-action/autobuild@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # pin@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -66,4 +66,4 @@ jobs:
       #   make bootstrap
       #   make release
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@a6611b86918424d4588efe7d6dbe18fe52d42518 # pin@v1
+        uses: github/codeql-action/analyze@c3b6fce4ee2ca25bc1066aa3bf73962fda0e8898 # pin@v2

.github/workflows/main.yml

@@ -22,20 +22,20 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # pin@v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v1
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v2
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # pin@v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # pin@v2
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2
         with:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_TOKEN }}
@@ -46,27 +46,27 @@ jobs:
           make info
           make release
           VERSION=$(make echo-version)
-          echo ::set-output name=VERSION::${VERSION}
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
           PACKAGE_VERSION=$(make echo-package-version)
-          echo ::set-output name=PACKAGE_VERSION::${PACKAGE_VERSION}
           echo "Version $VERSION, Package version $PACKAGE_VERSION"
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # pin@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # pin@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
           push: true
-          tags: fortio/fortio:${{ steps.build.outputs.VERSION }}, fortio/fortio:latest
+          tags: fortio/fortio:${{ env.VERSION }}, fortio/fortio:latest
 
       - name: Create Release
         id: create_release
+        # Need to find a replacement not using set-output
         uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # pin@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
         with:
           tag_name: ${{ github.ref }}
-          release_name: Fortio ${{ steps.build.outputs.VERSION }}
+          release_name: Fortio ${{ env.VERSION }}
           draft: true
 
       - name: Upload release artifacts

.github/workflows/manual-build.yml

@@ -21,26 +21,24 @@ jobs:
         run: |
           echo "tag is ${{ github.event.inputs.tag }}"
 
-      - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # pin@v2
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
         with:
           ref: ${{ github.event.inputs.tag }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v1
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # pin@v2
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # pin@v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # pin@v2
 
       - name: Build
         id: build
         run: |
           make info
           make release
           VERSION=$(make echo-version)
-          echo ::set-output name=VERSION::${VERSION}
           PACKAGE_VERSION=$(make echo-package-version)
-          echo ::set-output name=PACKAGE_VERSION::${PACKAGE_VERSION}
           echo "Version $VERSION, Package version $PACKAGE_VERSION"
 
       - name: Upload release artifacts

Dockerfile

@@ -1,5 +1,5 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v50 as build
+FROM docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c as build
 WORKDIR /build
 COPY . fortio
 ARG MODE=install

Dockerfile.echosrv

@@ -1,10 +1,10 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v50 as build
+FROM docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c as build
 WORKDIR /build
 COPY . fortio
-RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/echosrv OFFICIAL_BIN=../echosrv.bin
+RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/echosrv
 # Minimal image with just the binary
 FROM scratch
-COPY --from=build /build/echosrv.bin /usr/bin/echosrv
+COPY --from=build /build/result/echosrv /usr/bin/echosrv
 EXPOSE 8080
 ENTRYPOINT ["/usr/bin/echosrv"]

Dockerfile.fcurl

@@ -1,11 +1,10 @@
 # Build the binaries in larger image
-FROM docker.io/fortio/fortio.build:v50 as build
+FROM docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c as build
 WORKDIR /build
 COPY . fortio
-# fcurl should not need vendor/no dependencies
-RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/fcurl OFFICIAL_BIN=../fcurl.bin
+RUN make -C fortio official-build-version BUILD_DIR=/build OFFICIAL_TARGET=fortio.org/fortio/fcurl
 # Minimal image with just the binary and certs
 FROM scratch
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /build/fcurl.bin /usr/bin/fcurl
+COPY --from=build /build/result/fcurl /usr/bin/fcurl
 ENTRYPOINT ["/usr/bin/fcurl"]

Makefile

@@ -7,7 +7,7 @@
 IMAGES=echosrv fcurl # plus the combo image / Dockerfile without ext.
 
 DOCKER_PREFIX := docker.io/fortio/fortio
-BUILD_IMAGE_TAG := v50
+BUILD_IMAGE_TAG := v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c
 BUILDX_PLATFORMS := linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
 BUILDX_POSTFIX :=
 ifeq '$(shell echo $(BUILDX_PLATFORMS) | awk -F "," "{print NF-1}")' '0'
@@ -109,10 +109,14 @@ update-build-image:
 	docker buildx create --use
 	$(MAKE) docker-push-internal IMAGE=.build TAG=$(BUILD_IMAGE_TAG)
 
+# Get the sha (use after newly building a new build image) to put it back in BUILD_IMAGE_TAG
+build-image-sha:
+	docker inspect $(BUILD_IMAGE) | jq -r '.[0].RepoDigests[0]' | sed -e "s/^.*@/$(BUILD_IMAGE_TAG)@/"
+
 SED:=sed
 update-build-image-tag:
 	@echo 'Need to use gnu sed (brew install gnu-sed; make update-build-image-tag SED=gsed)'
-	$(SED) --in-place=.bak -e 's!$(DOCKER_PREFIX).build:v..!$(BUILD_IMAGE)!g' $(FILES_WITH_IMAGE)
+	$(SED) --in-place=.bak -E -e 's!$(DOCKER_PREFIX).build:v[^ ]+!$(BUILD_IMAGE)!g' $(FILES_WITH_IMAGE)
 
 docker-default-platform:
 	@docker buildx --builder default inspect | tail -1 | sed -e "s/Platforms: //" -e "s/,//g" | awk '{print $$1}'
@@ -136,15 +140,16 @@ release: dist
 
 .PHONY: all docker-internal docker-push-internal docker-version test dependencies
 
-.PHONY: go-install lint install-linters coverage webtest release-test update-build-image
+.PHONY: go-install lint install-linters coverage webtest release-test update-build-image build-image-sha
 
 .PHONY: local-lint update-build-image-tag release pull certs certs-clean
 
 # Targets used for official builds (initially from Dockerfile)
 BUILD_DIR := /tmp/fortio_build
 BUILD_DIR_ABS := $(abspath $(BUILD_DIR))
 BUILD_DIR_BIN := $(BUILD_DIR_ABS)/bin
-OFFICIAL_BIN ?= $(BUILD_DIR)/result/fortio
+OFFICIAL_EXE ?= $(notdir $(OFFICIAL_TARGET))
+OFFICIAL_BIN ?= $(BUILD_DIR)/result/$(OFFICIAL_EXE)
 OFFICIAL_DIR ?= $(dir $(OFFICIAL_BIN))
 
 GOOS :=
@@ -178,14 +183,16 @@ $(OFFICIAL_DIR):
 official-build: official-build-internal
 
 official-build-internal: $(BUILD_DIR) $(OFFICIAL_DIR)
+	@echo "Building OFFICIAL_EXE=$(OFFICIAL_EXE) BUILD_DIR=$(BUILD_DIR) BUILD_DIR_BIN=$(BUILD_DIR_BIN) MODE=$(MODE)"
+	@echo "OFFICIAL_BIN=$(OFFICIAL_BIN) OFFICIAL_DIR=$(OFFICIAL_DIR) OFFICIAL_TARGET=$(OFFICIAL_TARGET)"
 	$(GO_BIN) version
 ifeq ($(MODE),install)
 	GOPATH=$(BUILD_DIR_ABS) CGO_ENABLED=0 GOOS=$(GOOS) $(GO_BIN) install -a -ldflags -s $(OFFICIAL_TARGET)@v$(DIST_VERSION)
 	# rename when building cross architecture (on windows it has .exe suffix thus the *)
 	ls -lR $(BUILD_DIR_BIN)
-	-mv -f $(BUILD_DIR_BIN)/*_*/fortio* $(BUILD_DIR_BIN)
+	-mv -f $(BUILD_DIR_BIN)/*_*/$(OFFICIAL_EXE)* $(BUILD_DIR_BIN)
 	-rmdir $(BUILD_DIR_BIN)/*_*
-	mv -f $(BUILD_DIR_BIN)/fortio* $(OFFICIAL_DIR)
+	mv -f $(BUILD_DIR_BIN)/$(OFFICIAL_EXE)* $(OFFICIAL_DIR)
 else
 	CGO_ENABLED=0 GOOS=$(GOOS) $(GO_BIN) build -a -ldflags -s -o $(OFFICIAL_BIN) $(OFFICIAL_TARGET)
 endif

README.md

@@ -52,13 +52,13 @@ You can install from source:
 The [releases](https://github.com/fortio/fortio/releases) page has binaries for many OS/architecture combinations (see assets):
 
 ```shell
-curl -L https://github.com/fortio/fortio/releases/download/v1.38.2/fortio-linux_amd64-1.38.2.tgz \
+curl -L https://github.com/fortio/fortio/releases/download/v1.38.3/fortio-linux_amd64-1.38.3.tgz \
  | sudo tar -C / -xvzpf -
 # or the debian package
-wget https://github.com/fortio/fortio/releases/download/v1.38.2/fortio_1.38.2_amd64.deb
-dpkg -i fortio_1.38.2_amd64.deb
+wget https://github.com/fortio/fortio/releases/download/v1.38.3/fortio_1.38.3_amd64.deb
+dpkg -i fortio_1.38.3_amd64.deb
 # or the rpm
-rpm -i https://github.com/fortio/fortio/releases/download/v1.38.2/fortio-1.38.2-1.x86_64.rpm
+rpm -i https://github.com/fortio/fortio/releases/download/v1.38.3/fortio-1.38.3-1.x86_64.rpm
 # and more, see assets in release page
 ```
 
@@ -68,7 +68,7 @@ On a MacOS you can also install Fortio using [Homebrew](https://brew.sh/):
 brew install fortio
 ```
 
-On Windows, download https://github.com/fortio/fortio/releases/download/v1.38.2/fortio_win_1.38.2.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
+On Windows, download https://github.com/fortio/fortio/releases/download/v1.38.3/fortio_win_1.38.3.zip and extract `fortio.exe` to any location, then using the Windows Command Prompt:
 ```
 fortio.exe server
 ```
@@ -116,7 +116,7 @@ Full list of command line flags (`fortio help`):
 <details>
 <!-- use release/updateFlags.sh to update this section -->
 <pre>
-Φορτίο 1.38.2 usage:
+Φορτίο 1.38.3 usage:
     fortio command [flags] target
 where command is one of: load (load testing), server (starts ui, rest api,
  http-echo, redirect, proxies, tcp-echo and grpc ping servers), tcp-echo (only

Webtest.sh

@@ -125,7 +125,7 @@ fi
 PPROF_URL="$BASE_URL/debug/pprof/heap?debug=1"
 $CURL "$PPROF_URL" | grep -i TotalAlloc # should find this in memory profile
 # creating dummy container to hold a volume for test certs due to remote docker bind mount limitation.
-DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v50 sleep 120)
+DOCKERCURLID=$(docker run -d -v $TEST_CERT_VOL --net host --name $DOCKERSECVOLNAME docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c sleep 120)
 # while we have something with actual curl binary do
 # Test for h2c upgrade (#562)
 docker exec $DOCKERSECVOLNAME /usr/bin/curl -v --http2 -m 10 -d foo42 http://localhost:8080/debug | tee >(cat 1>&2) | grep foo42

dflag/dynstring_test.go

@@ -12,7 +12,7 @@ import (
 	"fortio.org/assert"
 )
 
-const notifierTimeout = 50 * time.Millisecond
+const notifierTimeout = 100 * time.Millisecond
 
 func TestDynString_SetAndGet(t *testing.T) {
 	set := flag.NewFlagSet("foobar", flag.ContinueOnError)

echosrv/echo.go

@@ -37,7 +37,7 @@ var (
 func main() {
 	flag.Parse()
 	if len(os.Args) >= 2 && strings.Contains(os.Args[1], "version") {
-		fmt.Println(version.Long())
+		fmt.Println(version.Full())
 		os.Exit(0)
 	}
 	if _, addr := fhttp.Serve(*port, *debugPath); addr == nil {

go.mod

@@ -3,7 +3,7 @@ module fortio.org/fortio
 go 1.18
 
 require (
-	fortio.org/assert v1.1.0
+	fortio.org/assert v1.1.2
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.3.0

go.sum

@@ -1,5 +1,5 @@
-fortio.org/assert v1.1.0 h1:AEkX3WzLx4Qsvgg+HyZTp9wHKo0lr1ZcAylZ2YJgGYc=
-fortio.org/assert v1.1.0/go.mod h1:039mG+/iYDPO8Ibx8TrNuJCm2T2SuhwRI3uL9nHTTls=
+fortio.org/assert v1.1.2 h1:t6WGDqPD5VFrUvx30U0+3mgXXcoPonrdKqt0vfJHn8E=
+fortio.org/assert v1.1.2/go.mod h1:039mG+/iYDPO8Ibx8TrNuJCm2T2SuhwRI3uL9nHTTls=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=

release/Dockerfile.in

@@ -1,5 +1,5 @@
 # Concatenated after ../Dockerfile to create the tgz
-FROM docker.io/fortio/fortio.build:v50 as stage
+FROM docker.io/fortio/fortio.build:v50@sha256:fe69c193d8ad40eb0d791984881f3678aead02660b8e3468c757f717892ada4c as stage
 ARG archs="amd64 arm64 ppc64le s390x"
 ENV archs=${archs}
 # Build image defaults to build user, switch back to root for

release/README.md

@@ -29,8 +29,8 @@
 
 Update [../Dockerfile.build](../Dockerfile.build)
 
-Edit the `BUILD_IMAGE_TAG := v5` line in the Makefile, set it to `v6`
-for instance (replace `v6` by whichever is the next one at the time)
+Edit the `BUILD_IMAGE_TAG := v50@sha...` line in the Makefile, set it to `v51`
+for instance (replace `v50` by whichever is the next one at the time and temporarily remove the SHA part)
 
 run
 
@@ -40,7 +40,9 @@ make update-build-image
 
 Make sure it gets successfully pushed to the fortio registry (requires org access)
 
-run
+Then do `make build-image-sha` to get the new image sha to replace/put in `BUILD_IMAGE_TAG` line of the Makefile
+
+Then run
 
 ```Shell
 make update-build-image-tag SED=gsed