Upgrade to Istio 1.19.0 and implement Gateway API support

.github/workflows/build-and-test-plugin.yaml

@@ -16,10 +16,10 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
+    - uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: 1.21.5
     # There are too many lint errors in current code bases
     # uncomment when we decide what lint should be addressed or ignored.
     # - run: make lint
@@ -46,7 +46,7 @@ jobs:
       - name: "Setup Go"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.21.5
 
       - name: Setup Golang Caches
         uses: actions/cache@v3

.github/workflows/build-and-test.yaml

@@ -13,7 +13,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: 1.19
+        go-version: 1.21.5
     # There are too many lint errors in current code bases
     # uncomment when we decide what lint should be addressed or ignored.
     # - run: make lint
@@ -23,14 +23,10 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Golang Caches
-      uses: actions/cache@v3
+    - name: "Setup Go"
+      uses: actions/setup-go@v3
       with:
-        path: |-
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ github.run_id }}
-        restore-keys: ${{ runner.os }}-go
+        go-version: 1.21.5
 
     - name: Setup Submodule Caches
       uses: actions/cache@v3
@@ -44,7 +40,9 @@ jobs:
 
     # test
     - name: Run Coverage Tests
-      run: GOPROXY="https://proxy.golang.org,direct" make go.test.coverage
+      run: |-
+        go version
+        GOPROXY="https://proxy.golang.org,direct" make go.test.coverage
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:
@@ -65,7 +63,7 @@ jobs:
       - name: "Setup Go"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.21.5
 
       - name: Setup Golang Caches
         uses: actions/cache@v3
@@ -110,7 +108,7 @@ jobs:
     - name: "Setup Go"
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19
+        go-version: 1.21.5
 
     - name: Setup Golang Caches
       uses: actions/cache@v3

.github/workflows/build-image-and-push.yaml

@@ -33,7 +33,7 @@ jobs:
       - name: "Setup Go"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.21.5
 
       - name: Setup Golang Caches
         uses: actions/cache@v3
@@ -109,7 +109,7 @@ jobs:
       - name: "Setup Go"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.21.5
 
       - name: Setup Golang Caches
         uses: actions/cache@v3
@@ -130,6 +130,17 @@ jobs:
           key: ${{ runner.os }}-submodules-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-submodules-new
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Calculate Docker metadata
         id: docker-meta
         uses: docker/metadata-action@v4
@@ -186,7 +197,7 @@ jobs:
       - name: "Setup Go"
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.21.5
 
       - name: Setup Golang Caches
         uses: actions/cache@v3
@@ -207,6 +218,17 @@ jobs:
           key: ${{ runner.os }}-submodules-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-submodules-new
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Calculate Docker metadata
         id: docker-meta
         uses: docker/metadata-action@v4

.gitignore

@@ -1,4 +1,3 @@
-external
 out
 *.out
 *.tgz
@@ -11,6 +10,7 @@ bazel-bin
 bazel-out
 bazel-testlogs
 bazel-wasm-cpp
+external/
 tools/bin/
 helm/**/charts/**.tgz
 target/

.gitmodules

@@ -1,21 +1,22 @@
-[submodule "istio/1.12/api"]
-	path = istio/1.12/api
-	url = https://github.com/istio/api
-[submodule "istio/1.12/istio"]
-	path = istio/1.12/istio
-	url = https://github.com/istio/istio
-[submodule "istio/1.12/client-go"]
-	path = istio/1.12/client-go
-	url = https://github.com/istio/client-go
-[submodule "istio/1.12/pkg"]
-	path = istio/1.12/pkg
-	url = https://github.com/istio/pkg
-[submodule "istio/1.12/proxy"]
-	path = istio/1.12/proxy
-	url = https://github.com/istio/proxy
-[submodule "envoy/1.20/go-control-plane"]
-	path = envoy/1.20/go-control-plane
-	url = https://github.com/envoyproxy/go-control-plane
-[submodule "envoy/1.20/envoy"]
-	path = envoy/1.20/envoy
-	url = https://github.com/envoyproxy/envoy
+[submodule "istio/api"]
+	path = istio/api
+	url = https://github.com/higress-group/api
+[submodule "istio/istio"]
+	path = istio/istio
+	url = https://github.com/higress-group/istio
+	branch = dev-1.19.5
+[submodule "istio/client-go"]
+	path = istio/client-go
+	url = https://github.com/higress-group/client-go
+[submodule "istio/pkg"]
+	path = istio/pkg
+	url = https://github.com/higress-group/pkg
+[submodule "istio/proxy"]
+	path = istio/proxy
+	url = https://github.com/higress-group/proxy
+[submodule "envoy/go-control-plane"]
+	path = envoy/go-control-plane
+	url = https://github.com/higress-group/go-control-plane
+[submodule "envoy/envoy"]
+	path = envoy/envoy
+	url = https://github.com/higress-group/envoy

.licenserc.yaml

@@ -27,8 +27,8 @@ header:
     - 'tools/'
     - 'test/README.md'
     - 'test/README_CN.md'
-    - 'cmd/hgctl/config/testdata/config'
-    - 'pkg/cmd/hgctl/manifests'
+    - 'hgctl/cmd/hgctl/config/testdata/config'
+    - 'hgctl/pkg/manifests'
 
   comment: on-failure
 dependency:

Makefile.core.mk

@@ -1,12 +1,12 @@
 SHELL := /bin/bash -o pipefail
 
-export BASE_VERSION ?= 2022-10-27T19-02-22
+export HIGRESS_BASE_VERSION ?= 2023-07-20T20-50-43
 
 export HUB ?= higress-registry.cn-hangzhou.cr.aliyuncs.com/higress
 
 export CHARTS ?= higress-registry.cn-hangzhou.cr.aliyuncs.com/charts
 
-VERSION_PACKAGE := github.com/alibaba/higress/pkg/cmd/version
+VERSION_PACKAGE := github.com/alibaba/higress/pkg/cmd/lversion
 
 GIT_COMMIT:=$(shell git rev-parse HEAD)
 
@@ -45,13 +45,15 @@ HIGRESS_DOCKER_BUILD_TOP:=${OUT_LINUX}/docker_build
 
 HIGRESS_BINARIES:=./cmd/higress
 
+HGCTL_PROJECT_DIR=./hgctl
 HGCTL_BINARIES:=./cmd/hgctl
 
 $(OUT):
 	@mkdir -p $@
 
 submodule:
 	git submodule update --init
+	git submodule update --remote istio/istio
 
 .PHONY: prebuild
 prebuild: submodule
@@ -80,20 +82,20 @@ $(ARM64_OUT_LINUX)/higress:
 
 .PHONY: build-hgctl
 build-hgctl: prebuild $(OUT)
-	GOPROXY=$(GOPROXY) GOOS=$(GOOS_LOCAL) GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT)/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=$(GOOS_LOCAL) GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh $(OUT)/ $(HGCTL_BINARIES)
 
 .PHONY: build-linux-hgctl
 build-linux-hgctl: prebuild $(OUT)
-	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh $(OUT_LINUX)/ $(HGCTL_BINARIES)
 
 .PHONY: build-hgctl-multiarch
 build-hgctl-multiarch: prebuild $(OUT)
-	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/linux_amd64/ $(HGCTL_BINARIES)
-	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/linux_arm64/ $(HGCTL_BINARIES)
-	GOPROXY=$(GOPROXY) GOOS=darwin GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/darwin_amd64/ $(HGCTL_BINARIES)
-	GOPROXY=$(GOPROXY) GOOS=darwin GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/darwin_arm64/ $(HGCTL_BINARIES)
-	GOPROXY=$(GOPROXY) GOOS=windows GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/windows_amd64/ $(HGCTL_BINARIES)
-	GOPROXY=$(GOPROXY) GOOS=windows GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/windows_arm64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/linux_amd64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/linux_arm64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=darwin GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/darwin_amd64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=darwin GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/darwin_arm64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=windows GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/windows_amd64/ $(HGCTL_BINARIES)
+	GOPROXY=$(GOPROXY) GOOS=windows GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) PROJECT_DIR="$(HGCTL_PROJECT_DIR)" tools/hack/gobuild.sh ./out/windows_arm64/ $(HGCTL_BINARIES)
 # Create targets for OUT_LINUX/binary
 # There are two use cases here:
 # * Building all docker images (generally in CI). In this case we want to build everything at once, so they share work
@@ -128,40 +130,38 @@ docker-build: docker.higress ## Build and push docker images to registry defined
 
 docker-buildx-push: clean-env docker.higress-buildx
 
-docker-build-base:
-	docker buildx build --no-cache --platform linux/amd64,linux/arm64 -t ${HUB}/base:${BASE_VERSION} -f docker/Dockerfile.base . --push
-
 export PARENT_GIT_TAG:=$(shell cat VERSION)
 export PARENT_GIT_REVISION:=$(TAG)
 
-export ENVOY_TAR_PATH:=/home/package/envoy.tar.gz
+export ENVOY_PACKAGE_URL_PATTERN?=https://github.com/alibaba/higress/releases/download/v1.3.4-rc.1/envoy-symbol-ARCH.tar.gz
+
+build-envoy: prebuild
+	./tools/hack/build-envoy.sh
 
 external/package/envoy-amd64.tar.gz:
-#	cd external/proxy; BUILD_WITH_CONTAINER=1  make test_release
-	cd external/package; wget -O envoy-amd64.tar.gz "https://github.com/alibaba/higress/releases/download/v1.3.4-rc.1/envoy-symbol-amd64.tar.gz"
+	cd external/package; wget $(subst ARCH,amd64,${ENVOY_PACKAGE_URL_PATTERN})
 
 external/package/envoy-arm64.tar.gz:
-#	cd external/proxy; BUILD_WITH_CONTAINER=1  make test_release
-	cd external/package; wget -O envoy-arm64.tar.gz "https://github.com/alibaba/higress/releases/download/v1.3.4-rc.1/envoy-symbol-arm64.tar.gz"
+	cd external/package; wget $(subst ARCH,arm64,${ENVOY_PACKAGE_URL_PATTERN})
 
-build-pilot:
-	cd external/istio; rm -rf out/linux_amd64; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 make build-linux
-	cd external/istio; rm -rf out/linux_arm64; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=arm64 BUILD_WITH_CONTAINER=1 make build-linux
+build-pilot: prebuild
+	TARGET_ARCH=amd64 ./tools/hack/build-istio-pilot.sh
+	TARGET_ARCH=arm64 ./tools/hack/build-istio-pilot.sh
 
-build-pilot-local:
-	cd external/istio; rm -rf out/linux_${GOARCH_LOCAL}; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=${GOARCH_LOCAL} BUILD_WITH_CONTAINER=1 make build-linux
+build-pilot-local: prebuild
+	TARGET_ARCH=${TARGET_ARCH} ./tools/hack/build-istio-pilot.sh
 
-build-gateway: prebuild external/package/envoy-amd64.tar.gz external/package/envoy-arm64.tar.gz build-pilot
-	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker
+build-gateway: prebuild
+	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker.buildx
 
 build-gateway-local: prebuild external/package/envoy-amd64.tar.gz external/package/envoy-arm64.tar.gz
-	cd external/istio; rm -rf out/linux_${GOARCH_LOCAL}; GOOS_LOCAL=linux TARGET_OS=linux BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=false DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker
+	TARGET_ARCH=${TARGET_ARCH} DOCKER_TARGETS="docker.proxyv2" ./tools/hack/build-istio-image.sh
 
-build-istio: prebuild build-pilot
-	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker
+build-istio: prebuild
+	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker.buildx
 
 build-istio-local: prebuild
-	cd external/istio; rm -rf out/linux_${GOARCH_LOCAL}; GOOS_LOCAL=linux TARGET_OS=linux BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=false DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker
+	TARGET_ARCH=${TARGET_ARCH} DOCKER_TARGETS="docker.pilot" ./tools/hack/build-istio-image.sh
 
 build-wasmplugins:
 	./tools/hack/build-wasm-plugins.sh