Skip to content

Commit

Permalink
[minor] Release v1.7.0 (#1926)
Browse files Browse the repository at this point in the history
🔖 🤖 Release v1.7.0
Format code with prettier and gofumpt (#1927)
:robot: Automatically update k8s manifests (#1928)
add vald agent qbg impl
fix

Signed-off-by: kpango <kpango@vdaas.org>
  • Loading branch information
kpango committed Jan 24, 2023
1 parent bf7921e commit aab3b20
Show file tree
Hide file tree
Showing 125 changed files with 56,978 additions and 2,391 deletions.
3 changes: 3 additions & 0 deletions .github/actions/detect-docker-image-tags/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ inputs:
description: "image names"
required: false
default: "vdaas/vald-agent-ngt \
vdaas/vald-agent-qbg \
vdaas/vald-agent-sidecar \
vdaas/vald-discoverer-k8s \
vdaas/vald-lb-gateway \
vdaas/vald-manager-index"
Expand All @@ -43,6 +45,7 @@ runs:
run: |
declare -A m=(
["vdaas/vald-agent-ngt"]="agent.image.tag"
["vdaas/vald-agent-qbg"]="agent.image.tag"
["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag"
["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
Expand Down
2 changes: 2 additions & 0 deletions .github/actions/wait-for-docker-image/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ inputs:
description: "image names"
required: false
default: "vdaas/vald-agent-ngt \
vdaas/vald-agent-qbg \
vdaas/vald-agent-sidecar \
vdaas/vald-discoverer-k8s \
vdaas/vald-lb-gateway \
vdaas/vald-manager-index"
Expand Down
153 changes: 153 additions & 0 deletions .github/workflows/dockers-agent-qbg-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Build docker image: agent-qbg"
on:
push:
branches:
- main
tags:
- "*.*.*"
- "v*.*.*"
- "*.*.*-*"
- "v*.*.*-*"
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/dockers-agent-qbg-image.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/qbg/**"
- "cmd/agent/core/qbg/**"
- "dockers/agent/core/qbg/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
pull_request:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/dockers-agent-qbg-image.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/qbg/**"
- "cmd/agent/core/qbg/**"
- "dockers/agent/core/qbg/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
pull_request_target:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/dockers-agent-qbg-image.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/qbg/**"
- "cmd/agent/core/qbg/**"
- "dockers/agent/core/qbg/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
cancel-in-progress: true

jobs:
build:
strategy:
max-parallel: 4
runs-on: ubuntu-latest
if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) }}
steps:
- uses: actions/checkout@v3
- name: set git config
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
- name: Setup QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: all
- name: Setup Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
with:
buildkitd-flags: "--debug"
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASS }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ secrets.PACKAGE_USER }}
password: ${{ secrets.PACKAGE_TOKEN }}
- name: Build and Publish
id: build_and_publish
uses: ./.github/actions/docker-build
with:
target: agent-qbg
builder: ${{ steps.buildx.outputs.name }}
- name: Initialize CodeQL
if: startsWith( github.ref, 'refs/tags/')
uses: github/codeql-action/init@v2
- name: Run vulnerability scanner (table)
if: startsWith( github.ref, 'refs/tags/')
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
format: "table"
- name: Run vulnerability scanner (sarif)
if: startsWith( github.ref, 'refs/tags/')
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to Security tab
if: startsWith( github.ref, 'refs/tags/')
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
slack:
name: Slack notification
needs: build
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/')
steps:
- uses: technote-space/workflow-conclusion-action@v2
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: 8398a7/action-slack@v3
with:
author_name: agent-qbg image build
status: ${{ env.WORKFLOW_CONCLUSION }}
only_mention_fail: channel
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}
4 changes: 4 additions & 0 deletions .github/workflows/dockers-dev-container-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@ jobs:
strategy:
max-parallel: 4
runs-on: ubuntu-latest
<<<<<<< HEAD
if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }}
=======
if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/'))) }}
>>>>>>> feature/agent/qbg
steps:
- uses: actions/checkout@v3
- name: set git config
Expand Down
33 changes: 33 additions & 0 deletions .github/workflows/dockers-image-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,39 @@ jobs:
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
agent-qbg:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: set git config
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
- name: Build the Docker image
id: build_image
run: |
make docker/build/agent-qbg
imagename=`make docker/name/agent-qbg`
docker tag ${imagename} ${imagename}:${{ github.sha }}
echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT
env:
DOCKER_BUILDKIT: 1
- name: Run vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}"
format: "table"
- name: Run vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}"
format: "template"
template: "@/contrib/sarif.tpl"
output: "trivy-results.sarif"
severity: "HIGH,CRITICAL"
- name: Upload Trivy scan results to Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
agent-sidecar:
runs-on: ubuntu-latest
steps:
Expand Down
42 changes: 42 additions & 0 deletions Makefile.d/build.mk
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
## build all binaries
binary/build: \
cmd/agent/core/ngt/ngt \
cmd/agent/core/qbg/qbg \
cmd/agent/sidecar/sidecar \
cmd/discoverer/k8s/discoverer \
cmd/gateway/lb/lb \
Expand Down Expand Up @@ -60,6 +61,42 @@ cmd/agent/core/ngt/ngt: \
$(dir $@)main.go
$@ -version

cmd/agent/core/qbg/qbg: \
qbg/install \
$(GO_SOURCES_INTERNAL) \
$(PBGOS) \
$(shell find $(ROOTDIR)/cmd/agent/core/qbg -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
$(shell find $(ROOTDIR)/pkg/agent/core/qbg $(ROOTDIR)/pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
CFLAGS="$(CFLAGS)" \
CXXFLAGS="$(CXXFLAGS)" \
CGO_ENABLED=1 \
CGO_CXXFLAGS="-g -Ofast -march=native" \
CGO_FFLAGS="-g -Ofast -march=native" \
CGO_LDFLAGS="-g -Ofast -march=native" \
GO111MODULE=on \
GOPRIVATE=$(GOPRIVATE) \
go build \
--ldflags "-w -linkmode 'external' \
-extldflags '-static -fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS)' \
-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
-X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \
-X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \
-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
-buildid=" \
-mod=readonly \
-modcacherw \
-a \
-tags "cgo osusergo netgo static_build" \
-trimpath \
-o $@ \
$(dir $@)main.go
$@ -version

cmd/agent/sidecar/sidecar: \
$(GO_SOURCES_INTERNAL) \
$(PBGOS) \
Expand Down Expand Up @@ -204,6 +241,7 @@ cmd/manager/index/index: \
## build all binaries and zip them
binary/build/zip: \
artifacts/vald-agent-ngt-$(GOOS)-$(GOARCH).zip \
artifacts/vald-agent-qbg-$(GOOS)-$(GOARCH).zip \
artifacts/vald-agent-sidecar-$(GOOS)-$(GOARCH).zip \
artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip \
artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \
Expand All @@ -214,6 +252,10 @@ artifacts/vald-agent-ngt-$(GOOS)-$(GOARCH).zip: cmd/agent/core/ngt/ngt
$(call mkdir, $(dir $@))
zip --junk-paths $@ $<

artifacts/vald-agent-qbg-$(GOOS)-$(GOARCH).zip: cmd/agent/core/qbg/qbg
$(call mkdir, $(dir $@))
zip --junk-paths $@ $<

artifacts/vald-agent-sidecar-$(GOOS)-$(GOARCH).zip: cmd/agent/sidecar/sidecar
$(call mkdir, $(dir $@))
zip --junk-paths $@ $<
Expand Down
17 changes: 17 additions & 0 deletions Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
## build all docker images
docker/build: \
docker/build/agent-ngt \
docker/build/agent-qbg \
docker/build/agent-sidecar \
docker/build/discoverer-k8s \
docker/build/gateway-lb \
Expand Down Expand Up @@ -52,6 +53,22 @@ docker/build/agent-ngt:
--build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \
--build-arg MAINTAINER=$(MAINTAINER)

.PHONY: docker/name/agent-qbg
docker/name/agent-qbg:
@echo "$(ORG)/$(AGENT_IMAGE)"

.PHONY: docker/build/agent-qbg
## build agent-qbg image
docker/build/agent-qbg:
$(DOCKER) build \
$(DOCKER_OPTS) \
-f dockers/agent/core/qbg/Dockerfile \
-t $(ORG)/$(AGENT_IMAGE):$(TAG) . \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \
--build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \
--build-arg MAINTAINER=$(MAINTAINER)

.PHONY: docker/name/agent-sidecar
docker/name/agent-sidecar:
@echo "$(ORG)/$(AGENT_SIDECAR_IMAGE)"
Expand Down
12 changes: 10 additions & 2 deletions Makefile.d/functions.mk
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,17 @@ define proto-code-gen
$(PROTO_PATHS:%=-I %) \
--go_out=$(GOPATH)/src --plugin protoc-gen-go="$(GOPATH)/bin/protoc-gen-go" \
--go-vtproto_out=$(GOPATH)/src --plugin protoc-gen-go-vtproto="$(GOPATH)/bin/protoc-gen-go-vtproto" \
--go-vtproto_opt=features=grpc+marshal+unmarshal+size+pool \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Search.Request \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Insert.MultiRequest \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Insert.Request \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Object.Vector \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Object.Vectors \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Search.ObjectRequest \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Search.Request \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Update.MultiRequest \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Update.Request \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Upsert.MultiRequest \
--go-vtproto_opt=pool=$(GOPKG)/apis/proto/v1/payload.Upsert.Request \
--go-vtproto_opt=features=pool+grpc+marshal+unmarshal+size+equal \
$1
endef

Expand Down
5 changes: 5 additions & 0 deletions Makefile.d/k8s.mk
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,11 @@ $(BINDIR)/telepresence:
env PREFIX=$(BINDIR:%/bin=%) telepresence-$(TELEPRESENCE_VERSION)/install.sh
rm -rf telepresence-$(TELEPRESENCE_VERSION)

.PHONY: telepresence/swap/agent-qbg
## swap agent-qbg deployment using telepresence
telepresence/swap/agent-qbg:
@$(call telepresence,vald-agent-qbg,vdaas/vald-agent-qbg)

.PHONY: telepresence/swap/agent-ngt
## swap agent-ngt deployment using telepresence
telepresence/swap/agent-ngt:
Expand Down
6 changes: 3 additions & 3 deletions apis/grpc/v1/agent/core/agent_vtproto.pb.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (c *agentClient) CreateAndSaveIndex(ctx context.Context, in *payload.Contro
}

func (c *agentClient) IndexInfo(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Count, error) {
out := new(payload.Info_Index_Count)
out := payload.Info_Index_CountFromVTPool()
err := c.cc.Invoke(ctx, "/core.v1.Agent/IndexInfo", in, out, opts...)
if err != nil {
return nil, err
Expand Down Expand Up @@ -141,7 +141,7 @@ func RegisterAgentServer(s grpc.ServiceRegistrar, srv AgentServer) {
}

func _Agent_CreateIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(payload.Control_CreateIndexRequest)
in := payload.Control_CreateIndexRequestFromVTPool()
if err := dec(in); err != nil {
return nil, err
}
Expand Down Expand Up @@ -177,7 +177,7 @@ func _Agent_SaveIndex_Handler(srv interface{}, ctx context.Context, dec func(int
}

func _Agent_CreateAndSaveIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(payload.Control_CreateIndexRequest)
in := payload.Control_CreateIndexRequestFromVTPool()
if err := dec(in); err != nil {
return nil, err
}
Expand Down

0 comments on commit aab3b20

Please sign in to comment.