Skip to content

Commit

Permalink
perf: Add full program benchmark for kustomize build
Browse files Browse the repository at this point in the history
This change introduces a benchmarking test that constructs a
complete kustomization tree using various features of Kustomize.

This update aims to address several objectives:
* Demonstrating current performance challenges in Kustomize in a reproducible manner.
* Evaluating the effects of performance enhancements.
* Guarding against potential performance setbacks and inadvertent quadratic behavior in the future.
* Considering the possibility of incorporating profile-guided optimization (PGO) in future iterations.

Usage:

    go test -run=x -bench=BenchmarkBuild ./kustomize/commands/build

    # sigs.k8s.io/kustomize/kustomize/v5/commands/build.test
    pkg: sigs.k8s.io/kustomize/kustomize/v5/commands/build
    BenchmarkBuild-8   	       1	8523677542 ns/op
    PASS
    ok  	sigs.k8s.io/kustomize/kustomize/v5/commands/build	8.798s

*Currently*, this benchmark requires 3000 seconds to run on my machine.
In order to run it on master today, you need to add `-timeout=30m` to the
`go test` command.

The dataset size was chosen because I believe it represents a real workload
which we could get a runtime of less than 10 seconds.

Updates #5084

Notes on PGO:

Real-life profiles would be better, but creating one based on a benchmark should not hurt:

https://go.dev/doc/pgo#collecting-profiles

> Will PGO with an unrepresentative profile make my program slower than no PGO?
> It should not. While a profile that is not representative of production behavior will result in optimizations in cold parts of the application, it should not make hot parts of the application slower. If you encounter a program where PGO results in worse performance than disabling PGO, please file an issue at https://go.dev/issue/new.

Collecting a profile:

    go test -cpuprofile cpu1.pprof -run=^$ -bench ^BenchmarkBuild$ sigs.k8s.io/kustomize/kustomize/v5/commands/build

    go build -pgo=./cpu1.pprof  -o kust-pgo ./kustomize
    go build -o kust-nopgo ./kustomize

Compare PGO and non-PGO-builds:

    ./kust-pgo build -o /dev/null testdata/  21.88s user 2.00s system 176% cpu 13.505 total
    ./kust-nopgo build -o /dev/null testdata/  22.76s user 1.98s system 174% cpu 14.170 total
  • Loading branch information
chlunde committed Oct 30, 2023
1 parent e002b49 commit 2c2d808
Showing 1 changed file with 157 additions and 0 deletions.
157 changes: 157 additions & 0 deletions kustomize/commands/build/build_benchmark_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
// Copyright 2023 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package build_test

import (
"bytes"
"fmt"
"path/filepath"
"testing"

. "sigs.k8s.io/kustomize/kustomize/v5/commands/build"
"sigs.k8s.io/kustomize/kyaml/filesys"
)

type GenConfig struct {
fileResources int
resources int
patches int
namespaced bool
namePrefix string
nameSuffix string
commonLabels map[string]string
commonAnnotations map[string]string
}

var genConfig = []GenConfig{

Check failure on line 27 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

genConfig is a global variable (gochecknoglobals)
{
resources: 4,
namePrefix: "foo-",
nameSuffix: "-bar",
commonLabels: map[string]string{
"foo": "bar",
},
commonAnnotations: map[string]string{
"baz": "blatti",
},
},
{
resources: 100,
},
{
resources: 3,
},
{
resources: 2,
namespaced: true,
fileResources: 30,
patches: 10,
},
{
fileResources: 2,
},
}

func makeKustomization(fSys filesys.FileSystem, path, id string, depth int) error {
cfg := genConfig[depth]
fSys.MkdirAll(path)

Check failure on line 58 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

Error return value of `fSys.MkdirAll` is not checked (errcheck)

var buf bytes.Buffer
if cfg.namespaced {
fmt.Fprintf(&buf, "namespace: %s\n", id)
}

if cfg.namePrefix != "" {
fmt.Fprintf(&buf, "namePrefix: %s\n", cfg.namePrefix)
}

if cfg.nameSuffix != "" {
fmt.Fprintf(&buf, "nameSuffix: %s\n", cfg.nameSuffix)
}

if len(cfg.commonLabels) > 0 {
fmt.Fprintf(&buf, "commonLabels:\n")
for k, v := range cfg.commonLabels {
fmt.Fprintf(&buf, " %s: %s\n", k, v)
}
}

if len(cfg.commonAnnotations) > 0 {
fmt.Fprintf(&buf, "commonAnnotations:\n")
for k, v := range cfg.commonAnnotations {
fmt.Fprintf(&buf, " %s: %s\n", k, v)
}
}

if cfg.fileResources > 0 || cfg.resources > 0 {
fmt.Fprintf(&buf, "resources:\n")
for res := 0; res < cfg.fileResources; res++ {
fn := fmt.Sprintf("res%d.yaml", res)
fmt.Fprintf(&buf, " - %v\n", fn)

buf := fmt.Sprintf(`kind: ConfigMap
apiVersion: v1
metadata:
name: %s-%d
labels:
foo: bar
annotations:
baz: blatti
data:
k: v
`, id, res)
fSys.WriteFile(filepath.Join(path, fn), []byte(buf))

Check failure on line 104 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

Error return value of `fSys.WriteFile` is not checked (errcheck)
}

for res := 0; res < cfg.resources; res++ {
fn := fmt.Sprintf("res%d", res)
fmt.Fprintf(&buf, " - %v\n", fn)
if err := makeKustomization(fSys, path+"/"+fn, fmt.Sprintf("%s-%d", id, res), depth+1); err != nil {
return err
}
}
}

if cfg.patches > 0 {
fmt.Fprintf(&buf, "patches:\n")
for res := 0; res < cfg.patches; res++ {
// alternate between json and yaml patches to test both kinds
if res%2 == 0 {
fn := fmt.Sprintf("patch%d.yaml", res)
fmt.Fprintf(&buf, " - path: %v\n", fn)
fSys.WriteFile(filepath.Join(path, fn), []byte(fmt.Sprintf(`kind: ConfigMap

Check failure on line 123 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

Error return value of `fSys.WriteFile` is not checked (errcheck)
apiVersion: v1
metadata:
name: %s-%d
data:
k: v2
`, id, res)))
} else {
fn := fmt.Sprintf("patch%d.json", res)
fmt.Fprintf(&buf, ` - path: %v
target:
version: v1
kind: ConfigMap
name: %s-%d
`, fn, id, res-1)
fSys.WriteFile(filepath.Join(path, fn), []byte(`[{"op": "add", "path": "/data/k2", "value": "3"} ]`))

Check failure on line 138 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

Error return value of `fSys.WriteFile` is not checked (errcheck)
}
}
}

return fSys.WriteFile(filepath.Join(path, "kustomization.yaml"), buf.Bytes())

Check failure on line 143 in kustomize/commands/build/build_benchmark_test.go

View workflow job for this annotation

GitHub Actions / Lint

error returned from interface method should be wrapped: sig: func (sigs.k8s.io/kustomize/kyaml/filesys.FileSystem).WriteFile(path string, data []byte) error (wrapcheck)
}

func BenchmarkBuild(b *testing.B) {
fSys := filesys.MakeFsInMemory()
if err := makeKustomization(fSys, "testdata", "res", 0); err != nil {
b.Fatal(err)
}
b.ResetTimer()
buffy := new(bytes.Buffer)
cmd := NewCmdBuild(fSys, MakeHelp("foo", "bar"), buffy)
if err := cmd.RunE(cmd, []string{"./testdata"}); err != nil {
b.Fatal(err)
}
}

0 comments on commit 2c2d808

Please sign in to comment.