Parametrize benchmark settings.

Signed-off-by: Nashwan Azhari <nazhari@cloudbasesolutions.com>

cmd/critest/cri_test.go

@@ -174,7 +174,6 @@ func TestCRISuite(t *testing.T) {
 		flag.Set("ginkgo.focus", "benchmark")
 		flag.Set("ginkgo.succinct", "true")
 	} else {
-		// Skip benchmark measurements for validation tests.
 		flag.Set("ginkgo.skipMeasurements", "true")
 	}
 	if *parallel > 1 {

pkg/benchmark/benchmark.go

@@ -33,6 +33,15 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+// Transforms a slice of `time.Duration`s into their `int64` nanosecond representations.
+func getNanosecondsForDurations(durations []time.Duration) []int64 {
+	var ns []int64
+	for _, duration := range durations {
+		ns = append(ns, duration.Nanoseconds())
+	}
+	return ns
+}
+
 // TestPerformance checks configuration parameters (specified through flags) and then runs
 // benchmark tests using the Ginkgo runner.
 // If a "report directory" is specified, one or more JUnit test reports will be

pkg/benchmark/container.go

@@ -1,6 +1,3 @@
-//go:build container
-// +build container
-
 /*
 Copyright 2021 The Kubernetes Authors.
 
@@ -20,19 +17,20 @@ limitations under the License.
 package benchmark
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"path"
+
+	"github.com/golang/glog"
 	"github.com/kubernetes-sigs/cri-tools/pkg/framework"
+	. "github.com/onsi/ginkgo"
 	"github.com/onsi/gomega/gmeasure"
 	internalapi "k8s.io/cri-api/pkg/apis"
-	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
-
-	"encoding/json"
-	"fmt"
-	. "github.com/onsi/ginkgo"
-	"io/ioutil"
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
 type ContainerExperimentData struct {
-	CreateContainer, StatusContainer, StopContainer, RemoveContainer, StartContainer string
+	CreateContainer, StatusContainer, StopContainer, RemoveContainer, StartContainer []int64
 }
 
 var _ = framework.KubeDescribe("Container", func() {
@@ -92,18 +90,30 @@ var _ = framework.KubeDescribe("Container", func() {
 				By("delete PodSandbox")
 				rc.RemovePodSandbox(podID)
 
-			}, gmeasure.SamplingConfig{N: 200, NumParallel: 1})
+			}, gmeasure.SamplingConfig{N: framework.TestContext.BenchmarkingParams.ContainersNumber, NumParallel: framework.TestContext.BenchmarkingParams.ContainersNumberParallel})
 
 			data := ContainerExperimentData{
-				CreateContainer: fmt.Sprintf("%v", experiment.Get("CreateContainer").Durations),
-				StatusContainer: fmt.Sprintf("%v", experiment.Get("StatusContainer").Durations),
-				StopContainer:   fmt.Sprintf("%v", experiment.Get("StopContainer").Durations),
-				RemoveContainer: fmt.Sprintf("%v", experiment.Get("RemoveContainer").Durations),
-				StartContainer:  fmt.Sprintf("%v", experiment.Get("StartContainer").Durations),
+				CreateContainer: getNanosecondsForDurations(experiment.Get("CreateContainer").Durations),
+				StartContainer:  getNanosecondsForDurations(experiment.Get("StartContainer").Durations),
+				StatusContainer: getNanosecondsForDurations(experiment.Get("StatusContainer").Durations),
+				StopContainer:   getNanosecondsForDurations(experiment.Get("StopContainer").Durations),
+				RemoveContainer: getNanosecondsForDurations(experiment.Get("RemoveContainer").Durations),
 			}
 
-			file, _ := json.MarshalIndent(data, "", " ")
-			_ = ioutil.WriteFile("c:/experiment_container.json", file, 0644)
+			if framework.TestContext.BenchmarkingOutputDir != "" {
+				filepath := path.Join(framework.TestContext.BenchmarkingOutputDir, "container_benchmark_data.json")
+				data, err := json.MarshalIndent(data, "", " ")
+				if err == nil {
+					err = ioutil.WriteFile(filepath, data, 0644)
+					if err != nil {
+						glog.Errorf("Failed to write container benchmark data: %v", filepath)
+					}
+				} else {
+					glog.Errorf("Failed to serialize benchmark data: %v", err)
+				}
+			} else {
+				glog.Infof("No benchmarking output dir provided, skipping writing benchmarking resulsts.")
+			}
 		})
 
 	})

pkg/benchmark/pod.go

@@ -1,6 +1,3 @@
-//go:build pod
-// +build pod
-
 /*
 Copyright 2021 The Kubernetes Authors.
 
@@ -20,6 +17,9 @@ limitations under the License.
 package benchmark
 
 import (
+	"path"
+
+	"github.com/golang/glog"
 	"github.com/kubernetes-sigs/cri-tools/pkg/framework"
 	. "github.com/onsi/ginkgo"
 
@@ -28,7 +28,6 @@ import (
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 )
 
@@ -37,7 +36,7 @@ const (
 )
 
 type ExperimentData struct {
-	CreatePod, StatusPod, StopPod, RemovePod string
+	CreatePod, StatusPod, StopPod, RemovePod []int64
 }
 
 var _ = framework.KubeDescribe("PodSandbox", func() {
@@ -92,17 +91,29 @@ var _ = framework.KubeDescribe("PodSandbox", func() {
 				stopwatch.Record("RemovePod")
 				framework.ExpectNoError(err, "failed to remove PodSandbox: %v", err)
 
-			}, gmeasure.SamplingConfig{N: 1000, NumParallel: 1})
+			}, gmeasure.SamplingConfig{N: framework.TestContext.BenchmarkingParams.PodsNumber, NumParallel: framework.TestContext.BenchmarkingParams.PodsNumberParallel})
 
 			data := ExperimentData{
-				CreatePod: fmt.Sprintf("%v", experiment.Get("CreatePod").Durations),
-				StatusPod: fmt.Sprintf("%v", experiment.Get("StatusPod").Durations),
-				StopPod:   fmt.Sprintf("%v", experiment.Get("StopPod").Durations),
-				RemovePod: fmt.Sprintf("%v", experiment.Get("RemovePod").Durations),
+				CreatePod: getNanosecondsForDurations(experiment.Get("CreatePod").Durations),
+				StatusPod: getNanosecondsForDurations(experiment.Get("StatusPod").Durations),
+				StopPod:   getNanosecondsForDurations(experiment.Get("StopPod").Durations),
+				RemovePod: getNanosecondsForDurations(experiment.Get("RemovePod").Durations),
 			}
 
-			file, _ := json.MarshalIndent(data, "", " ")
-			_ = ioutil.WriteFile("c:/experiment_old_hcsshim.json", file, 0644)
+			if framework.TestContext.BenchmarkingOutputDir != "" {
+				filepath := path.Join(framework.TestContext.BenchmarkingOutputDir, "pod_benchmark_data.json")
+				data, err := json.MarshalIndent(data, "", " ")
+				if err == nil {
+					err = ioutil.WriteFile(filepath, data, 0644)
+					if err != nil {
+						glog.Errorf("Failed to write container benchmark data: %v", filepath)
+					}
+				} else {
+					glog.Errorf("Failed to serialize benchmark data: %v", err)
+				}
+			} else {
+				glog.Infof("No benchmarking out dir provided, skipping writing benchmarking resulsts.")
+			}
 		})
 	})
 

pkg/framework/test_context.go

@@ -25,7 +25,11 @@ import (
 	"time"
 
 	"github.com/onsi/ginkgo/config"
-	"gopkg.in/yaml.v3"
+)
+
+var (
+	testImagesFilePath       string
+	benchamrkSettingFilePath string
 )
 
 // TestImageList aggregates references to the images used in tests.
@@ -34,6 +38,24 @@ type TestImageList struct {
 	WebServerTestImage        string `yaml:"webServerTestImage"`
 }
 
+// BenchmarkingParamsType is the type of benchmarking-related params.
+type BenchmarkingParamsType struct {
+	// ContainersNumber is the number of Containers to run as part of
+	// the container-related benchmarks.
+	ContainersNumber int `yaml:"containersNumber"`
+
+	// ContainersNumberParallel is the maximum number of container-related benchmarks
+	// to run in parallel.
+	ContainersNumberParallel int `yaml:"containersNumberParallel"`
+
+	// PodsNumber is the number of Pods to run as part of the pod-related benchmarks.
+	PodsNumber int `yaml:"podsNumber"`
+
+	// PodsNumberParallel is the maximum number of pod -related benchmarks
+	// to run in parallel.
+	PodsNumberParallel int `yaml:"podsNumberParallel"`
+}
+
 // TestContextType is the type of test context.
 type TestContextType struct {
 	// Report related settings.
@@ -49,11 +71,11 @@ type TestContextType struct {
 	RuntimeHandler        string
 
 	// Test images-related settings.
-	TestImagesFilePath string
-	TestImageList      TestImageList
+	TestImageList TestImageList
 
-	// Benchmark setting.
-	Number int
+	// Benchmarking settings.
+	BenchmarkingOutputDir string
+	BenchmarkingParams    BenchmarkingParamsType
 
 	// Test configuration.
 	IsLcow bool
@@ -81,7 +103,7 @@ func RegisterFlags() {
 	flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
 	flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
 	flag.StringVar(&TestContext.ImageServiceAddr, "image-endpoint", "", "Image service socket for client to connect.")
-	flag.StringVar(&TestContext.TestImagesFilePath, "test-images-file", "", "Optional path to a YAML file containing references to custom container images to be used in tests.")
+	flag.StringVar(&testImagesFilePath, "test-images-file", "", "Optional path to a YAML file containing references to custom container images to be used in tests.")
 	flag.DurationVar(&TestContext.ImageServiceTimeout, "image-service-timeout", 300*time.Second, "Timeout when trying to connect to image service.")
 
 	svcaddr := "unix:///var/run/dockershim.sock"
@@ -94,7 +116,9 @@ func RegisterFlags() {
 	flag.StringVar(&TestContext.RuntimeServiceAddr, "runtime-endpoint", svcaddr, "Runtime service socket for client to connect.")
 	flag.DurationVar(&TestContext.RuntimeServiceTimeout, "runtime-service-timeout", 300*time.Second, "Timeout when trying to connect to a runtime service.")
 	flag.StringVar(&TestContext.RuntimeHandler, "runtime-handler", "", "Runtime handler to use in the test.")
-	flag.IntVar(&TestContext.Number, "number", 5, "Number of PodSandbox/container in listing benchmark test.")
+
+	flag.StringVar(&benchamrkSettingFilePath, "benchmarking-params-file", "", "Optional path to a YAML file specifying benchmarking configuration options.")
+	flag.StringVar(&TestContext.BenchmarkingOutputDir, "benchmarking-output-dir", "", "Optional path to a directory in which benchmarking data should be placed.")
 
 	if runtime.GOOS == "windows" {
 		flag.BoolVar(&TestContext.IsLcow, "lcow", false, "Run Linux container on Windows tests instead of Windows container tests")
@@ -104,21 +128,25 @@ func RegisterFlags() {
 	flag.StringVar(&TestContext.RegistryPrefix, "registry-prefix", DefaultRegistryPrefix, "A possible registry prefix added to all images, like 'localhost:5000/'")
 }
 
-// Loads the custom images mapping file (if defined) into the TestContextType.
-func (tc TestContextType) LoadCustomImagesFileIntoTestingContext() error {
-	Logf("Testing context container image list file: %s", TestContext.TestImagesFilePath)
-	if TestContext.TestImagesFilePath != "" {
-		fileContent, err := os.ReadFile(TestContext.TestImagesFilePath)
+// Loads any external file-based parameters into the TestContextType.
+func (tc TestContextType) LoadYamlConfigFiles() error {
+	// Attempt to load cusom images file:
+	if testImagesFilePath != "" {
+		err := LoadYamlFile(testImagesFilePath, &TestContext.TestImageList)
 		if err != nil {
-			return fmt.Errorf("error reading '%v' file contents: %v", TestContext.TestImagesFilePath, err)
+			return fmt.Errorf("Error loading custom test images file: %v", err)
 		}
+	}
+	Logf("Testing context container image list: %+v", TestContext.TestImageList)
 
-		err = yaml.Unmarshal(fileContent, &TestContext.TestImageList)
+	// Attempt to load benchmark settings file:
+	if benchamrkSettingFilePath != "" {
+		err := LoadYamlFile(benchamrkSettingFilePath, &TestContext.BenchmarkingParams)
 		if err != nil {
-			return fmt.Errorf("error unmarshalling '%v' YAML file: %v", TestContext.TestImagesFilePath, err)
+			return err
 		}
 	}
+	Logf("Testing context benchmarking params: %+v", TestContext.BenchmarkingParams)
 
-	Logf("Testing context container image list: %+v", TestContext.TestImageList)
 	return nil
 }

pkg/framework/util.go

@@ -18,13 +18,15 @@ package framework
 
 import (
 	"fmt"
+	"os"
 	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/docker/distribution/reference"
 	"github.com/pborman/uuid"
+	"gopkg.in/yaml.v3"
 	internalapi "k8s.io/cri-api/pkg/apis"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
@@ -115,7 +117,7 @@ var _ = BeforeSuite(func() {
 	}
 
 	// Load any custom image definitions:
-	err := TestContext.LoadCustomImagesFileIntoTestingContext()
+	err := TestContext.LoadYamlConfigFiles()
 	if err != nil {
 		panic(err)
 	}
@@ -355,3 +357,20 @@ func PullPublicImage(c internalapi.ImageManagerService, imageName string, podCon
 	ExpectNoError(err, "failed to pull image: %v", err)
 	return id
 }
+
+// LoadYamlFile attempts to load the given YAML file into the given struct.
+func LoadYamlFile(filepath string, obj interface{}) error {
+	Logf("Attempting to load YAML file %q into %+v", filepath, obj)
+	fileContent, err := os.ReadFile(filepath)
+	if err != nil {
+		return fmt.Errorf("error reading %q file contents: %v", filepath, err)
+	}
+
+	err = yaml.Unmarshal(fileContent, obj)
+	if err != nil {
+		return fmt.Errorf("error unmarshalling %q YAML file: %v", filepath, err)
+	}
+
+	Logf("Successfully loaded YAML file %q into %+v", filepath, obj)
+	return nil
+}