Merge pull request #2164 from crosbymichael/density

Add density stress test

cmd/containerd-stress/density.go

@@ -0,0 +1,232 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/oci"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var densityCommand = cli.Command{
+	Name:  "density",
+	Usage: "stress tests density of containers running on a system",
+	Flags: []cli.Flag{
+		cli.IntFlag{
+			Name:  "count",
+			Usage: "number of containers to run",
+			Value: 10,
+		},
+	},
+	Action: func(cliContext *cli.Context) error {
+		config := config{
+			Address:     cliContext.GlobalString("address"),
+			Duration:    cliContext.GlobalDuration("duration"),
+			Concurrency: cliContext.GlobalInt("concurrent"),
+			Exec:        cliContext.GlobalBool("exec"),
+			JSON:        cliContext.GlobalBool("json"),
+			Metrics:     cliContext.GlobalString("metrics"),
+		}
+		client, err := config.newClient()
+		if err != nil {
+			return err
+		}
+		defer client.Close()
+		ctx := namespaces.WithNamespace(context.Background(), "density")
+		if err := cleanup(ctx, client); err != nil {
+			return err
+		}
+		logrus.Infof("pulling %s", imageName)
+		image, err := client.Pull(ctx, imageName, containerd.WithPullUnpack)
+		if err != nil {
+			return err
+		}
+		logrus.Info("generating spec from image")
+
+		s := make(chan os.Signal, 1)
+		signal.Notify(s, syscall.SIGTERM, syscall.SIGINT)
+
+		spec, err := oci.GenerateSpec(ctx, client,
+			&containers.Container{},
+			oci.WithImageConfig(image),
+			oci.WithProcessArgs("sleep", "120m"),
+		)
+		if err != nil {
+			return err
+		}
+		var (
+			pids  []uint32
+			count = cliContext.Int("count")
+		)
+	loop:
+		for i := 0; i < count+1; i++ {
+			select {
+			case <-s:
+				break loop
+			default:
+				id := fmt.Sprintf("density-%d", i)
+				spec.Linux.CgroupsPath = filepath.Join("/", "density", id)
+
+				c, err := client.NewContainer(ctx, id,
+					containerd.WithNewSnapshot(id, image),
+					containerd.WithSpec(spec, oci.WithUsername("games")),
+				)
+				if err != nil {
+					return err
+				}
+				defer c.Delete(ctx, containerd.WithSnapshotCleanup)
+
+				t, err := c.NewTask(ctx, cio.NullIO)
+				if err != nil {
+					return err
+				}
+				defer t.Delete(ctx, containerd.WithProcessKill)
+				if err := t.Start(ctx); err != nil {
+					return err
+				}
+				pids = append(pids, t.Pid())
+			}
+		}
+		var results struct {
+			PSS             int `json:"pss"`
+			RSS             int `json:"rss"`
+			PSSPerContainer int `json:"pssPerContainer"`
+			RSSPerContainer int `json:"rssPerContainer"`
+		}
+
+		for _, pid := range pids {
+			shimPid, err := getppid(int(pid))
+			if err != nil {
+				return err
+			}
+			smaps, err := getMaps(shimPid)
+			if err != nil {
+				return err
+			}
+			results.RSS += smaps["Rss:"]
+			results.PSS += smaps["Pss:"]
+		}
+		results.PSSPerContainer = results.PSS / count
+		results.RSSPerContainer = results.RSS / count
+
+		return json.NewEncoder(os.Stdout).Encode(results)
+	},
+}
+
+func getMaps(pid int) (map[string]int, error) {
+	f, err := os.Open(fmt.Sprintf("/proc/%d/smaps", pid))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var (
+		smaps = make(map[string]int)
+		s     = bufio.NewScanner(f)
+	)
+	for s.Scan() {
+		var (
+			fields = strings.Fields(s.Text())
+			name   = fields[0]
+		)
+		if len(fields) < 2 {
+			continue
+		}
+		n, err := strconv.Atoi(fields[1])
+		if err != nil {
+			continue
+		}
+		smaps[name] += n
+	}
+	if err := s.Err(); err != nil {
+		return nil, err
+	}
+	return smaps, nil
+}
+
+func getppid(pid int) (int, error) {
+	bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat"))
+	if err != nil {
+		return 0, err
+	}
+	s, err := parseStat(string(bytes))
+	if err != nil {
+		return 0, err
+	}
+	return int(s.PPID), nil
+}
+
+// Stat represents the information from /proc/[pid]/stat, as
+// described in proc(5) with names based on the /proc/[pid]/status
+// fields.
+type Stat struct {
+	// PID is the process ID.
+	PID uint
+
+	// Name is the command run by the process.
+	Name string
+
+	// StartTime is the number of clock ticks after system boot (since
+	// Linux 2.6).
+	StartTime uint64
+	// Parent process ID.
+	PPID uint
+}
+
+func parseStat(data string) (stat Stat, err error) {
+	// From proc(5), field 2 could contain space and is inside `(` and `)`.
+	// The following is an example:
+	// 89653 (gunicorn: maste) S 89630 89653 89653 0 -1 4194560 29689 28896 0 3 146 32 76 19 20 0 1 0 2971844 52965376 3920 18446744073709551615 1 1 0 0 0 0 0 16781312 137447943 0 0 0 17 1 0 0 0 0 0 0 0 0 0 0 0 0 0
+	i := strings.LastIndex(data, ")")
+	if i <= 2 || i >= len(data)-1 {
+		return stat, fmt.Errorf("invalid stat data: %q", data)
+	}
+
+	parts := strings.SplitN(data[:i], "(", 2)
+	if len(parts) != 2 {
+		return stat, fmt.Errorf("invalid stat data: %q", data)
+	}
+
+	stat.Name = parts[1]
+	_, err = fmt.Sscanf(parts[0], "%d", &stat.PID)
+	if err != nil {
+		return stat, err
+	}
+
+	// parts indexes should be offset by 3 from the field number given
+	// proc(5), because parts is zero-indexed and we've removed fields
+	// one (PID) and two (Name) in the paren-split.
+	parts = strings.Split(data[i+2:], " ")
+	fmt.Sscanf(parts[22-3], "%d", &stat.StartTime)
+	fmt.Sscanf(parts[4-3], "%d", &stat.PPID)
+	return stat, nil
+}

cmd/containerd-stress/main.go

@@ -55,6 +55,11 @@ func init() {
 	binarySizeGauge = ns.NewLabeledGauge("binary_size", "Binary size of compiled binaries", metrics.Bytes, "name")
 	errCounter = ns.NewLabeledCounter("errors", "Errors encountered running the stress tests", "err")
 	metrics.Register(ns)
+
+	// set higher ulimits
+	if err := setRlimit(); err != nil {
+		panic(err)
+	}
 }
 
 type run struct {
@@ -150,6 +155,9 @@ func main() {
 		}
 		return nil
 	}
+	app.Commands = []cli.Command{
+		densityCommand,
+	}
 	app.Action = func(context *cli.Context) error {
 		config := config{
 			Address:     context.GlobalString("address"),

cmd/containerd-stress/rlimit_unix.go

@@ -0,0 +1,40 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"syscall"
+)
+
+func setRlimit() error {
+	rlimit := uint64(100000)
+	if rlimit > 0 {
+		var limit syscall.Rlimit
+		if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+			return err
+		}
+		if limit.Cur < rlimit {
+			limit.Cur = rlimit
+			if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

cmd/containerd-stress/rlimit_windows.go

@@ -0,0 +1,23 @@
+// +build windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+func setRlimit() error {
+	return nil
+}