From ed3277365e85650b652427eb1132050493d06f3a Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy <84944216+helsaawy@users.noreply.github.com> Date: Fri, 16 Sep 2022 09:17:19 -0400 Subject: [PATCH] Linux GCS tests and benchmarks (#1352) * Added GCS tests and benchmarks Added testing suite that can built and run directly on the Linux uVM by sharing or adding it to the rootfs. It primarily focuses on container (standalone and CRI) management. Signed-off-by: Hamza El-Saawy * PR: rebase, comments, bugs, cleanup, security policy, linting Fixed bug with calling `*hcsv2.Host.GetContainer` instead of `*hcsv2.Host.GetCreatedContainer`. Removed left over comments, added clarifying comments to `assertNumberContainers` and `listContaienrStates` interactions. Reordered namespace and rootfs cleanup. Removed underscore from consts Removed unneeded constants Added flag to test-lcow-uvm script to change boot type from vhd to initrd. Update security policy code to use enforcer. Updated script for changes to `uvmboot` and to use default executable name (`gcs.test`), as produced by `go test -c`. Linting issues: - `switch` to `if` - unused `getContainer()` - unused receivers Signed-off-by: Hamza El-Saawy Signed-off-by: Hamza El-Saawy --- scripts/Test-LCOW-UVM.ps1 | 148 ++++++++++++++ test/gcs/container_bench_test.go | 226 +++++++++++++++++++++ test/gcs/container_test.go | 237 ++++++++++++++++++++++ test/gcs/cri_bench_test.go | 242 +++++++++++++++++++++++ test/gcs/cri_test.go | 98 +++++++++ test/gcs/doc.go | 3 + test/gcs/helper_conn_test.go | 319 ++++++++++++++++++++++++++++++ test/gcs/helper_container_test.go | 282 ++++++++++++++++++++++++++ test/gcs/helper_cri_test.go | 134 +++++++++++++ test/gcs/main_test.go | 167 ++++++++++++++++ test/go.mod | 12 +- test/go.sum | 10 + test/internal/oci/cri.go | 2 +- 13 files changed, 1877 insertions(+), 3 deletions(-) create mode 100644 scripts/Test-LCOW-UVM.ps1 create mode 100644 test/gcs/container_bench_test.go create mode 100644 test/gcs/container_test.go create mode 100644 test/gcs/cri_bench_test.go create mode 100644 test/gcs/cri_test.go create mode 100644 test/gcs/doc.go create mode 100644 test/gcs/helper_conn_test.go create mode 100644 test/gcs/helper_container_test.go create mode 100644 test/gcs/helper_cri_test.go create mode 100644 test/gcs/main_test.go diff --git a/scripts/Test-LCOW-UVM.ps1 b/scripts/Test-LCOW-UVM.ps1 new file mode 100644 index 0000000000..fbea64e2f5 --- /dev/null +++ b/scripts/Test-LCOW-UVM.ps1 @@ -0,0 +1,148 @@ +#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Action Bench -BootFilesPath C:\ContainerPlat\LinuxBootFiles\ -MountGCSTest -Count 2 -Benchtime '3s' +# benchstat via `go install golang.org/x/perf/cmd/benchstat@latest` + +[CmdletBinding()] +param ( + [ValidateSet('Test', 'Bench', 'List', 'Shell')] + [alias('a')] + [string] + $Action = 'Bench', + + [string] + $Note = '', + + # test parameters + [int] + $Count = 1, + + [string] + $BenchTime = '5s', + + [string] + $Timeout = '10m', + + [alias('tv')] + [switch] + $TestVerbose, + + [string] + $Run = '', + + [string] + $CodePath = '.', + + [string] + $OutDirectory = '.\test\results', + + # uvm parameters + + [string] + $BootFilesPath = 'C:\ContainerPlat\LinuxBootFiles', + + [ValidateSet('vhd', 'initrd')] + [string] + $BootFSType = 'vhd', + + [switch] + $DisableTimeSync, + + # gcs test/container options + + [string] + $ContainerRootFSMount = '/run/rootfs', + + [string] + $ContainerRootFSPath = (Join-Path $BootFilesPath 'rootfs.vhd'), + + [string] + $GCSTestMount = '/run/bin', + + [string] + $GCSTestPath = '.\bin\test\gcs.test', + + [switch] + $MountGCSTest, + + [string] + $Feature = '' +) + +Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force + +$CodePath = Resolve-Path $CodePath +$OutDirectory = Resolve-Path $OutDirectory +$BootFilesPath = Resolve-Path $BootFilesPath +$ContainerRootFSPath = Resolve-Path $ContainerRootFSPath +$GCSTestPath = Resolve-Path $GCSTestPath + +$shell = ( $Action -eq 'Shell' ) + +if ( $shell ) { + $cmd = 'ash' +} else { + $date = Get-Date + $waitfiles = "$ContainerRootFSMount" + $gcspath = 'gcs.test' + if ( $MountGCSTest ) { + $waitfiles += ",$GCSTestMount" + $gcspath = "$GCSTestMount/gcs.test" + } + + $pre = "wait-paths -p $waitfiles -t 5 ; " + ` + 'echo nproc: `$(nproc) ; ' + ` + 'echo kernel: `$(uname -a) ; ' + ` + 'echo gcs.commit: `$(cat /info/gcs.commit 2>/dev/null) ; ' + ` + 'echo gcs.branch: `$(cat /info/gcs.branch 2>/dev/null) ; ' + ` + 'echo tar.date: `$(cat /info/tar.date 2>/dev/null) ; ' + ` + 'echo image.name: `$(cat /info/image.name 2>/dev/null) ; ' + ` + 'echo build.date: `$(cat /info/build.date 2>/dev/null) ; ' + + $testcmd, $out = New-TestCommand ` + -Action $Action ` + -Path $gcspath ` + -Name gcstest ` + -OutDirectory $OutDirectory ` + -Date $date ` + -Note $Note ` + -TestVerbose:$TestVerbose ` + -Count $Count ` + -BenchTime $BenchTime ` + -Timeout $Timeout ` + -Run $Run ` + -Feature $Feature ` + -Verbose:$Verbose + + $testcmd += " `'-rootfs-path=$ContainerRootFSMount`' " + $cmd = $pre + $testcmd +} + +$boot = '.\bin\tool\uvmboot.exe -gcs lcow ' + ` + '-fwd-stdout -fwd-stderr -output-handling stdout ' + ` + "-boot-files-path $BootFilesPath " + ` + "-root-fs-type $BootFSType " + ` + '-kernel-file vmlinux ' + ` + "-mount-scsi `"$ContainerRootFSPath,$ContainerRootFSMount`" " + +if ( $MountGCSTest ) { + $boot += "-share `"$GCSTestPath,$GCSTestMount`" " +} + +if ( $DisableTimeSync ) { + $boot += ' -disable-time-sync ' +} + +if ( $shell ) { + $boot += ' -t ' +} + +$boot += " -exec `"$cmd`" " + +Invoke-TestCommand ` + -TestCmd $boot ` + -TestCmdPreamble $testcmd ` + -OutputFile (&{ if ( $Action -ne 'Shell' ) { $out } }) ` + -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` + -Preamble ` + -Date $Date ` + -Note $Note ` + -Verbose:$Verbose diff --git a/test/gcs/container_bench_test.go b/test/gcs/container_bench_test.go new file mode 100644 index 0000000000..160b8f6ee6 --- /dev/null +++ b/test/gcs/container_bench_test.go @@ -0,0 +1,226 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + cri_util "github.com/containerd/containerd/pkg/cri/util" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +func BenchmarkContainerCreate(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := b.Name() + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + + s := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + oci.WithRootFSPath(rootfs), + oci.WithProcessArgs("/bin/sh", "-c", tailNull), + )..., + ) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + deleteContainer(ctx, b, c) + removeContainer(ctx, b, host, id) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkContainerStart(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerKill(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +// benchmark container create through wait until exit. +func BenchmarkContainerCompleteExit(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host, oci.WithProcessArgs("/bin/sh", "-c", "true")) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + e, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtGracefulExit, prot.NtUnexpectedExit: + default: + b.Fatalf("container exit was %s", n) + } + + if e != 0 { + b.Fatalf("container exit code was %d", e) + } + + killContainer(ctx, b, c) + c.Wait() + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerCompleteKill(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerExec(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, _ := getTestState(ctx, b) + + id := b.Name() + c := createStandaloneContainer(ctx, b, host, id) + ip := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps := testoci.CreateLinuxSpec(ctx, b, id, + oci.WithDefaultPathEnv, + oci.WithProcessArgs("/bin/sh", "-c", "true"), + ).Process + + b.StartTimer() + p := execProcess(ctx, b, c, ps, stdio.ConnectionSettings{}) + exch, dch := p.Wait() + if e := <-exch; e != 0 { + b.Errorf("process exited with error code %d", e) + } + b.StopTimer() + + dch <- true + close(dch) + } + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, ip, true) + cleanupContainer(ctx, b, host, c) +} + +func standaloneContainerRequest( + ctx context.Context, + t testing.TB, + host *hcsv2.Host, + extra ...oci.SpecOpts, +) (string, *prot.VMHostedContainerSettingsV2, func()) { + ctx = namespaces.WithNamespace(ctx, testoci.DefaultNamespace) + id := t.Name() + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, t, host, id) + + opts := testoci.DefaultLinuxSpecOpts(id, + oci.WithRootFSPath(rootfs), + oci.WithProcessArgs("/bin/sh", "-c", tailNull), + ) + opts = append(opts, extra...) + s := testoci.CreateLinuxSpec(ctx, t, id, opts...) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + f := func() { + unmountRootfs(ctx, t, scratch) + } + + return id, r, f +} diff --git a/test/gcs/container_test.go b/test/gcs/container_test.go new file mode 100644 index 0000000000..6a68e55c33 --- /dev/null +++ b/test/gcs/container_test.go @@ -0,0 +1,237 @@ +//go:build linux + +package gcs + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "golang.org/x/sync/errgroup" + + "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// +// tests for operations on standalone containers +// + +// todo: using `oci.WithTTY` for IO tests is broken and hangs + +func TestContainerCreate(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + c := createStandaloneContainer(ctx, t, host, id) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + + p := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, c) + waitContainer(ctx, t, c, p, true) + }) + + assertNumberContainers(ctx, t, rtime, 1) + css := listContainerStates(ctx, t, rtime) + // guaranteed by assertNumberContainers that css will only have 1 element + cs := css[0] + if cs.ID != id { + t.Fatalf("got id %q, wanted %q", cs.ID, id) + } + pid := p.Pid() + if pid != cs.Pid { + t.Fatalf("got pid %d, wanted %d", pid, cs.Pid) + } + if cs.Status != "running" { + t.Fatalf("got status %q, wanted %q", cs.Status, "running") + } +} + +func TestContainerDelete(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + + c := createStandaloneContainer(ctx, t, host, id, + oci.WithProcessArgs("/bin/sh", "-c", "true"), + ) + + p := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + waitContainer(ctx, t, c, p, false) + + cleanupContainer(ctx, t, host, c) + + _, err := host.GetCreatedContainer(id) + if hr, herr := gcserr.GetHresult(err); herr != nil || hr != gcserr.HrVmcomputeSystemNotFound { + t.Fatalf("GetCreatedContainer returned %v, wanted %v", err, gcserr.HrVmcomputeSystemNotFound) + } + assertNumberContainers(ctx, t, rtime, 0) +} + +// +// IO +// + +var ioTests = []struct { + name string + args []string + in string + want string +}{ + { + name: "true", + args: []string{"/bin/sh", "-c", "true"}, + want: "", + }, + { + name: "echo", + args: []string{"/bin/sh", "-c", `echo -n "hi y'all"`}, + want: "hi y'all", + }, + { + name: "tee", + args: []string{"/bin/sh", "-c", "tee"}, + in: "are you copying me?", + want: "are you copying me?", + }, +} + +func TestContainerIO(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + id := strings.ReplaceAll(t.Name(), "/", "") + + con := newConnectionSettings(tt.in != "", true, true) + f := createStdIO(ctx, t, con) + + var outStr, errStr string + g := &errgroup.Group{} + g.Go(func() error { + outStr = f.ReadAllOut(ctx, t) + + return nil + }) + g.Go(func() error { + errStr = f.ReadAllErr(ctx, t) + + return nil + }) + + c := createStandaloneContainer(ctx, t, host, id, + oci.WithProcessArgs(tt.args...), + ) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + p := startContainer(ctx, t, c, con) + + f.WriteIn(ctx, t, tt.in) + f.CloseIn(ctx, t) + t.Logf("wrote to stdin: %q", tt.in) + + waitContainer(ctx, t, c, p, false) + + _ = g.Wait() + t.Logf("stdout: %q", outStr) + t.Logf("stderr: %q", errStr) + + if errStr != "" { + t.Fatalf("container returned error %q", errStr) + } + if outStr != tt.want { + t.Fatalf("container returned %q; wanted %q", outStr, tt.want) + } + }) + } + + assertNumberContainers(ctx, t, rtime, 0) +} + +func TestContainerExec(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + c := createStandaloneContainer(ctx, t, host, id) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + + ip := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, c) + waitContainer(ctx, t, c, ip, true) + }) + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + ps := testoci.CreateLinuxSpec(ctx, t, id, + oci.WithDefaultPathEnv, + oci.WithProcessArgs(tt.args...), + ).Process + con := newConnectionSettings(tt.in != "", true, true) + f := createStdIO(ctx, t, con) + + var outStr, errStr string + g := &errgroup.Group{} + g.Go(func() error { + outStr = f.ReadAllOut(ctx, t) + + return nil + }) + g.Go(func() error { + errStr = f.ReadAllErr(ctx, t) + + return nil + }) + + // OS pipes can lose some data, so sleep a bit to let ReadAll* kick off + time.Sleep(10 * time.Millisecond) + + p := execProcess(ctx, t, c, ps, con) + f.WriteIn(ctx, t, tt.in) + f.CloseIn(ctx, t) + t.Logf("wrote std in: %q", tt.in) + + exch, _ := p.Wait() + if i := <-exch; i != 0 { + t.Errorf("process exited with error code %d", i) + } + + _ = g.Wait() + t.Logf("stdout: %q", outStr) + t.Logf("stderr: %q", errStr) + + if errStr != "" { + t.Fatalf("exec returned error %q", errStr) + } else if outStr != tt.want { + t.Fatalf("process returned %q; wanted %q", outStr, tt.want) + } + }) + } +} diff --git a/test/gcs/cri_bench_test.go b/test/gcs/cri_bench_test.go new file mode 100644 index 0000000000..4f475e691b --- /dev/null +++ b/test/gcs/cri_bench_test.go @@ -0,0 +1,242 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + cri_util "github.com/containerd/containerd/pkg/cri/util" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" +) + +func BenchmarkCRISanboxCreate(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRISandboxStart(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRISandboxKill(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRIWorkload(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + sid := b.Name() + sScratch, sRootfs := mountRootfs(ctx, b, host, sid) + b.Cleanup(func() { + unmountRootfs(ctx, b, sScratch) + }) + nns := sid + createNamespace(ctx, b, nns) + b.Cleanup(func() { + removeNamespace(ctx, b, nns) + }) + + sSpec := sandboxSpec(ctx, b, "test-bench-sandbox", sid, nns, sRootfs) + sandbox := createContainer(ctx, b, host, sid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: sScratch, + OCISpecification: sSpec, + }) + b.Cleanup(func() { + cleanupContainer(ctx, b, host, sandbox) + }) + + sandboxInit := startContainer(ctx, b, sandbox, stdio.ConnectionSettings{}) + b.Cleanup(func() { + killContainer(ctx, b, sandbox) + waitContainer(ctx, b, sandbox, sandboxInit, true) + }) + + b.Run("Create", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + // edge case where workload container transitions from "created" to "paused" + // then "stopped" + waitContainerRaw(c, c.InitProcess()) + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) + + b.Run("Start", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) + + b.Run("Kill", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %q, expected %q", n, prot.NtForcedExit) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) +} + +func workloadContainerRequest( + ctx context.Context, + t testing.TB, + host *hcsv2.Host, + sid string, + spid uint32, + nns string, +) (string, *prot.VMHostedContainerSettingsV2, func()) { + id := sid + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, t, host, id) + spec := containerSpec(ctx, t, + sid, + spid, + "test-bench-container", + id, + []string{"/bin/sh", "-c"}, + []string{tailNull}, + "/", + nns, + rootfs, + ) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + f := func() { + unmountRootfs(ctx, t, scratch) + } + + return id, r, f +} diff --git a/test/gcs/cri_test.go b/test/gcs/cri_test.go new file mode 100644 index 0000000000..33f3a374f7 --- /dev/null +++ b/test/gcs/cri_test.go @@ -0,0 +1,98 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/stdio" +) + +// +// tests for operations on sandbox and workload (CRI) containers +// + +// TestCRILifecycle tests the entire CRI container workflow: creating and starting a CRI sandbox +// pod container, creating, starting, and stopping a workload container within that pod, asserting +// that all operations were successful, and mounting (and unmounting) rootfs's as necessary. +func TestCRILifecycle(t *testing.T) { + requireFeatures(t, featureCRI) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + sid := t.Name() + scratch, rootfs := mountRootfs(ctx, t, host, sid) + t.Cleanup(func() { + unmountRootfs(ctx, t, scratch) + }) + createNamespace(ctx, t, sid) + t.Cleanup(func() { + removeNamespace(ctx, t, sid) + }) + + spec := sandboxSpec(ctx, t, "test-sandbox", sid, sid, rootfs) + sandbox := createContainer(ctx, t, host, sid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + }) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, sandbox) + assertNumberContainers(ctx, t, rtime, 0) + }) + + assertNumberContainers(ctx, t, rtime, 1) + assertContainerState(ctx, t, rtime, sid, "created") + + sandboxInit := startContainer(ctx, t, sandbox, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, sandbox) + waitContainer(ctx, t, sandbox, sandboxInit, true) + }) + + assertContainerState(ctx, t, rtime, sid, "running") + cs := getContainerState(ctx, t, rtime, sid) + pid := sandboxInit.Pid() + if pid != cs.Pid { + t.Fatalf("got sandbox pid %d, wanted %d", pid, cs.Pid) + } + + cid := "container" + sid + cscratch, crootfs := mountRootfs(ctx, t, host, cid) + t.Cleanup(func() { + unmountRootfs(ctx, t, cscratch) + }) + + cspec := containerSpec(ctx, t, sid, uint32(sandboxInit.Pid()), "test-container", cid, + []string{"/bin/sh", "-c"}, + []string{tailNull}, + "/", sid, crootfs, + ) + workload := createContainer(ctx, t, host, cid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: cscratch, + OCISpecification: cspec, + }) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, workload) + assertNumberContainers(ctx, t, rtime, 1) + }) + + assertNumberContainers(ctx, t, rtime, 2) + assertContainerState(ctx, t, rtime, cid, "created") + + workloadInit := startContainer(ctx, t, workload, stdio.ConnectionSettings{}) + assertContainerState(ctx, t, rtime, cid, "running") + t.Cleanup(func() { + killContainer(ctx, t, workload) + waitContainer(ctx, t, workload, workloadInit, true) + }) + + cs = getContainerState(ctx, t, rtime, cid) + pid = workloadInit.Pid() + if pid != cs.Pid { + t.Fatalf("got sandbox pid %d, wanted %d", pid, cs.Pid) + } +} diff --git a/test/gcs/doc.go b/test/gcs/doc.go new file mode 100644 index 0000000000..19d05fcc86 --- /dev/null +++ b/test/gcs/doc.go @@ -0,0 +1,3 @@ +// This package builds a test binary that can be run directly on uVM guest, +// alongside ./cmd/gcs, for testing and benchmarking. +package gcs diff --git a/test/gcs/helper_conn_test.go b/test/gcs/helper_conn_test.go new file mode 100644 index 0000000000..a80ee5d7f3 --- /dev/null +++ b/test/gcs/helper_conn_test.go @@ -0,0 +1,319 @@ +//go:build linux + +package gcs + +import ( + "context" + "errors" + "io" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/Microsoft/hcsshim/internal/guest/transport" +) + +const ( + dialRetries = 4 + dialWait = 50 * time.Millisecond +) + +// port numbers to assign to connections. +var ( + _pipes sync.Map + _portNumber uint32 = 1 +) + +type PipeTransport struct{} + +var _ transport.Transport = &PipeTransport{} + +func (*PipeTransport) Dial(port uint32) (c transport.Connection, err error) { + for i := 0; i < dialRetries; i++ { + c, err = getFakeSocket(port) + + if errors.Is(err, unix.ENOENT) { + // socket hasn't been created + time.Sleep(dialWait) + continue + } + break + } + if err != nil { + return nil, err + } + + logrus.Debugf("dialed port %d", port) + return c, nil +} + +type fakeIO struct { + stdin, stdout, stderr *fakeSocket +} + +func createStdIO(ctx context.Context, t testing.TB, con stdio.ConnectionSettings) *fakeIO { + f := &fakeIO{} + if con.StdIn != nil { + f.stdin = newFakeSocket(ctx, t, *con.StdIn, "stdin") + } + if con.StdOut != nil { + f.stdout = newFakeSocket(ctx, t, *con.StdOut, "stdout") + } + if con.StdErr != nil { + f.stderr = newFakeSocket(ctx, t, *con.StdErr, "stderr") + } + + return f +} + +func (f *fakeIO) WriteIn(_ context.Context, t testing.TB, s string) { + if f.stdin == nil { + return + } + + b := []byte(s) + n := len(b) + + nn, err := f.stdin.Write(b) + if err != nil { + t.Helper() + t.Errorf("write to std in: %v", err) + } + if n != nn { + t.Helper() + t.Errorf("only wrote %d bytes, expected %d", nn, n) + } +} + +func (f *fakeIO) CloseIn(_ context.Context, t testing.TB) { + if f.stdin == nil { + return + } + + if err := f.stdin.CloseWrite(); err != nil { + t.Helper() + t.Errorf("close write std in: %v", err) + } + + if err := f.stdin.Close(); err != nil { + t.Helper() + t.Errorf("close std in: %v", err) + } +} + +func (f *fakeIO) ReadAllOut(ctx context.Context, t testing.TB) string { + return f.stdout.readAll(ctx, t) +} + +func (f *fakeIO) ReadAllErr(ctx context.Context, t testing.TB) string { + return f.stderr.readAll(ctx, t) +} + +type fakeSocket struct { + id uint32 + n string + ch chan struct{} // closed when dialed (via getFakeSocket) + r, w *os.File +} + +var _ transport.Connection = &fakeSocket{} + +func newFakeSocket(_ context.Context, t testing.TB, id uint32, n string) *fakeSocket { + t.Helper() + + _, ok := _pipes.Load(id) + if ok { + t.Fatalf("socket %d already exits", id) + } + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("could not create socket: %v", err) + } + + s := &fakeSocket{ + id: id, + n: n, + r: r, + w: w, + ch: make(chan struct{}), + } + _pipes.Store(id, s) + + return s +} + +func getFakeSocket(id uint32) (*fakeSocket, error) { + f, ok := _pipes.Load(id) + if !ok { + return nil, unix.ENOENT + } + + s := f.(*fakeSocket) + select { + case <-s.ch: + default: + close(s.ch) + } + + return s, nil +} + +func (s *fakeSocket) Read(b []byte) (int, error) { + <-s.ch + return s.r.Read(b) +} + +func (s *fakeSocket) Write(b []byte) (int, error) { + <-s.ch + return s.w.Write(b) +} + +func (s *fakeSocket) Close() (err error) { + if _, ok := _pipes.LoadAndDelete(s.id); ok { + return nil + } + + err = s.r.Close() + if err := s.w.Close(); err != nil { + return err + } + + return err +} + +func (s *fakeSocket) CloseRead() error { + return s.r.Close() +} + +func (s *fakeSocket) CloseWrite() error { + return s.w.Close() +} + +func (*fakeSocket) File() (*os.File, error) { + return nil, errors.New("fakeSocket does not support File()") +} + +func (s *fakeSocket) readAll(ctx context.Context, t testing.TB) string { + return string(s.readAllByte(ctx, t)) +} + +func (s *fakeSocket) readAllByte(ctx context.Context, t testing.TB) (b []byte) { + if s == nil { + return nil + } + + var err error + ch := make(chan struct{}) + go func() { + defer close(ch) + b, err = io.ReadAll(s) + }() + + select { + case <-ch: + if err != nil { + t.Helper() + t.Errorf("read all %s: %v", s.n, err) + } + case <-ctx.Done(): + t.Helper() + t.Errorf("read all %s context cancelled: %v", s.n, ctx.Err()) + } + + return b +} + +func newConnectionSettings(in, out, err bool) stdio.ConnectionSettings { + c := stdio.ConnectionSettings{} + + if in { + p := nextPortNumber() + c.StdIn = &p + } + if out { + p := nextPortNumber() + c.StdOut = &p + } + if err { + p := nextPortNumber() + c.StdErr = &p + } + + return c +} + +func nextPortNumber() uint32 { + return atomic.AddUint32(&_portNumber, 2) +} + +func TestFakeSocket(t *testing.T) { + ctx := context.Background() + tpt := getTransport() + + ch := make(chan struct{}) + chs := make(chan struct{}) + con := newConnectionSettings(true, true, false) + + // host + f := createStdIO(ctx, t, con) + + var err error + go func() { // guest + defer close(ch) + + var cin, cout transport.Connection + cin, err = tpt.Dial(*con.StdIn) + if err != nil { + t.Logf("dial error %v", err) + + return + } + defer cin.Close() + + cout, err = tpt.Dial(*con.StdOut) + if err != nil { + t.Logf("dial error %v", err) + + return + } + defer cout.Close() + + close(chs) + var b []byte + b, err = io.ReadAll(cin) + if err != nil { + t.Logf("read all error: %v", err) + + return + } + t.Logf("guest read %s", b) + + _, err = cout.Write(b) + _ = cout.CloseWrite() + }() + + <-chs // wait for guest to dial + f.WriteIn(ctx, t, "hello") + f.WriteIn(ctx, t, " world") + f.CloseIn(ctx, t) + t.Logf("host wrote") + + <-ch + t.Logf("go routine closed") + if err != nil { + t.Fatalf("go routine error: %v", err) + } + + s := f.ReadAllOut(ctx, t) + t.Logf("host read %q", s) + if s != "hello world" { + t.Fatalf("got %q, wanted %q", s, "hello world") + } +} diff --git a/test/gcs/helper_container_test.go b/test/gcs/helper_container_test.go new file mode 100644 index 0000000000..1e95bb5ffd --- /dev/null +++ b/test/gcs/helper_container_test.go @@ -0,0 +1,282 @@ +//go:build linux + +package gcs + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/containerd/containerd/namespaces" + ctrdoci "github.com/containerd/containerd/oci" + oci "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/Microsoft/hcsshim/internal/guest/storage" + "github.com/Microsoft/hcsshim/internal/guest/storage/overlay" + "github.com/Microsoft/hcsshim/internal/guestpath" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// todo: autogenerate/fuzz realistic specs + +// +// testing helper functions for generic container management +// + +const tailNull = "tail -f /dev/null" + +// Creates an overlay mount, and then a container using that mount that runs until stopped. +// The container is created on its own, and not associated with a sandbox pod, and is therefore not CRI compliant. +// [unmountRootfs] is added to the test cleanup. +func createStandaloneContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, id string, extra ...ctrdoci.SpecOpts) *hcsv2.Container { + ctx = namespaces.WithNamespace(ctx, testoci.DefaultNamespace) + scratch, rootfs := mountRootfs(ctx, t, host, id) + // spec is passed in from containerd and then updated in internal\hcsoci\create.go:CreateContainer() + opts := testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithRootFSPath(rootfs), + ctrdoci.WithProcessArgs("/bin/sh", "-c", tailNull), + ) + opts = append(opts, extra...) + s := testoci.CreateLinuxSpec(ctx, t, id, opts...) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + + t.Cleanup(func() { + unmountRootfs(ctx, t, scratch) + }) + + return createContainer(ctx, t, host, id, r) +} + +func createContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, id string, s *prot.VMHostedContainerSettingsV2) *hcsv2.Container { + c, err := host.CreateContainer(ctx, id, s) + if err != nil { + t.Helper() + t.Fatalf("could not create container %q: %v", id, err) + } + + return c +} + +func removeContainer(_ context.Context, _ testing.TB, host *hcsv2.Host, id string) { + host.RemoveContainer(id) +} + +func startContainer(ctx context.Context, t testing.TB, c *hcsv2.Container, conn stdio.ConnectionSettings) hcsv2.Process { + pid, err := c.Start(ctx, conn) + if err != nil { + t.Helper() + t.Fatalf("could not start container %q: %v", c.ID(), err) + } + + return getProcess(ctx, t, c, uint32(pid)) +} + +// waitContainer waits on the container's init process, p. +func waitContainer(ctx context.Context, t testing.TB, c *hcsv2.Container, p hcsv2.Process, forced bool) { + t.Helper() + + var e int + ch := make(chan prot.NotificationType) + + // have to read the init process exit code to close the container + exch, dch := p.Wait() + defer close(dch) + go func() { + e = <-exch + dch <- true + ch <- c.Wait() + close(ch) + }() + + select { + case n, ok := <-ch: + if !ok { + t.Fatalf("container %q did not return a notification", c.ID()) + } + + switch { + // UnexpectedExit is the default, ForcedExit if killed + case n == prot.NtGracefulExit: + case n == prot.NtUnexpectedExit: + case forced && n == prot.NtForcedExit: + default: + t.Fatalf("container %q exited with %s", c.ID(), n) + } + case <-ctx.Done(): + t.Fatalf("context canceled: %v", ctx.Err()) + } + + switch { + case e == 0: + case forced && e == 137: + default: + t.Fatalf("got exit code %d", e) + } +} + +func waitContainerRaw(c *hcsv2.Container, p hcsv2.Process) (int, prot.NotificationType) { + exch, dch := p.Wait() + defer close(dch) + r := <-exch + dch <- true + n := c.Wait() + + return r, n +} + +func execProcess(ctx context.Context, t testing.TB, c *hcsv2.Container, p *oci.Process, con stdio.ConnectionSettings) hcsv2.Process { + pid, err := c.ExecProcess(ctx, p, con) + if err != nil { + t.Helper() + t.Fatalf("could not exec process: %v", err) + } + + return getProcess(ctx, t, c, uint32(pid)) +} + +func getProcess(_ context.Context, t testing.TB, c *hcsv2.Container, pid uint32) hcsv2.Process { + p, err := c.GetProcess(pid) + if err != nil { + t.Helper() + t.Fatalf("could not get process %d: %v", pid, err) + } + + return p +} + +func killContainer(ctx context.Context, t testing.TB, c *hcsv2.Container) { + if err := c.Kill(ctx, syscall.SIGKILL); err != nil { + t.Helper() + t.Fatalf("could not kill container %q: %v", c.ID(), err) + } +} + +func deleteContainer(ctx context.Context, t testing.TB, c *hcsv2.Container) { + if err := c.Delete(ctx); err != nil { + t.Helper() + t.Fatalf("could not delete container %q: %v", c.ID(), err) + } +} + +func cleanupContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, c *hcsv2.Container) { + deleteContainer(ctx, t, c) + removeContainer(ctx, t, host, c.ID()) +} + +// +// runtime +// + +func listContainerStates(_ context.Context, t testing.TB, rt runtime.Runtime) []runtime.ContainerState { + css, err := rt.ListContainerStates() + if err != nil { + t.Helper() + t.Fatalf("could not list containers: %v", err) + } + + return css +} + +// assertNumberContainers asserts that n containers are found, and then returns the container states. +func assertNumberContainers(ctx context.Context, t testing.TB, rt runtime.Runtime, n int) { + fmt := "found %d running containers, wanted %d" + css := listContainerStates(ctx, t, rt) + nn := len(css) + if nn != n { + t.Helper() + + if nn == 0 { + t.Fatalf(fmt, nn, n) + } + + cs := make([]string, nn) + for i, c := range css { + cs[i] = c.ID + } + + t.Fatalf(fmt+":\n%#+v", nn, n, cs) + } +} + +func getContainerState(ctx context.Context, t testing.TB, rt runtime.Runtime, id string) runtime.ContainerState { + css := listContainerStates(ctx, t, rt) + + for _, cs := range css { + if cs.ID == id { + return cs + } + } + + t.Helper() + t.Fatalf("could not find container %q", id) + return runtime.ContainerState{} // just to make the linter happy +} + +func assertContainerState(ctx context.Context, t testing.TB, rt runtime.Runtime, id, state string) { + cs := getContainerState(ctx, t, rt, id) + if cs.Status != state { + t.Helper() + t.Fatalf("got container %q status %q, wanted %q", id, cs.Status, state) + } +} + +// +// mount management +// + +func mountRootfs(ctx context.Context, t testing.TB, host *hcsv2.Host, id string) (scratch string, rootfs string) { + scratch = filepath.Join(guestpath.LCOWRootPrefixInUVM, id) + rootfs = filepath.Join(scratch, "rootfs") + if err := overlay.MountLayer(ctx, + []string{*flagRootfsPath}, + filepath.Join(scratch, "upper"), + filepath.Join(scratch, "work"), + rootfs, + false, // readonly + id, + host.SecurityPolicyEnforcer(), + ); err != nil { + t.Helper() + t.Fatalf("could not mount overlay layers from %q: %v", *flagRootfsPath, err) + } + + return scratch, rootfs +} + +func unmountRootfs(ctx context.Context, t testing.TB, path string) { + if err := storage.UnmountAllInPath(ctx, path, true); err != nil { + t.Fatalf("could not unmount container rootfs: %v", err) + } + if err := os.RemoveAll(path); err != nil { + t.Fatalf("could not remove container directory: %v", err) + } +} + +// +// network namespaces +// + +func createNamespace(ctx context.Context, t testing.TB, nns string) { + ns := hcsv2.GetOrAddNetworkNamespace(nns) + if err := ns.Sync(ctx); err != nil { + t.Helper() + t.Fatalf("could not sync new namespace %q: %v", nns, err) + } +} + +func removeNamespace(ctx context.Context, t testing.TB, nns string) { + if err := hcsv2.RemoveNetworkNamespace(ctx, nns); err != nil { + t.Helper() + t.Fatalf("could not remove namespace %q: %v", nns, err) + } +} diff --git a/test/gcs/helper_cri_test.go b/test/gcs/helper_cri_test.go new file mode 100644 index 0000000000..cf9dd09147 --- /dev/null +++ b/test/gcs/helper_cri_test.go @@ -0,0 +1,134 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cri/annotations" + criopts "github.com/containerd/containerd/pkg/cri/opts" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// +// testing helper functions for generic container management +// + +func sandboxSpec( + ctx context.Context, + t testing.TB, + name string, + id string, + nns string, + root string, + extra ...oci.SpecOpts, +) *oci.Spec { + ctx = namespaces.WithNamespace(ctx, testoci.CRINamespace) + opts := sandboxSpecOpts(ctx, t, name, id, nns, root) + opts = append(opts, extra...) + + return testoci.CreateLinuxSpec(ctx, t, id, opts...) +} + +func sandboxSpecOpts(_ context.Context, t testing.TB, + name string, + id string, + nns string, + root string, +) []oci.SpecOpts { + img := testoci.LinuxSandboxImageConfig(*flagSandboxPause) + cfg := testoci.LinuxSandboxRuntimeConfig(name) + + opts := testoci.DefaultLinuxSpecOpts(nns, + oci.WithEnv(img.Env), + oci.WithHostname(cfg.GetHostname()), + oci.WithRootFSPath(root), + ) + + if usr := img.User; usr != "" { + oci.WithUser(usr) + } + + if img.WorkingDir != "" { + opts = append(opts, oci.WithProcessCwd(img.WorkingDir)) + } + + if len(img.Entrypoint) == 0 && len(img.Cmd) == 0 { + t.Helper() + t.Fatalf("invalid empty entrypoint and cmd in image config %+v", img) + } + opts = append(opts, oci.WithProcessArgs(append(img.Entrypoint, img.Cmd...)...)) + + opts = append(opts, + criopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox), + criopts.WithAnnotation(annotations.SandboxID, id), + criopts.WithAnnotation(annotations.SandboxNamespace, cfg.GetMetadata().GetNamespace()), + criopts.WithAnnotation(annotations.SandboxName, cfg.GetMetadata().GetName()), + criopts.WithAnnotation(annotations.SandboxLogDir, cfg.GetLogDirectory()), + ) + + return opts +} + +func containerSpec( + ctx context.Context, + t testing.TB, + sandboxID string, + sandboxPID uint32, + name string, + id string, + cmd []string, + args []string, + wd string, + nns string, + root string, + extra ...oci.SpecOpts, +) *oci.Spec { + ctx = namespaces.WithNamespace(ctx, testoci.CRINamespace) + opts := containerSpecOpts(ctx, t, sandboxID, sandboxPID, name, cmd, args, wd, nns, root) + opts = append(opts, extra...) + + return testoci.CreateLinuxSpec(ctx, t, id, opts...) +} + +func containerSpecOpts(_ context.Context, _ testing.TB, + sandboxID string, + sandboxPID uint32, + name string, + cmd []string, + args []string, + wd string, + nns string, + root string, +) []oci.SpecOpts { + cfg := testoci.LinuxWorkloadRuntimeConfig(name, cmd, args, wd) + img := testoci.LinuxWorkloadImageConfig() + + opts := testoci.DefaultLinuxSpecOpts(nns, + oci.WithRootFSPath(root), + oci.WithEnv(nil), + // this will be set based on the security context below + oci.WithNewPrivileges, + criopts.WithProcessArgs(cfg, img), + criopts.WithPodNamespaces(nil, sandboxPID, sandboxPID), + ) + + hostname := name + env := append([]string{testoci.HostnameEnv + "=" + hostname}, img.Env...) + for _, e := range cfg.GetEnvs() { + env = append(env, e.GetKey()+"="+e.GetValue()) + } + opts = append(opts, oci.WithEnv(env)) + + opts = append(opts, + criopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer), + criopts.WithAnnotation(annotations.SandboxID, sandboxID), + criopts.WithAnnotation(annotations.ContainerName, name), + ) + + return opts +} diff --git a/test/gcs/main_test.go b/test/gcs/main_test.go new file mode 100644 index 0000000000..5ab9f9be6e --- /dev/null +++ b/test/gcs/main_test.go @@ -0,0 +1,167 @@ +//go:build linux + +package gcs + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "strconv" + "testing" + + "github.com/containerd/cgroups" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guest/runtime" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/runtime/runc" + "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + + testflag "github.com/Microsoft/hcsshim/test/internal/flag" + "github.com/Microsoft/hcsshim/test/internal/require" +) + +const ( + featureCRI = "CRI" + featureStandalone = "StandAlone" +) + +var allFeatures = []string{ + featureCRI, + featureStandalone, +} + +var ( + flagFeatures = testflag.NewFeatureFlag(allFeatures) + flagJoinGCSCgroup = flag.Bool( + "join-gcs-cgroup", + false, + "If true, join the same cgroup as the gcs daemon, `/gcs`", + ) + flagRootfsPath = flag.String( + "rootfs-path", + "/run/rootfs", + "The path on the uVM of the unpacked rootfs to use for the containers", + ) + flagSandboxPause = flag.Bool( + "pause-sandbox", + false, + "Use `/pause` as the sandbox container command", + ) +) + +var securityPolicy string + +func init() { + var err error + if securityPolicy, err = securitypolicy.NewOpenDoorPolicy().EncodeToString(); err != nil { + log.Fatal("could not encode open door policy to string: %w", err) + } +} + +func TestMain(m *testing.M) { + flag.Parse() + + if err := setup(); err != nil { + logrus.WithError(err).Fatal("could not set up testing") + } + + os.Exit(m.Run()) +} + +func setup() (err error) { + _ = os.MkdirAll(guestpath.LCOWRootPrefixInUVM, 0755) + + if vf := flag.Lookup("test.v"); vf != nil { + if vf.Value.String() == strconv.FormatBool(true) { + logrus.SetLevel(logrus.DebugLevel) + } else { + logrus.SetLevel(logrus.ErrorLevel) + } + } + + // should already start gcs cgroup + if !*flagJoinGCSCgroup { + gcsControl, err := cgroups.Load(cgroups.V1, cgroups.StaticPath("/")) + if err != nil { + return fmt.Errorf("failed to load root cgroup: %w", err) + } + if err := gcsControl.Add(cgroups.Process{Pid: os.Getpid()}); err != nil { + return fmt.Errorf("failed join root cgroup: %w", err) + } + logrus.Debug("joined root cgroup") + } + + // initialize runtime + rt, err := getRuntimeErr() + if err != nil { + return err + } + + // check policy will be parsed properly + if _, err = getHostErr(rt, getTransport()); err != nil { + return err + } + + return nil +} + +// +// host and runtime management +// + +func getTestState(ctx context.Context, t testing.TB) (*hcsv2.Host, runtime.Runtime) { + rt := getRuntime(ctx, t) + + return getHost(ctx, t, rt), rt +} + +func getHost(_ context.Context, t testing.TB, rt runtime.Runtime) *hcsv2.Host { + h, err := getHostErr(rt, getTransport()) + if err != nil { + t.Helper() + t.Fatalf("could not get host: %v", err) + } + + return h +} + +func getHostErr(rt runtime.Runtime, tp transport.Transport) (*hcsv2.Host, error) { + h := hcsv2.NewHost(rt, tp) + if err := h.SetConfidentialUVMOptions("", securityPolicy, ""); err != nil { + return nil, fmt.Errorf("could not set host security policy: %w", err) + } + + return h, nil +} + +func getRuntime(_ context.Context, t testing.TB) runtime.Runtime { + rt, err := getRuntimeErr() + if err != nil { + t.Helper() + t.Fatalf("could not get runtime: %v", err) + } + + return rt +} + +func getRuntimeErr() (runtime.Runtime, error) { + rt, err := runc.NewRuntime(guestpath.LCOWRootPrefixInUVM) + if err != nil { + return rt, fmt.Errorf("failed to initialize runc runtime: %w", err) + } + + return rt, nil +} + +func getTransport() transport.Transport { + return &PipeTransport{} +} + +func requireFeatures(t testing.TB, features ...string) { + require.Features(t, flagFeatures.S, features...) +} diff --git a/test/go.mod b/test/go.mod index 14bd43e43b..5e42bb99be 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,7 +4,8 @@ go 1.17 require ( github.com/Microsoft/go-winio v0.5.2 - github.com/Microsoft/hcsshim v0.9.3 + github.com/Microsoft/hcsshim v0.9.4 + github.com/containerd/cgroups v1.0.3 github.com/containerd/containerd v1.6.6 github.com/containerd/go-runc v1.0.0 github.com/containerd/ttrpc v1.1.0 @@ -29,18 +30,20 @@ require ( github.com/agnivade/levenshtein v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.2.2 // indirect github.com/containerd/fifo v1.0.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.17+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.17+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/gogo/googleapis v1.4.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -49,9 +52,12 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 // indirect + github.com/mattn/go-shellwords v1.0.12 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.5.0 // indirect github.com/moby/sys/signal v0.6.0 // indirect + github.com/moby/sys/symlink v0.2.0 // indirect github.com/open-policy-agent/opa v0.42.2 // indirect github.com/opencontainers/runc v1.1.2 // indirect github.com/opencontainers/selinux v1.10.1 // indirect @@ -60,6 +66,8 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vektah/gqlparser/v2 v2.4.5 // indirect + github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect diff --git a/test/go.sum b/test/go.sum index 39855f1af8..33a0520ffd 100644 --- a/test/go.sum +++ b/test/go.sum @@ -275,9 +275,11 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -330,6 +332,7 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -410,9 +413,11 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c h1:RBUpb2b14UnmRHNd2uHz20ZHLDK+SW5Us/vWF5IHRaY= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= @@ -618,6 +623,7 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 h1:jUp75lepDg0phMUJBCmvaeFDldD2N3S1lBuPwUTszio= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -633,6 +639,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= @@ -664,6 +671,7 @@ github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx github.com/moby/sys/signal v0.6.0 h1:aDpY94H8VlhTGa9sNYUFCFsMZIUh5wm0B6XkIoJj/iY= github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= @@ -897,10 +905,12 @@ github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rty github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA= github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= diff --git a/test/internal/oci/cri.go b/test/internal/oci/cri.go index 5157755cfb..99787e6be1 100644 --- a/test/internal/oci/cri.go +++ b/test/internal/oci/cri.go @@ -2,7 +2,7 @@ package oci import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) //