diff --git a/integration/main_test.go b/integration/main_test.go index 650ee2701739..27df2b80678c 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -164,6 +164,15 @@ func WithPodHostname(hostname string) PodSandboxOpts { } } +// Add pod labels. +func WithPodLabels(kvs map[string]string) PodSandboxOpts { + return func(p *runtime.PodSandboxConfig) { + for k, v := range kvs { + p.Labels[k] = v + } + } +} + // PodSandboxConfig generates a pod sandbox config for test. func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandboxConfig { config := &runtime.PodSandboxConfig{ @@ -176,6 +185,7 @@ func PodSandboxConfig(name, ns string, opts ...PodSandboxOpts) *runtime.PodSandb }, Linux: &runtime.LinuxPodSandboxConfig{}, Annotations: make(map[string]string), + Labels: make(map[string]string), } for _, opt := range opts { opt(config) diff --git a/integration/sandbox_run_rollback_test.go b/integration/sandbox_run_rollback_test.go index 4ab5829b6a0e..465b97924ba6 100644 --- a/integration/sandbox_run_rollback_test.go +++ b/integration/sandbox_run_rollback_test.go @@ -28,10 +28,10 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/containerd/containerd/pkg/failpoint" - "github.com/stretchr/testify/require" ) const ( @@ -89,6 +89,146 @@ func TestRunPodSandboxWithShimStartFailure(t *testing.T) { require.Equal(t, true, strings.Contains(err.Error(), "no hard feelings")) } +// TestRunPodSandboxWithShimDeleteFailure should keep the sandbox record if +// failed to rollback shim by shim.Delete API. +func TestRunPodSandboxWithShimDeleteFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + if os.Getenv("ENABLE_CRI_SANDBOXES") != "" { + t.Skip() + } + + testCase := func(restart bool) func(*testing.T) { + return func(t *testing.T) { + t.Log("Init PodSandboxConfig with specific label") + labels := map[string]string{ + t.Name(): "true", + } + sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels)) + + t.Log("Inject Shim failpoint") + injectShimFailpoint(t, sbConfig, map[string]string{ + "Start": "1*error(failed to start shim)", + "Delete": "1*error(please retry)", // inject failpoint during rollback shim + }) + + t.Log("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to start shim") + + t.Log("ListPodSandbox with the specific label") + l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels}) + require.NoError(t, err) + require.Len(t, l, 1) + + sb := l[0] + require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name) + require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace) + require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid) + require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt) + + t.Log("Check PodSandboxStatus") + sbStatus, err := runtimeService.PodSandboxStatus(sb.Id) + require.NoError(t, err) + require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + require.Greater(t, len(sbStatus.Network.Ip), 0) + + if restart { + t.Log("Restart containerd") + RestartContainerd(t) + + t.Log("ListPodSandbox with the specific label") + l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id}) + require.NoError(t, err) + require.Len(t, l, 1) + require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + + t.Log("Check PodSandboxStatus") + sbStatus, err := runtimeService.PodSandboxStatus(sb.Id) + require.NoError(t, err) + t.Log(sbStatus.Network) + require.Equal(t, sbStatus.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + } + + t.Log("Cleanup leaky sandbox") + err = runtimeService.RemovePodSandbox(sb.Id) + require.NoError(t, err) + } + } + + t.Run("CleanupAfterRestart", testCase(true)) + t.Run("JustCleanup", testCase(false)) +} + +// TestRunPodSandboxWithShimStartAndTeardownCNIFailure should keep the sandbox +// record if failed to rollback CNI API. +func TestRunPodSandboxWithShimStartAndTeardownCNIFailure(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + if os.Getenv("ENABLE_CRI_SANDBOXES") != "" { + t.Skip() + } + + testCase := func(restart bool) func(*testing.T) { + return func(t *testing.T) { + t.Log("Init PodSandboxConfig with specific key") + labels := map[string]string{ + t.Name(): "true", + } + sbConfig := PodSandboxConfig(t.Name(), "failpoint", WithPodLabels(labels)) + + t.Log("Inject Shim failpoint") + injectShimFailpoint(t, sbConfig, map[string]string{ + "Start": "1*error(failed to start shim)", + }) + + t.Log("Inject CNI failpoint") + conf := &failpointConf{ + Del: "1*error(please retry)", + } + injectCNIFailpoint(t, sbConfig, conf) + + t.Log("Create a sandbox") + _, err := runtimeService.RunPodSandbox(sbConfig, failpointRuntimeHandler) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to start shim") + + t.Log("ListPodSandbox with the specific label") + l, err := runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{LabelSelector: labels}) + require.NoError(t, err) + require.Len(t, l, 1) + + sb := l[0] + require.Equal(t, sb.State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + require.Equal(t, sb.Metadata.Name, sbConfig.Metadata.Name) + require.Equal(t, sb.Metadata.Namespace, sbConfig.Metadata.Namespace) + require.Equal(t, sb.Metadata.Uid, sbConfig.Metadata.Uid) + require.Equal(t, sb.Metadata.Attempt, sbConfig.Metadata.Attempt) + + if restart { + t.Log("Restart containerd") + RestartContainerd(t) + + t.Log("ListPodSandbox with the specific label") + l, err = runtimeService.ListPodSandbox(&criapiv1.PodSandboxFilter{Id: sb.Id}) + require.NoError(t, err) + require.Len(t, l, 1) + require.Equal(t, l[0].State, criapiv1.PodSandboxState_SANDBOX_NOTREADY) + } + + t.Log("Cleanup leaky sandbox") + err = runtimeService.RemovePodSandbox(sb.Id) + require.NoError(t, err) + } + } + t.Run("CleanupAfterRestart", testCase(true)) + t.Run("JustCleanup", testCase(false)) +} + // failpointConf is used to describe cmdAdd/cmdDel/cmdCheck command's failpoint. type failpointConf struct { Add string `json:"cmdAdd"` @@ -101,7 +241,7 @@ func injectCNIFailpoint(t *testing.T, sbConfig *criapiv1.PodSandboxConfig, conf metadata := sbConfig.Metadata fpFilename := filepath.Join(stateDir, - fmt.Sprintf("%s-%s.json", metadata.Namespace, metadata.Name)) + fmt.Sprintf("%s-%s.json", metadata.Namespace, strings.Replace(metadata.Name, "/", "-", -1))) data, err := json.Marshal(conf) require.NoError(t, err)