diff --git a/integration/main_test.go b/integration/main_test.go index 4c4ebddd73203..a540dadfc62ed 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -417,3 +417,15 @@ func RestartContainerd(t *testing.T) { return ConnectDaemons() == nil, nil }, time.Second, 30*time.Second), "wait for containerd to be restarted") } + +func ListSandbox() ([]*runtime.PodSandbox, error) { + client, err := RawRuntimeClient() + if err != nil { + return nil, errors.Wrap(err, "failed to get raw runtime client") + } + resp, err := client.ListPodSandbox(context.Background(), &runtime.ListPodSandboxRequest{}) + if err != nil { + return nil, errors.Wrap(err, "failed to get sandbox status") + } + return resp.Items, nil +} diff --git a/integration/sandbox_clean_remove_test.go b/integration/sandbox_clean_remove_test.go index 02f5d5124fc3c..268dec80e4d6f 100644 --- a/integration/sandbox_clean_remove_test.go +++ b/integration/sandbox_clean_remove_test.go @@ -125,3 +125,106 @@ func TestSandboxRemoveWithoutIPLeakage(t *testing.T) { t.Logf("Should not be able to find the pod ip in host-local checkpoint") assert.False(t, checkIP(ip)) } + +// This test simulates the network setup failure and teardown failure by renaming the name of portmap binary. +// Due to the nature of chained cni plugin, host-local ipam should have allocated IP address. +// RunPodSandx should leave such sandbox in the metadata store to cleanup IP address later. +func TestNetworkTeardownFailureWithoutIPLeakage(t *testing.T) { + + t.Logf("Make sure host-local ipam and portmap are in use") + config, err := CRIConfig() + require.NoError(t, err) + fs, err := os.ReadDir(config.NetworkPluginConfDir) + require.NoError(t, err) + require.NotEmpty(t, fs) + f := filepath.Join(config.NetworkPluginConfDir, fs[0].Name()) + cniConfig, err := os.ReadFile(f) + require.NoError(t, err) + if !strings.Contains(string(cniConfig), "host-local") { + t.Skip("host-local ipam is not in use") + } + if !strings.Contains(string(cniConfig), "portmap") { + t.Skip("portmap is not in use") + } + + t.Logf("Rename the portmap binary") + portmapBin := filepath.Join(config.NetworkPluginBinDir, "portmap") + err = os.Rename(portmapBin, portmapBin+"_tmp") + require.NoError(t, err) + + t.Logf("Count the numbers of IPs allocated currently") + const ipDirPath = "/var/lib/cni/networks/containerd-net" + ipDir, err := os.ReadDir(ipDirPath) + require.NoError(t, err) + originalIps := len(ipDir) + + t.Logf("Create a sandbox") + const sandboxName = "sandbox-teardown" + + defer func() { + // Change the portmap binary + os.Rename(portmapBin+"_tmp", portmapBin) + + // Make sure the sandbox is cleaned up in any case. + sandboxes, _ := ListSandbox() + for _, sandbox := range sandboxes { + if sandbox.Metadata.Name == sandboxName { + runtimeService.StopPodSandbox(sandbox.Id) + runtimeService.RemovePodSandbox(sandbox.Id) + } + } + }() + + sbConfig := PodSandboxConfig(sandboxName, "teardown-failure-without-ip-leakage") + // The returned sandbox id is empty in error cases + _, err = runtimeService.RunPodSandbox(sbConfig, *runtimeHandler) + require.Error(t, err) + + // Ensure the failure is due to plugin binary is not found + require.Contains(t, err.Error(), "failed to setup network for sandbox") + require.Contains(t, err.Error(), "failed to find plugin") + + t.Logf("Get pod information") + sandboxes, err := ListSandbox() + require.NoError(t, err) + + var sandboxID string + for _, sandbox := range sandboxes { + if sandbox.Metadata.Name == sandboxName { + sandboxID = sandbox.Id + } + } + require.NotEmpty(t, sandboxID, "sandboxId should be found") + + // sandbox status has no IP information when networkSetup fails + sb, info, err := SandboxInfo(sandboxID) + require.NoError(t, err) + assert.Equal(t, runtime.PodSandboxState_SANDBOX_NOTREADY, sb.State) + + require.NotNil(t, info.RuntimeSpec.Linux) + var netNS string + for _, n := range info.RuntimeSpec.Linux.Namespaces { + if n.Type == runtimespec.NetworkNamespace { + netNS = n.Path + } + } + require.NotEmpty(t, netNS, "network namespace should be set") + + t.Logf("Count the numbers of IPs allocated after teardown failure") + ipDir, err = os.ReadDir(ipDirPath) + require.NoError(t, err) + // New IPv4 and IPv6 are allocated + assert.Equal(t, originalIps+2, len(ipDir)) + + err = os.Rename(portmapBin+"_tmp", portmapBin) + require.NoError(t, err) + + t.Logf("Should be able to stop and remove the sandbox") + assert.NoError(t, runtimeService.StopPodSandbox(sandboxID)) + assert.NoError(t, runtimeService.RemovePodSandbox(sandboxID)) + + t.Logf("Count the numbers of IPs allocated after removing the sandbox") + ipDir, err = os.ReadDir(ipDirPath) + require.NoError(t, err) + assert.Equal(t, originalIps, len(ipDir)) +}