feat(k8s): introduce several retry functions to deal w/ flaky apis

Signed-off-by: Mitch Hulscher <mitch.hulscher@lib.io>

modules/k8s/kubectl.go

@@ -4,9 +4,11 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/gruntwork-io/terratest/modules/retry"
 	"github.com/gruntwork-io/terratest/modules/shell"
 	"github.com/gruntwork-io/terratest/modules/testing"
 )
@@ -22,6 +24,26 @@ func RunKubectlE(t testing.TestingT, options *KubectlOptions, args ...string) er
 	return err
 }
 
+// RunKubectlWithRetry will call kubectl using the provided options and args, failing the test after errors have been retried.
+func RunKubectlWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) {
+	err := RunKubectlWithRetryE(t, options, retries, sleepBetweenRetries, args...)
+	require.NoError(t, err)
+}
+
+// RunKubectlWithRetryE will call kubectl using the provided options and args, reytring on error.
+func RunKubectlWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) error {
+	_, err := retry.DoWithRetryE(
+		t,
+		"",
+		retries,
+		sleepBetweenRetries,
+		func() (string, error) {
+			return "", RunKubectlE(t, options, args...)
+		},
+	)
+	return err
+}
+
 // RunKubectlAndGetOutputE will call kubectl using the provided options and args, returning the output of stdout and
 // stderr.
 func RunKubectlAndGetOutputE(t testing.TestingT, options *KubectlOptions, args ...string) (string, error) {
@@ -44,6 +66,29 @@ func RunKubectlAndGetOutputE(t testing.TestingT, options *KubectlOptions, args .
 	return shell.RunCommandAndGetOutputE(t, command)
 }
 
+// RunKubectlAndGetOutputWithRetry will call kubectl using the provided options and args, returning the output of stdout and
+// stderr, failing the test after errors have been retried.
+func RunKubectlAndGetOutputWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) string {
+	output, err := RunKubectlAndGetOutputWithRetryE(t, options, retries, sleepBetweenRetries, args...)
+	require.NoError(t, err)
+	return output
+}
+
+// RunKubectlAndGetOutputWithRetryE will call kubectl using the provided options and args, returning the output of stdout and
+// stderr, reytring on error.
+func RunKubectlAndGetOutputWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) (string, error) {
+	output, err := retry.DoWithRetryE(
+		t,
+		"",
+		retries,
+		sleepBetweenRetries,
+		func() (string, error) {
+			return RunKubectlAndGetOutputE(t, options, args...)
+		},
+	)
+	return output, err
+}
+
 // KubectlDelete will take in a file path and delete it from the cluster targeted by KubectlOptions. If there are any
 // errors, fail the test immediately.
 func KubectlDelete(t testing.TestingT, options *KubectlOptions, configPath string) {
@@ -122,6 +167,28 @@ func KubectlApplyFromStringE(t testing.TestingT, options *KubectlOptions, config
 	return KubectlApplyE(t, options, tmpfile)
 }
 
+// KubectlApplyFromStringE will take in a kubernetes resource config as a string and apply it on the cluster specified
+// by the provided kubectl options. The test fails after errors have been retried.
+func KubectlApplyFromStringWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, configData string) {
+	err := KubectlApplyFromStringWithRetryE(t, options, retries, sleepBetweenRetries, configData)
+	require.NoError(t, err)
+}
+
+// KubectlApplyFromStringE will take in a kubernetes resource config as a string and apply it on the cluster specified
+// by the provided kubectl options. Errors are retried.
+func KubectlApplyFromStringWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, configData string) error {
+	_, err := retry.DoWithRetryE(
+		t,
+		"",
+		retries,
+		sleepBetweenRetries,
+		func() (string, error) {
+			return "", KubectlApplyFromStringE(t, options, configData)
+		},
+	)
+	return err
+}
+
 // StoreConfigToTempFile will store the provided config data to a temporary file created on the os and return the
 // filename.
 func StoreConfigToTempFile(t testing.TestingT, configData string) string {