Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(k8s): introduce several retry functions to deal w/ flaky apis #1219

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(k8s): introduce several retry functions to deal w/ flaky apis #1219

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
67 changes: 67 additions & 0 deletions modules/k8s/kubectl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"io/ioutil"
"net/url"
"os"
"time"

"github.com/stretchr/testify/require"

"github.com/gruntwork-io/terratest/modules/retry"
"github.com/gruntwork-io/terratest/modules/shell"
"github.com/gruntwork-io/terratest/modules/testing"
)
Expand All @@ -22,6 +24,26 @@ func RunKubectlE(t testing.TestingT, options *KubectlOptions, args ...string) er
return err
}

// RunKubectlWithRetry will call kubectl using the provided options and args, failing the test after errors have been retried.
func RunKubectlWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) {
err := RunKubectlWithRetryE(t, options, retries, sleepBetweenRetries, args...)
require.NoError(t, err)
}

// RunKubectlWithRetryE will call kubectl using the provided options and args, reytring on error.
func RunKubectlWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) error {
_, err := retry.DoWithRetryE(
t,
"",
retries,
sleepBetweenRetries,
func() (string, error) {
return "", RunKubectlE(t, options, args...)
},
)
return err
}

// RunKubectlAndGetOutputE will call kubectl using the provided options and args, returning the output of stdout and
// stderr.
func RunKubectlAndGetOutputE(t testing.TestingT, options *KubectlOptions, args ...string) (string, error) {
Expand All @@ -44,6 +66,29 @@ func RunKubectlAndGetOutputE(t testing.TestingT, options *KubectlOptions, args .
return shell.RunCommandAndGetOutputE(t, command)
}

// RunKubectlAndGetOutputWithRetry will call kubectl using the provided options and args, returning the output of stdout and
// stderr, failing the test after errors have been retried.
func RunKubectlAndGetOutputWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) string {
output, err := RunKubectlAndGetOutputWithRetryE(t, options, retries, sleepBetweenRetries, args...)
require.NoError(t, err)
return output
}

// RunKubectlAndGetOutputWithRetryE will call kubectl using the provided options and args, returning the output of stdout and
// stderr, reytring on error.
func RunKubectlAndGetOutputWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, args ...string) (string, error) {
output, err := retry.DoWithRetryE(
t,
"",
retries,
sleepBetweenRetries,
func() (string, error) {
return RunKubectlAndGetOutputE(t, options, args...)
},
)
return output, err
}

// KubectlDelete will take in a file path and delete it from the cluster targeted by KubectlOptions. If there are any
// errors, fail the test immediately.
func KubectlDelete(t testing.TestingT, options *KubectlOptions, configPath string) {
Expand Down Expand Up @@ -122,6 +167,28 @@ func KubectlApplyFromStringE(t testing.TestingT, options *KubectlOptions, config
return KubectlApplyE(t, options, tmpfile)
}

// KubectlApplyFromStringWithRetry will take in a kubernetes resource config as a string and apply it on the cluster specified
// by the provided kubectl options. The test fails after errors have been retried.
func KubectlApplyFromStringWithRetry(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, configData string) {
err := KubectlApplyFromStringWithRetryE(t, options, retries, sleepBetweenRetries, configData)
require.NoError(t, err)
}

// KubectlApplyFromStringWithRetryE will take in a kubernetes resource config as a string and apply it on the cluster specified
// by the provided kubectl options. Errors are retried.
func KubectlApplyFromStringWithRetryE(t testing.TestingT, options *KubectlOptions, retries int, sleepBetweenRetries time.Duration, configData string) error {
_, err := retry.DoWithRetryE(
t,
"",
retries,
sleepBetweenRetries,
func() (string, error) {
return "", KubectlApplyFromStringE(t, options, configData)
},
)
return err
}

// StoreConfigToTempFile will store the provided config data to a temporary file created on the os and return the
// filename.
func StoreConfigToTempFile(t testing.TestingT, configData string) string {
Expand Down