Skip to content

Commit

Permalink
add logs when pod failed to start when using WaitUntilPodAvailableE
Browse files Browse the repository at this point in the history
Fix gruntwork-io#1251

Signed-off-by: Charly Molter <charly.molter@konghq.com>
  • Loading branch information
lahabana committed Mar 2, 2023
1 parent 81b5820 commit 632e07d
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 6 deletions.
2 changes: 1 addition & 1 deletion modules/k8s/errors.go
Expand Up @@ -68,7 +68,7 @@ type PodNotAvailable struct {

// Error is a simple function to return a formatted error message as a string
func (err PodNotAvailable) Error() string {
return fmt.Sprintf("Pod %s is not available, reason: %s, message: %s", err.pod.Name, err.pod.Status.Reason, err.pod.Status.Message)
return fmt.Sprintf("Pod %s is not available, reason: %s, message: %s, phase: %s, conditions: %+v", err.pod.Name, err.pod.Status.Reason, err.pod.Status.Message, err.pod.Status.Phase, err.pod.Status.Conditions)
}

// NewPodNotAvailableError returnes a PodNotAvailable struct when Kubernetes deems a pod is not available
Expand Down
14 changes: 12 additions & 2 deletions modules/k8s/pod.go
Expand Up @@ -2,6 +2,7 @@ package k8s

import (
"context"
"errors"
"fmt"
"time"

Expand Down Expand Up @@ -129,10 +130,19 @@ func WaitUntilPodAvailableE(t testing.TestingT, options *KubectlOptions, podName
},
)
if err != nil {
logger.Logf(t, "Timedout waiting for Pod to be provisioned: %s", err)
logger.Log(t, "Timedout waiting for Pod to be provisioned", err)
notAvailableError := PodNotAvailable{}
if errors.As(err, &notAvailableError) {
if notAvailableError.pod.Status.Phase == corev1.PodRunning {
_, logsError := GetPodLogsE(t, options, notAvailableError.pod, "")
if logsError != nil {
logger.Log(t, "Failed to retrieve logs", err)
}
}
}
return err
}
logger.Logf(t, message)
logger.Log(t, message)
return nil
}

Expand Down
16 changes: 13 additions & 3 deletions modules/retry/retry.go
Expand Up @@ -2,6 +2,7 @@
package retry

import (
"errors"
"fmt"
"regexp"
"time"
Expand Down Expand Up @@ -87,15 +88,15 @@ func DoWithRetryInterfaceE(t testing.TestingT, actionDescription string, maxRetr
var output interface{}
var err error

logger.Log(t, actionDescription)
for i := 0; i <= maxRetries; i++ {
logger.Log(t, actionDescription)

output, err = action()
if err == nil {
return output, nil
}

if _, isFatalErr := err.(FatalError); isFatalErr {
if errors.Is(err, FatalError{}) {
logger.Logf(t, "Returning due to fatal error: %v", err)
return output, err
}
Expand All @@ -104,7 +105,7 @@ func DoWithRetryInterfaceE(t testing.TestingT, actionDescription string, maxRetr
time.Sleep(sleepBetweenRetries)
}

return output, MaxRetriesExceeded{Description: actionDescription, MaxRetries: maxRetries}
return output, MaxRetriesExceeded{Description: actionDescription, MaxRetries: maxRetries, LastError: err}
}

// DoWithRetryableErrors runs the specified action. If it returns a value, return that value. If it returns an error,
Expand Down Expand Up @@ -202,12 +203,17 @@ func (err TimeoutExceeded) Error() string {
type MaxRetriesExceeded struct {
Description string
MaxRetries int
LastError error
}

func (err MaxRetriesExceeded) Error() string {
return fmt.Sprintf("'%s' unsuccessful after %d retries", err.Description, err.MaxRetries)
}

func (err MaxRetriesExceeded) Unwrap() error {
return err.LastError
}

// FatalError is a marker interface for errors that should not be retried.
type FatalError struct {
Underlying error
Expand All @@ -216,3 +222,7 @@ type FatalError struct {
func (err FatalError) Error() string {
return fmt.Sprintf("FatalError{Underlying: %v}", err.Underlying)
}

func (err FatalError) Unwrap() error {
return err.Underlying
}

0 comments on commit 632e07d

Please sign in to comment.