Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Backport of agent/auto-auth: add exit_on_err configurable into release/1.11.x #17156

Merged
merged 1 commit into from Sep 15, 2022
Merged

Backport of agent/auto-auth: add exit_on_err configurable into release/1.11.x #17156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog/17091.txt
View file
Open in desktop
@@ -0,0 +1,3 @@
```release-note:improvement
agent/auto-auth: Add `exit_on_err` which when set to true, will cause Agent to exit if any errors are encountered during authentication.
```
1 change: 1 addition & 0 deletions command/agent.go
View file
Open in desktop
Expand Up @@ -849,6 +849,7 @@ func (c *AgentCommand) Run(args []string) int {
EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials,
EnableTemplateTokenCh: enableTokenCh,
Token: previousToken,
ExitOnError: config.AutoAuth.Method.ExitOnError,
})

ss := sink.NewSinkServer(&sink.SinkServerConfig{
Expand Down
132 changes: 88 additions & 44 deletions command/agent/auth/auth.go
View file
Open in desktop
Expand Up @@ -58,6 +58,7 @@ type AuthHandler struct {
minBackoff time.Duration
enableReauthOnNewCredentials bool
enableTemplateTokenCh bool
exitOnError bool
}

type AuthHandlerConfig struct {
Expand All @@ -69,6 +70,7 @@ type AuthHandlerConfig struct {
Token string
EnableReauthOnNewCredentials bool
EnableTemplateTokenCh bool
ExitOnError bool
}

func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
Expand All @@ -86,12 +88,17 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
maxBackoff: conf.MaxBackoff,
enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials,
enableTemplateTokenCh: conf.EnableTemplateTokenCh,
exitOnError: conf.ExitOnError,
}

return ah
}

func backoffOrQuit(ctx context.Context, backoff *agentBackoff) {
func backoff(ctx context.Context, backoff *agentBackoff) bool {
if backoff.exitOnErr {
return false
}

select {
case <-time.After(backoff.current):
case <-ctx.Done():
Expand All @@ -100,6 +107,7 @@ func backoffOrQuit(ctx context.Context, backoff *agentBackoff) {
// Increase exponential backoff for the next time if we don't
// successfully auth/renew/etc.
backoff.next()
return true
}

func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
Expand All @@ -111,9 +119,9 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
ah.minBackoff = defaultMinBackoff
}

backoff := newAgentBackoff(ah.minBackoff, ah.maxBackoff)
backoffCfg := newAgentBackoff(ah.minBackoff, ah.maxBackoff, ah.exitOnError)

if backoff.min >= backoff.max {
if backoffCfg.min >= backoffCfg.max {
return errors.New("auth handler: min_backoff cannot be greater than max_backoff")
}

Expand Down Expand Up @@ -167,9 +175,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
clientToUse, err = am.(AuthMethodWithClient).AuthClient(ah.client)
if err != nil {
ah.logger.Error("error creating client for authentication call", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}

return err
}
default:
clientToUse = ah.client
Expand All @@ -189,10 +201,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {

secret, err = clientToUse.Auth().Token().LookupSelfWithContext(ctx)
if err != nil {
ah.logger.Error("could not look up token", "err", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("could not look up token", "err", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}

duration, _ := secret.Data["ttl"].(json.Number).Int64()
Expand All @@ -206,20 +221,26 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {

path, header, data, err = am.Authenticate(ctx, ah.client)
if err != nil {
ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("error getting path or data from method", "error", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
}

if ah.wrapTTL > 0 {
wrapClient, err := clientToUse.Clone()
if err != nil {
ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("error creating client for wrapped call", "error", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
wrapClient.SetWrappingLookupFunc(func(string, string) string {
return ah.wrapTTL.String()
Expand All @@ -238,33 +259,45 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
secret, err = clientToUse.Logical().WriteWithContext(ctx, path, data)
// Check errors/sanity
if err != nil {
ah.logger.Error("error authenticating", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("error authenticating", "error", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
}

switch {
case ah.wrapTTL > 0:
if secret.WrapInfo == nil {
ah.logger.Error("authentication returned nil wrap info", "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("authentication returned nil wrap info", "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
if secret.WrapInfo.Token == "" {
ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("authentication returned empty wrapped client token", "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
wrappedResp, err := jsonutil.EncodeJSON(secret.WrapInfo)
if err != nil {
ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("failed to encode wrapinfo", "error", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
ah.logger.Info("authentication successful, sending wrapped token to sinks and pausing")
ah.OutputCh <- string(wrappedResp)
Expand All @@ -273,7 +306,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
}

am.CredSuccess()
backoff.reset()
backoffCfg.reset()

select {
case <-ctx.Done():
Expand All @@ -287,16 +320,22 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {

default:
if secret == nil || secret.Auth == nil {
ah.logger.Error("authentication returned nil auth info", "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("authentication returned nil auth info", "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
if secret.Auth.ClientToken == "" {
ah.logger.Error("authentication returned empty client token", "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("authentication returned empty client token", "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}
ah.logger.Info("authentication successful, sending token to sinks")
ah.OutputCh <- secret.Auth.ClientToken
Expand All @@ -305,7 +344,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
}

am.CredSuccess()
backoff.reset()
backoffCfg.reset()
}

if watcher != nil {
Expand All @@ -316,10 +355,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
Secret: secret,
})
if err != nil {
ah.logger.Error("error creating lifetime watcher, backing off and retrying", "error", err, "backoff", backoff)
backoffOrQuit(ctx, backoff)
ah.logger.Error("error creating lifetime watcher", "error", err, "backoff", backoffCfg)
metrics.IncrCounter([]string{"agent", "auth", "failure"}, 1)
continue

if backoff(ctx, backoffCfg) {
continue
}
return err
}

// Start the renewal process
Expand Down Expand Up @@ -357,12 +399,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {

// agentBackoff tracks exponential backoff state.
type agentBackoff struct {
min time.Duration
max time.Duration
current time.Duration
min time.Duration
max time.Duration
current time.Duration
exitOnErr bool
}

func newAgentBackoff(min, max time.Duration) *agentBackoff {
func newAgentBackoff(min, max time.Duration, exitErr bool) *agentBackoff {
if max <= 0 {
max = defaultMaxBackoff
}
Expand All @@ -372,9 +415,10 @@ func newAgentBackoff(min, max time.Duration) *agentBackoff {
}

return &agentBackoff{
current: min,
max: max,
min: min,
current: min,
max: max,
min: min,
exitOnErr: exitErr,
}
}

Expand Down
4 changes: 2 additions & 2 deletions command/agent/auth/auth_test.go
View file
Open in desktop
Expand Up @@ -109,7 +109,7 @@ consumption:

func TestAgentBackoff(t *testing.T) {
max := 1024 * time.Second
backoff := newAgentBackoff(defaultMinBackoff, max)
backoff := newAgentBackoff(defaultMinBackoff, max, false)

// Test initial value
if backoff.current != defaultMinBackoff {
Expand Down Expand Up @@ -159,7 +159,7 @@ func TestAgentMinBackoffCustom(t *testing.T) {

for _, test := range tests {
max := 1024 * time.Second
backoff := newAgentBackoff(test.minBackoff, max)
backoff := newAgentBackoff(test.minBackoff, max, false)

// Test initial value
if backoff.current != test.want {
Expand Down
1 change: 1 addition & 0 deletions command/agent/config/config.go
View file
Open in desktop
Expand Up @@ -130,6 +130,7 @@ type Method struct {
MaxBackoffRaw interface{} `hcl:"max_backoff"`
MaxBackoff time.Duration `hcl:"-"`
Namespace string `hcl:"namespace"`
ExitOnError bool `hcl:"exit_on_err"`
Config map[string]interface{}
}

Expand Down
64 changes: 55 additions & 9 deletions command/agent/config/config_test.go
View file
Open in desktop
Expand Up @@ -260,10 +260,11 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) {
},
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
WrapTTL: 5 * time.Minute,
MaxBackoff: 2 * time.Minute,
Type: "aws",
MountPath: "auth/aws",
ExitOnError: false,
WrapTTL: 5 * time.Minute,
MaxBackoff: 2 * time.Minute,
Config: map[string]interface{}{
"role": "foobar",
},
Expand Down Expand Up @@ -302,11 +303,56 @@ func TestLoadConfigFile_Method_InitialBackoff(t *testing.T) {
},
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
WrapTTL: 5 * time.Minute,
MinBackoff: 5 * time.Second,
MaxBackoff: 2 * time.Minute,
Type: "aws",
MountPath: "auth/aws",
ExitOnError: false,
WrapTTL: 5 * time.Minute,
MinBackoff: 5 * time.Second,
MaxBackoff: 2 * time.Minute,
Config: map[string]interface{}{
"role": "foobar",
},
},
Sinks: []*Sink{
{
Type: "file",
Config: map[string]interface{}{
"path": "/tmp/file-foo",
},
},
},
},
Vault: &Vault{
Retry: &Retry{
NumRetries: 12,
},
},
}

config.Prune()
if diff := deep.Equal(config, expected); diff != nil {
t.Fatal(diff)
}
}

func TestLoadConfigFile_Method_ExitOnErr(t *testing.T) {
config, err := LoadConfig("./test-fixtures/config-method-exit-on-err.hcl")
if err != nil {
t.Fatalf("err: %s", err)
}

expected := &Config{
SharedConfig: &configutil.SharedConfig{
PidFile: "./pidfile",
},
AutoAuth: &AutoAuth{
Method: &Method{
Type: "aws",
MountPath: "auth/aws",
ExitOnError: true,
WrapTTL: 5 * time.Minute,
MinBackoff: 5 * time.Second,
MaxBackoff: 2 * time.Minute,
Config: map[string]interface{}{
"role": "foobar",
},
Expand Down