From 2e138fd85f07bc8c2fde5cf24ca50bc5436faf53 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Jul 2022 17:22:02 -0400 Subject: [PATCH 1/3] agent: add disable_keep_alives config --- api/client.go | 18 + command/agent.go | 14 +- command/agent/config/config.go | 51 ++- command/agent/config/config_test.go | 307 ++++++++++++++++++ .../bad-config-disable-keep-alives.hcl | 27 ++ .../config-disable-keep-alives-all.hcl | 27 ++ .../config-disable-keep-alives-auto-auth.hcl | 27 ++ .../config-disable-keep-alives-caching.hcl | 27 ++ .../config-disable-keep-alives-empty.hcl | 27 ++ .../config-disable-keep-alives-templating.hcl | 27 ++ command/agent/template/template.go | 4 + website/content/docs/agent/index.mdx | 4 + 12 files changed, 548 insertions(+), 12 deletions(-) create mode 100644 command/agent/config/test-fixtures/bad-config-disable-keep-alives.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-auto-auth.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-caching.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-empty.hcl create mode 100644 command/agent/config/test-fixtures/config-disable-keep-alives-templating.hcl diff --git a/api/client.go b/api/client.go index ee8bdd0511b89..650eeaa5662a7 100644 --- a/api/client.go +++ b/api/client.go @@ -756,6 +756,24 @@ func (c *Client) MaxIdleConnections() int { return c.config.HttpClient.Transport.(*http.Transport).MaxIdleConns } +func (c *Client) SetDisableKeepAlives(disable bool) { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.Lock() + defer c.config.modifyLock.Unlock() + + c.config.HttpClient.Transport.(*http.Transport).DisableKeepAlives = disable +} + +func (c *Client) DisableKeepAlives() bool { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + c.config.modifyLock.Lock() + defer c.config.modifyLock.Unlock() + + return c.config.HttpClient.Transport.(*http.Transport).DisableKeepAlives +} + func (c *Client) MaxRetries() int { c.modifyLock.RLock() defer c.modifyLock.RUnlock() diff --git a/command/agent.go b/command/agent.go index 16fa9fa38e18d..898c9a641ef26 100644 --- a/command/agent.go +++ b/command/agent.go @@ -379,6 +379,10 @@ func (c *AgentCommand) Run(args []string) int { sinkClient.SetMaxIdleConnections(-1) } + if config.DisableKeepAlivesAutoAuth { + sinkClient.SetDisableKeepAlives(true) + } + for _, sc := range config.AutoAuth.Sinks { switch sc.Type { case "file": @@ -507,10 +511,14 @@ func (c *AgentCommand) Run(args []string) int { return 1 } - if config.DisableIdleConnsAutoAuth { + if config.DisableIdleConnsCaching { proxyClient.SetMaxIdleConnections(-1) } + if config.DisableKeepAlivesCaching { + proxyClient.SetDisableKeepAlives(true) + } + // Create the API proxier apiProxy, err := cache.NewAPIProxy(&cache.APIProxyConfig{ Client: proxyClient, @@ -824,6 +832,10 @@ func (c *AgentCommand) Run(args []string) int { ahClient.SetMaxIdleConnections(-1) } + if config.DisableKeepAlivesAutoAuth { + ahClient.SetDisableKeepAlives(true) + } + ah := auth.NewAuthHandler(&auth.AuthHandlerConfig{ Logger: c.logger.Named("auth.handler"), Client: ahClient, diff --git a/command/agent/config/config.go b/command/agent/config/config.go index b1cad8d843244..0c8da45451f0a 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -24,19 +24,26 @@ import ( type Config struct { *configutil.SharedConfig `hcl:"-"` - AutoAuth *AutoAuth `hcl:"auto_auth"` - ExitAfterAuth bool `hcl:"exit_after_auth"` - Cache *Cache `hcl:"cache"` - Vault *Vault `hcl:"vault"` - TemplateConfig *TemplateConfig `hcl:"template_config"` - Templates []*ctconfig.TemplateConfig `hcl:"templates"` - DisableIdleConns []string `hcl:"disable_idle_connections"` - DisableIdleConnsCaching bool `hcl:"-"` - DisableIdleConnsTemplating bool `hcl:"-"` - DisableIdleConnsAutoAuth bool `hcl:"-"` + AutoAuth *AutoAuth `hcl:"auto_auth"` + ExitAfterAuth bool `hcl:"exit_after_auth"` + Cache *Cache `hcl:"cache"` + Vault *Vault `hcl:"vault"` + TemplateConfig *TemplateConfig `hcl:"template_config"` + Templates []*ctconfig.TemplateConfig `hcl:"templates"` + DisableIdleConns []string `hcl:"disable_idle_connections"` + DisableIdleConnsCaching bool `hcl:"-"` + DisableIdleConnsTemplating bool `hcl:"-"` + DisableIdleConnsAutoAuth bool `hcl:"-"` + DisableKeepAlives []string `hcl:"disable_keep_alives"` + DisableKeepAlivesCaching bool `hcl:"-"` + DisableKeepAlivesTemplating bool `hcl:"-"` + DisableKeepAlivesAutoAuth bool `hcl:"-"` } -const DisableIdleConnsEnv = "VAULT_AGENT_DISABLE_IDLE_CONNECTIONS" +const ( + DisableIdleConnsEnv = "VAULT_AGENT_DISABLE_IDLE_CONNECTIONS" + DisableKeepAlivesEnv = "VAULT_AGENT_DISABLE_KEEP_ALIVES" +) func (c *Config) Prune() { for _, l := range c.Listeners { @@ -288,6 +295,28 @@ func LoadConfig(path string) (*Config, error) { } } + if disableKeepAlivesEnv := os.Getenv(DisableKeepAlivesEnv); disableKeepAlivesEnv != "" { + result.DisableKeepAlives, err = parseutil.ParseCommaStringSlice(strings.ToLower(disableKeepAlivesEnv)) + if err != nil { + return nil, fmt.Errorf("error parsing environment variable %s: %v", DisableKeepAlivesEnv, err) + } + } + + for _, subsystem := range result.DisableKeepAlives { + switch subsystem { + case "auto-auth": + result.DisableKeepAlivesAutoAuth = true + case "caching": + result.DisableKeepAlivesCaching = true + case "templating": + result.DisableKeepAlivesTemplating = true + case "": + continue + default: + return nil, fmt.Errorf("unknown disable_keep_alives value: %s", subsystem) + } + } + return result, nil } diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index c3fde94a2ab9b..3565e196f6a47 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -1344,3 +1344,310 @@ func TestLoadConfigFile_Bad_Value_Disable_Idle_Conns(t *testing.T) { t.Fatal("should have error, it didn't") } } + +func TestLoadConfigFile_Disable_Keep_Alives_All(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-all.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"auto-auth", "caching", "templating"}, + DisableKeepAlivesCaching: true, + DisableKeepAlivesAutoAuth: true, + DisableKeepAlivesTemplating: true, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Auto_Auth(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-auto-auth.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"auto-auth"}, + DisableKeepAlivesCaching: false, + DisableKeepAlivesAutoAuth: true, + DisableKeepAlivesTemplating: false, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Templating(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-templating.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"templating"}, + DisableKeepAlivesCaching: false, + DisableKeepAlivesAutoAuth: false, + DisableKeepAlivesTemplating: true, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Caching(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-caching.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"caching"}, + DisableKeepAlivesCaching: true, + DisableKeepAlivesAutoAuth: false, + DisableKeepAlivesTemplating: false, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Empty(t *testing.T) { + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-empty.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{}, + DisableKeepAlivesCaching: false, + DisableKeepAlivesAutoAuth: false, + DisableKeepAlivesTemplating: false, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Disable_Keep_Alives_Env(t *testing.T) { + err := os.Setenv(DisableKeepAlivesEnv, "auto-auth,caching,templating") + defer os.Unsetenv(DisableKeepAlivesEnv) + + if err != nil { + t.Fatal(err) + } + config, err := LoadConfig("./test-fixtures/config-disable-keep-alives-empty.hcl") + if err != nil { + t.Fatal(err) + } + + expected := &Config{ + SharedConfig: &configutil.SharedConfig{ + PidFile: "./pidfile", + }, + DisableKeepAlives: []string{"auto-auth", "caching", "templating"}, + DisableKeepAlivesCaching: true, + DisableKeepAlivesAutoAuth: true, + DisableKeepAlivesTemplating: true, + AutoAuth: &AutoAuth{ + Method: &Method{ + Type: "aws", + MountPath: "auth/aws", + Namespace: "my-namespace/", + Config: map[string]interface{}{ + "role": "foobar", + }, + }, + Sinks: []*Sink{ + { + Type: "file", + DHType: "curve25519", + DHPath: "/tmp/file-foo-dhpath", + AAD: "foobar", + Config: map[string]interface{}{ + "path": "/tmp/file-foo", + }, + }, + }, + }, + Vault: &Vault{ + Address: "http://127.0.0.1:1111", + Retry: &Retry{ + ctconfig.DefaultRetryAttempts, + }, + }, + } + + config.Prune() + if diff := deep.Equal(config, expected); diff != nil { + t.Fatal(diff) + } +} + +func TestLoadConfigFile_Bad_Value_Disable_Keep_Alives(t *testing.T) { + _, err := LoadConfig("./test-fixtures/bad-config-disable-keep-alives.hcl") + if err == nil { + t.Fatal("should have error, it didn't") + } +} diff --git a/command/agent/config/test-fixtures/bad-config-disable-keep-alives.hcl b/command/agent/config/test-fixtures/bad-config-disable-keep-alives.hcl new file mode 100644 index 0000000000000..3f1b9f0a198ef --- /dev/null +++ b/command/agent/config/test-fixtures/bad-config-disable-keep-alives.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["foo","caching","templating"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl new file mode 100644 index 0000000000000..9b22cfd8be67d --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-all.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["auto-auth","caching","templating"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-auto-auth.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-auto-auth.hcl new file mode 100644 index 0000000000000..11393bfb57a60 --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-auto-auth.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["auto-auth"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-caching.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-caching.hcl new file mode 100644 index 0000000000000..5712296924edf --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-caching.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["caching"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-empty.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-empty.hcl new file mode 100644 index 0000000000000..8cddcebd8f106 --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-empty.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = [] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/config/test-fixtures/config-disable-keep-alives-templating.hcl b/command/agent/config/test-fixtures/config-disable-keep-alives-templating.hcl new file mode 100644 index 0000000000000..d4731cbd90e2e --- /dev/null +++ b/command/agent/config/test-fixtures/config-disable-keep-alives-templating.hcl @@ -0,0 +1,27 @@ +pid_file = "./pidfile" +disable_keep_alives = ["templating"] + +auto_auth { + method { + type = "aws" + namespace = "my-namespace/" + + config = { + role = "foobar" + } + } + + sink { + type = "file" + config = { + path = "/tmp/file-foo" + } + aad = "foobar" + dh_type = "curve25519" + dh_path = "/tmp/file-foo-dhpath" + } +} + +vault { + address = "http://127.0.0.1:1111" +} diff --git a/command/agent/template/template.go b/command/agent/template/template.go index 0fa1e9a0d2730..1dbfc8e613b01 100644 --- a/command/agent/template/template.go +++ b/command/agent/template/template.go @@ -250,6 +250,10 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc conf.Vault.Transport.MaxIdleConns = &idleConns } + if sc.AgentConfig.DisableKeepAlivesTemplating { + conf.Vault.Transport.DisableKeepAlives = pointerutil.BoolPtr(true) + } + conf.Vault.SSL = &ctconfig.SSLConfig{ Enabled: pointerutil.BoolPtr(false), Verify: pointerutil.BoolPtr(false), diff --git a/website/content/docs/agent/index.mdx b/website/content/docs/agent/index.mdx index 52bd1e57b262f..ee805eac04944 100644 --- a/website/content/docs/agent/index.mdx +++ b/website/content/docs/agent/index.mdx @@ -148,6 +148,10 @@ These are the currently-available general configuration option: Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_IDLE_CONNECTIONS` environment variable as a comma separated string. This environment variable will override any values found in a configuration file. +- `disable_keep_alives` `(string array: [])` - A list of strings that disables keep alives for various features in Vault Agent. + Valid values include: `auto-auth`, `caching` and `templating`. Can also be configured by setting the `VAULT_AGENT_DISABLE_KEEP_ALIVES` + environment variable as a comma separated string. This environment variable will override any values found in a configuration file. + - `template` ([template][template]: ) - Specifies options used for templating Vault secrets to files. - `template_config` ([template_config][template-config]: ) - Specifies templating engine behavior. From 68b2cd825ff7a71429df86f777ac29fdd42933a4 Mon Sep 17 00:00:00 2001 From: Jason O'Donnell <2160810+jasonodonnell@users.noreply.github.com> Date: Wed, 27 Jul 2022 17:28:32 -0400 Subject: [PATCH 2/3] changelog --- changelog/16479.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/16479.txt diff --git a/changelog/16479.txt b/changelog/16479.txt new file mode 100644 index 0000000000000..43b5258d5bed4 --- /dev/null +++ b/changelog/16479.txt @@ -0,0 +1,3 @@ +```release-note:improvement +agent: Added `disable_keep_alives` configuration to disable keep alives in auto-auth, caching and templating. +``` From d5ea352abbca67dfe684ae57e9af4e0ed1da6f64 Mon Sep 17 00:00:00 2001 From: Christopher Swenson Date: Thu, 28 Jul 2022 12:37:09 -0700 Subject: [PATCH 3/3] Change write lock to read lock --- api/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client.go b/api/client.go index 650eeaa5662a7..7c17981059fb7 100644 --- a/api/client.go +++ b/api/client.go @@ -768,8 +768,8 @@ func (c *Client) SetDisableKeepAlives(disable bool) { func (c *Client) DisableKeepAlives() bool { c.modifyLock.RLock() defer c.modifyLock.RUnlock() - c.config.modifyLock.Lock() - defer c.config.modifyLock.Unlock() + c.config.modifyLock.RLock() + defer c.config.modifyLock.RUnlock() return c.config.HttpClient.Transport.(*http.Transport).DisableKeepAlives }