From 56329911158d3d01a321f320bd13037a36794c4d Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Sat, 29 Jan 2022 12:00:37 +0100 Subject: [PATCH 1/3] Local Traffic Detector: Add two new modes This PR introduces two new modes for detecting local traffic in a cluster. 1) detectLocalByBridgeInterface: This takes a bridge name as argument and decides all traffic that match on their originating interface being that of this bridge, shall be considered as local pod traffic. 2) detectLocalByInterfaceNamePrefix: This takes an interface prefix name as argument and decides all traffic that match on their originating interface names having a prefix that matches this argument shall be considered as local pod traffic. Signed-off-by: Surya Seetharaman --- pkg/proxy/util/iptables/traffic.go | 59 +++++++++++ pkg/proxy/util/iptables/traffic_test.go | 132 ++++++++++++++++++++++++ 2 files changed, 191 insertions(+) diff --git a/pkg/proxy/util/iptables/traffic.go b/pkg/proxy/util/iptables/traffic.go index f96f8f44e097..4666c6c3de68 100644 --- a/pkg/proxy/util/iptables/traffic.go +++ b/pkg/proxy/util/iptables/traffic.go @@ -87,3 +87,62 @@ func (d *detectLocalByCIDR) IfLocal() []string { func (d *detectLocalByCIDR) IfNotLocal() []string { return d.ifNotLocal } + +type detectLocalByBridgeInterface struct { + ifLocal []string + ifNotLocal []string +} + +// NewDetectLocalByBridgeInterface implements the LocalTrafficDetector interface using a bridge interface name. +// This can be used when a bridge can be used to capture the notion of local traffic from pods. +func NewDetectLocalByBridgeInterface(interfaceName string) (LocalTrafficDetector, error) { + if len(interfaceName) == 0 { + return nil, fmt.Errorf("no bridge interface name set") + } + return &detectLocalByBridgeInterface{ + ifLocal: []string{"-i", interfaceName}, + ifNotLocal: []string{"!", "-i", interfaceName}, + }, nil +} + +func (d *detectLocalByBridgeInterface) IsImplemented() bool { + return true +} + +func (d *detectLocalByBridgeInterface) IfLocal() []string { + return d.ifLocal +} + +func (d *detectLocalByBridgeInterface) IfNotLocal() []string { + return d.ifNotLocal +} + +type detectLocalByInterfaceNamePrefix struct { + ifLocal []string + ifNotLocal []string +} + +// NewDetectLocalByInterfaceNamePrefix implements the LocalTrafficDetector interface using an interface name prefix. +// This can be used when a pod interface name prefix can be used to capture the notion of local traffic. Note +// that this will match on all interfaces that start with the given prefix. +func NewDetectLocalByInterfaceNamePrefix(interfacePrefix string) (LocalTrafficDetector, error) { + if len(interfacePrefix) == 0 { + return nil, fmt.Errorf("no interface prefix set") + } + return &detectLocalByInterfaceNamePrefix{ + ifLocal: []string{"-i", interfacePrefix + "+"}, + ifNotLocal: []string{"!", "-i", interfacePrefix + "+"}, + }, nil +} + +func (d *detectLocalByInterfaceNamePrefix) IsImplemented() bool { + return true +} + +func (d *detectLocalByInterfaceNamePrefix) IfLocal() []string { + return d.ifLocal +} + +func (d *detectLocalByInterfaceNamePrefix) IfNotLocal() []string { + return d.ifNotLocal +} diff --git a/pkg/proxy/util/iptables/traffic_test.go b/pkg/proxy/util/iptables/traffic_test.go index d54ec61f8964..f74b850b2fa0 100644 --- a/pkg/proxy/util/iptables/traffic_test.go +++ b/pkg/proxy/util/iptables/traffic_test.go @@ -144,3 +144,135 @@ func TestDetectLocalByCIDR(t *testing.T) { } } } + +func TestNewDetectLocalByBridgeInterface(t *testing.T) { + cases := []struct { + ifaceName string + errExpected bool + }{ + { + ifaceName: "avz", + errExpected: false, + }, + { + ifaceName: "", + errExpected: true, + }, + } + for i, c := range cases { + r, err := NewDetectLocalByBridgeInterface(c.ifaceName) + if c.errExpected { + if err == nil { + t.Errorf("Case[%d] expected error, but succeeded with: %q", i, r) + } + continue + } + if err != nil { + t.Errorf("Case[%d] failed with error: %v", i, err) + } + } +} + +func TestNewDetectLocalByInterfaceNamePrefix(t *testing.T) { + cases := []struct { + ifacePrefix string + errExpected bool + }{ + { + ifacePrefix: "veth", + errExpected: false, + }, + { + ifacePrefix: "cbr0", + errExpected: false, + }, + { + ifacePrefix: "", + errExpected: true, + }, + } + for i, c := range cases { + r, err := NewDetectLocalByInterfaceNamePrefix(c.ifacePrefix) + if c.errExpected { + if err == nil { + t.Errorf("Case[%d] expected error, but succeeded with: %q", i, r) + } + continue + } + if err != nil { + t.Errorf("Case[%d] failed with error: %v", i, err) + } + } +} + +func TestDetectLocalByBridgeInterface(t *testing.T) { + cases := []struct { + ifaceName string + expectedJumpIfOutput []string + expectedJumpIfNotOutput []string + }{ + { + ifaceName: "eth0", + expectedJumpIfOutput: []string{"-i", "eth0"}, + expectedJumpIfNotOutput: []string{"!", "-i", "eth0"}, + }, + } + for _, c := range cases { + localDetector, err := NewDetectLocalByBridgeInterface(c.ifaceName) + if err != nil { + t.Errorf("Error initializing localDetector: %v", err) + continue + } + if !localDetector.IsImplemented() { + t.Error("DetectLocalByBridgeInterface returns false for IsImplemented") + } + + ifLocal := localDetector.IfLocal() + ifNotLocal := localDetector.IfNotLocal() + + if !reflect.DeepEqual(ifLocal, c.expectedJumpIfOutput) { + t.Errorf("IfLocal, expected: '%v', but got: '%v'", c.expectedJumpIfOutput, ifLocal) + } + + if !reflect.DeepEqual(ifNotLocal, c.expectedJumpIfNotOutput) { + t.Errorf("IfNotLocal, expected: '%v', but got: '%v'", c.expectedJumpIfNotOutput, ifNotLocal) + } + } +} + +func TestDetectLocalByInterfaceNamePrefix(t *testing.T) { + cases := []struct { + ifacePrefix string + chain string + args []string + expectedJumpIfOutput []string + expectedJumpIfNotOutput []string + }{ + { + ifacePrefix: "eth0", + expectedJumpIfOutput: []string{"-i", "eth0+"}, + expectedJumpIfNotOutput: []string{"!", "-i", "eth0+"}, + }, + } + for _, c := range cases { + localDetector, err := NewDetectLocalByInterfaceNamePrefix(c.ifacePrefix) + if err != nil { + t.Errorf("Error initializing localDetector: %v", err) + continue + } + if !localDetector.IsImplemented() { + t.Error("DetectLocalByInterfaceNamePrefix returns false for IsImplemented") + } + + ifLocal := localDetector.IfLocal() + ifNotLocal := localDetector.IfNotLocal() + + if !reflect.DeepEqual(ifLocal, c.expectedJumpIfOutput) { + t.Errorf("IfLocal, expected: '%v', but got: '%v'", c.expectedJumpIfOutput, ifLocal) + } + + if !reflect.DeepEqual(ifNotLocal, c.expectedJumpIfNotOutput) { + t.Errorf("IfNotLocal, expected: '%v', but got: '%v'", c.expectedJumpIfNotOutput, ifNotLocal) + } + } +} From 7d480d8ac8e33330af8c8ea863d19d9b547f3bdb Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Sat, 29 Jan 2022 12:01:09 +0100 Subject: [PATCH 2/3] Enable local traffic detection using the interface options This commit adds the framework for the new local detection modes BridgeInterface and InterfaceNamePrefix to work. Signed-off-by: Surya Seetharaman --- cmd/kube-proxy/app/server.go | 2 + cmd/kube-proxy/app/server_others.go | 19 +++- cmd/kube-proxy/app/server_others_test.go | 94 +++++++++++++++++++ cmd/kube-proxy/app/server_test.go | 9 +- pkg/generated/openapi/zz_generated.openapi.go | 42 ++++++++- .../after/v1alpha1.yaml | 3 + .../roundtrip/default/v1alpha1.yaml | 3 + pkg/proxy/apis/config/types.go | 20 +++- .../v1alpha1/zz_generated.conversion.go | 38 ++++++++ .../apis/config/zz_generated.deepcopy.go | 17 ++++ .../kube-proxy/config/v1alpha1/types.go | 14 +++ .../config/v1alpha1/zz_generated.deepcopy.go | 17 ++++ 12 files changed, 272 insertions(+), 6 deletions(-) diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 6a14314d56ff..82fed5143acd 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -211,6 +211,8 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.Float32Var(&o.config.ClientConnection.QPS, "kube-api-qps", o.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver") fs.Var(&o.config.DetectLocalMode, "detect-local-mode", "Mode to use to detect local traffic. This parameter is ignored if a config file is specified by --config.") + fs.StringVar(&o.config.DetectLocal.BridgeInterface, "pod-bridge-interface", o.config.DetectLocal.BridgeInterface, "A bridge interface name in the cluster. Kube-proxy considers traffic as local if originating from an interface which matches the value. This argument should be set if DetectLocalMode is set to BridgeInterface.") + fs.StringVar(&o.config.DetectLocal.InterfaceNamePrefix, "pod-interface-name-prefix", o.config.DetectLocal.InterfaceNamePrefix, "An interface prefix in the cluster. Kube-proxy considers traffic as local if originating from interfaces that match the given prefix. This argument should be set if DetectLocalMode is set to InterfaceNamePrefix.") } // NewOptions returns initialized Options diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go index c0da5b56d50d..8a5d580f93d0 100644 --- a/cmd/kube-proxy/app/server_others.go +++ b/cmd/kube-proxy/app/server_others.go @@ -436,7 +436,7 @@ func detectNumCPU() int { func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) { mode := config.DetectLocalMode switch mode { - case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR: + case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR, proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: return mode, nil default: if strings.TrimSpace(mode.String()) != "" { @@ -461,6 +461,16 @@ func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.Kube break } return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) + case proxyconfigapi.LocalModeBridgeInterface: + if len(strings.TrimSpace(config.DetectLocal.BridgeInterface)) == 0 { + return nil, fmt.Errorf("Detect-local-mode set to BridgeInterface, but no bridge-interface-name %s is defined", config.DetectLocal.BridgeInterface) + } + return proxyutiliptables.NewDetectLocalByBridgeInterface(config.DetectLocal.BridgeInterface) + case proxyconfigapi.LocalModeInterfaceNamePrefix: + if len(strings.TrimSpace(config.DetectLocal.InterfaceNamePrefix)) == 0 { + return nil, fmt.Errorf("Detect-local-mode set to InterfaceNamePrefix, but no interface-prefix %s is defined", config.DetectLocal.InterfaceNamePrefix) + } + return proxyutiliptables.NewDetectLocalByInterfaceNamePrefix(config.DetectLocal.InterfaceNamePrefix) } klog.V(0).InfoS("Defaulting to no-op detect-local", "detect-local-mode", string(mode)) return proxyutiliptables.NewNoOpLocalDetector(), nil @@ -518,6 +528,13 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy } } return localDetectors, err + case proxyconfigapi.LocalModeBridgeInterface, proxyconfigapi.LocalModeInterfaceNamePrefix: + localDetector, err := getLocalDetector(mode, config, ipt[0], nodeInfo) + if err == nil { + localDetectors[0] = localDetector + localDetectors[1] = localDetector + } + return localDetectors, err default: klog.InfoS("Unknown detect-local-mode", "detect-local-mode", mode) } diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go index bc3b270b7203..9e20d10d7861 100644 --- a/cmd/kube-proxy/app/server_others_test.go +++ b/cmd/kube-proxy/app/server_others_test.go @@ -196,6 +196,16 @@ func Test_getDetectLocalMode(t *testing.T) { expected: proxyconfigapi.LocalModeClusterCIDR, errExpected: false, }, + { + detectLocal: string(proxyconfigapi.LocalModeInterfaceNamePrefix), + expected: proxyconfigapi.LocalModeInterfaceNamePrefix, + errExpected: false, + }, + { + detectLocal: string(proxyconfigapi.LocalModeBridgeInterface), + expected: proxyconfigapi.LocalModeBridgeInterface, + errExpected: false, + }, { detectLocal: "abcd", expected: proxyconfigapi.LocalMode("abcd"), @@ -450,6 +460,54 @@ func Test_getLocalDetector(t *testing.T) { expected: proxyutiliptables.NewNoOpLocalDetector(), errExpected: false, }, + // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: "eth"}, + }, + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: ""}, + }, + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: "1234567890123456789"}, + }, + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByBridgeInterface("1234567890123456789")), + errExpected: false, + }, + // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: "eth"}, + }, + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("eth")), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: ""}, + }, + errExpected: true, + }, + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: "1234567890123456789"}, + }, + expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("1234567890123456789")), + errExpected: false, + }, } for i, c := range cases { r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) @@ -587,6 +645,42 @@ func Test_getDualStackLocalDetectorTuple(t *testing.T) { expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, errExpected: false, }, + // LocalModeBridgeInterface, nodeInfo and ipt are not needed for these cases + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: "eth"}, + }, + expected: resolveDualStackLocalDetectors(t)( + proxyutiliptables.NewDetectLocalByBridgeInterface("eth"))( + proxyutiliptables.NewDetectLocalByBridgeInterface("eth")), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeBridgeInterface, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{BridgeInterface: ""}, + }, + errExpected: true, + }, + // LocalModeInterfaceNamePrefix, nodeInfo and ipt are not needed for these cases + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: "veth"}, + }, + expected: resolveDualStackLocalDetectors(t)( + proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("veth"))( + proxyutiliptables.NewDetectLocalByInterfaceNamePrefix("veth")), + errExpected: false, + }, + { + mode: proxyconfigapi.LocalModeInterfaceNamePrefix, + config: &proxyconfigapi.KubeProxyConfiguration{ + DetectLocal: proxyconfigapi.DetectLocalConfiguration{InterfaceNamePrefix: ""}, + }, + errExpected: true, + }, } for i, c := range cases { r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index 5a6787ace62c..e947de110ae6 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -122,6 +122,9 @@ oomScoreAdj: 17 portRange: "2-7" udpIdleTimeout: 123ms detectLocalMode: "ClusterCIDR" +detectLocal: + bridgeInterface: "cbr0" + interfaceNamePrefix: "veth" nodePortAddresses: - "10.20.30.40/16" - "fd00:1::0/64" @@ -263,6 +266,10 @@ nodePortAddresses: UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond}, NodePortAddresses: []string{"10.20.30.40/16", "fd00:1::0/64"}, DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR, + DetectLocal: kubeproxyconfig.DetectLocalConfiguration{ + BridgeInterface: string("cbr0"), + InterfaceNamePrefix: string("veth"), + }, } options := NewOptions() @@ -450,7 +457,7 @@ mode: "" nodePortAddresses: null oomScoreAdj: -999 portRange: "" -detectLocalMode: "ClusterCIDR" +detectLocalMode: "BridgeInterface" udpIdleTimeout: 250ms`) if err != nil { return nil, "", fmt.Errorf("unexpected error when writing content to temp kube-proxy config file: %v", err) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index e7c13ac71816..6a09f8bbaac0 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -991,6 +991,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/kube-controller-manager/config/v1alpha1.StatefulSetControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_StatefulSetControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.TTLAfterFinishedControllerConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_TTLAfterFinishedControllerConfiguration(ref), "k8s.io/kube-controller-manager/config/v1alpha1.VolumeConfiguration": schema_k8sio_kube_controller_manager_config_v1alpha1_VolumeConfiguration(ref), + "k8s.io/kube-proxy/config/v1alpha1.DetectLocalConfiguration": schema_k8sio_kube_proxy_config_v1alpha1_DetectLocalConfiguration(ref), "k8s.io/kube-proxy/config/v1alpha1.KubeProxyConfiguration": schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyConfiguration(ref), "k8s.io/kube-proxy/config/v1alpha1.KubeProxyConntrackConfiguration": schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyConntrackConfiguration(ref), "k8s.io/kube-proxy/config/v1alpha1.KubeProxyIPTablesConfiguration": schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyIPTablesConfiguration(ref), @@ -49371,6 +49372,36 @@ func schema_k8sio_kube_controller_manager_config_v1alpha1_VolumeConfiguration(re } } +func schema_k8sio_kube_proxy_config_v1alpha1_DetectLocalConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "DetectLocalConfiguration contains optional settings related to DetectLocalMode option", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "bridgeInterface": { + SchemaProps: spec.SchemaProps{ + Description: "BridgeInterface is a string argument which represents a single bridge interface name. Kube-proxy considers traffic as local if originating from this given bridge. This argument should be set if DetectLocalMode is set to LocalModeBridgeInterface.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "interfaceNamePrefix": { + SchemaProps: spec.SchemaProps{ + Description: "InterfaceNamePrefix is a string argument which represents a single interface prefix name. Kube-proxy considers traffic as local if originating from one or more interfaces which match the given prefix. This argument should be set if DetectLocalMode is set to LocalModeInterfaceNamePrefix.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"bridgeInterface", "interfaceNamePrefix"}, + }, + }, + } +} + func schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -49567,12 +49598,19 @@ func schema_k8sio_kube_proxy_config_v1alpha1_KubeProxyConfiguration(ref common.R Format: "", }, }, + "detectLocal": { + SchemaProps: spec.SchemaProps{ + Description: "DetectLocal contains optional configuration settings related to DetectLocalMode.", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/kube-proxy/config/v1alpha1.DetectLocalConfiguration"), + }, + }, }, - Required: []string{"bindAddress", "healthzBindAddress", "metricsBindAddress", "bindAddressHardFail", "enableProfiling", "clusterCIDR", "hostnameOverride", "clientConnection", "iptables", "ipvs", "oomScoreAdj", "mode", "portRange", "udpIdleTimeout", "conntrack", "configSyncPeriod", "nodePortAddresses", "winkernel", "showHiddenMetricsForVersion", "detectLocalMode"}, + Required: []string{"bindAddress", "healthzBindAddress", "metricsBindAddress", "bindAddressHardFail", "enableProfiling", "clusterCIDR", "hostnameOverride", "clientConnection", "iptables", "ipvs", "oomScoreAdj", "mode", "portRange", "udpIdleTimeout", "conntrack", "configSyncPeriod", "nodePortAddresses", "winkernel", "showHiddenMetricsForVersion", "detectLocalMode", "detectLocal"}, }, }, Dependencies: []string{ - "k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/component-base/config/v1alpha1.ClientConnectionConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyConntrackConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyIPTablesConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyIPVSConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyWinkernelConfiguration"}, + "k8s.io/apimachinery/pkg/apis/meta/v1.Duration", "k8s.io/component-base/config/v1alpha1.ClientConnectionConfiguration", "k8s.io/kube-proxy/config/v1alpha1.DetectLocalConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyConntrackConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyIPTablesConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyIPVSConfiguration", "k8s.io/kube-proxy/config/v1alpha1.KubeProxyWinkernelConfiguration"}, } } diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml index a4b8d20d4e97..d4084f9e6a69 100644 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/after/v1alpha1.yaml @@ -14,6 +14,9 @@ conntrack: min: 131072 tcpCloseWaitTimeout: 1h0m0s tcpEstablishedTimeout: 24h0m0s +detectLocal: + bridgeInterface: "" + interfaceNamePrefix: "" detectLocalMode: "" enableProfiling: false healthzBindAddress: 0.0.0.0:10256 diff --git a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml index a4b8d20d4e97..d4084f9e6a69 100644 --- a/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml +++ b/pkg/proxy/apis/config/scheme/testdata/KubeProxyConfiguration/roundtrip/default/v1alpha1.yaml @@ -14,6 +14,9 @@ conntrack: min: 131072 tcpCloseWaitTimeout: 1h0m0s tcpEstablishedTimeout: 24h0m0s +detectLocal: + bridgeInterface: "" + interfaceNamePrefix: "" detectLocalMode: "" enableProfiling: false healthzBindAddress: 0.0.0.0:10256 diff --git a/pkg/proxy/apis/config/types.go b/pkg/proxy/apis/config/types.go index 817a79d79ff0..0f162043d90e 100644 --- a/pkg/proxy/apis/config/types.go +++ b/pkg/proxy/apis/config/types.go @@ -107,6 +107,18 @@ type KubeProxyWinkernelConfiguration struct { ForwardHealthCheckVip bool } +// DetectLocalConfiguration contains optional settings related to DetectLocalMode option +type DetectLocalConfiguration struct { + // BridgeInterface is a string argument which represents a single bridge interface name. + // Kube-proxy considers traffic as local if originating from this given bridge. + // This argument should be set if DetectLocalMode is set to BridgeInterface. + BridgeInterface string + // InterfaceNamePrefix is a string argument which represents a single interface prefix name. + // Kube-proxy considers traffic as local if originating from one or more interfaces which match + // the given prefix. This argument should be set if DetectLocalMode is set to InterfaceNamePrefix. + InterfaceNamePrefix string +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeProxyConfiguration contains everything necessary to configure the @@ -174,6 +186,8 @@ type KubeProxyConfiguration struct { ShowHiddenMetricsForVersion string // DetectLocalMode determines mode to use for detecting local traffic, defaults to LocalModeClusterCIDR DetectLocalMode LocalMode + // DetectLocal contains optional configuration settings related to DetectLocalMode. + DetectLocal DetectLocalConfiguration } // ProxyMode represents modes used by the Kubernetes proxy server. Currently, three modes of proxy are available in @@ -204,8 +218,10 @@ type LocalMode string // Currently supported modes for LocalMode const ( - LocalModeClusterCIDR LocalMode = "ClusterCIDR" - LocalModeNodeCIDR LocalMode = "NodeCIDR" + LocalModeClusterCIDR LocalMode = "ClusterCIDR" + LocalModeNodeCIDR LocalMode = "NodeCIDR" + LocalModeBridgeInterface LocalMode = "BridgeInterface" + LocalModeInterfaceNamePrefix LocalMode = "InterfaceNamePrefix" ) // IPVSSchedulerMethod is the algorithm for allocating TCP connections and diff --git a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go index f8bf726b4900..3f4e9eed6a9d 100644 --- a/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go +++ b/pkg/proxy/apis/config/v1alpha1/zz_generated.conversion.go @@ -39,6 +39,16 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*v1alpha1.DetectLocalConfiguration)(nil), (*config.DetectLocalConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration(a.(*v1alpha1.DetectLocalConfiguration), b.(*config.DetectLocalConfiguration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.DetectLocalConfiguration)(nil), (*v1alpha1.DetectLocalConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration(a.(*config.DetectLocalConfiguration), b.(*v1alpha1.DetectLocalConfiguration), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*v1alpha1.KubeProxyConfiguration)(nil), (*config.KubeProxyConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_KubeProxyConfiguration_To_config_KubeProxyConfiguration(a.(*v1alpha1.KubeProxyConfiguration), b.(*config.KubeProxyConfiguration), scope) }); err != nil { @@ -92,6 +102,28 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } +func autoConvert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration(in *v1alpha1.DetectLocalConfiguration, out *config.DetectLocalConfiguration, s conversion.Scope) error { + out.BridgeInterface = in.BridgeInterface + out.InterfaceNamePrefix = in.InterfaceNamePrefix + return nil +} + +// Convert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration(in *v1alpha1.DetectLocalConfiguration, out *config.DetectLocalConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration(in, out, s) +} + +func autoConvert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration(in *config.DetectLocalConfiguration, out *v1alpha1.DetectLocalConfiguration, s conversion.Scope) error { + out.BridgeInterface = in.BridgeInterface + out.InterfaceNamePrefix = in.InterfaceNamePrefix + return nil +} + +// Convert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration is an autogenerated conversion function. +func Convert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration(in *config.DetectLocalConfiguration, out *v1alpha1.DetectLocalConfiguration, s conversion.Scope) error { + return autoConvert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration(in, out, s) +} + func autoConvert_v1alpha1_KubeProxyConfiguration_To_config_KubeProxyConfiguration(in *v1alpha1.KubeProxyConfiguration, out *config.KubeProxyConfiguration, s conversion.Scope) error { out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.BindAddress = in.BindAddress @@ -124,6 +156,9 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_config_KubeProxyConfiguratio } out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion out.DetectLocalMode = config.LocalMode(in.DetectLocalMode) + if err := Convert_v1alpha1_DetectLocalConfiguration_To_config_DetectLocalConfiguration(&in.DetectLocal, &out.DetectLocal, s); err != nil { + return err + } return nil } @@ -164,6 +199,9 @@ func autoConvert_config_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguratio } out.ShowHiddenMetricsForVersion = in.ShowHiddenMetricsForVersion out.DetectLocalMode = v1alpha1.LocalMode(in.DetectLocalMode) + if err := Convert_config_DetectLocalConfiguration_To_v1alpha1_DetectLocalConfiguration(&in.DetectLocal, &out.DetectLocal, s); err != nil { + return err + } return nil } diff --git a/pkg/proxy/apis/config/zz_generated.deepcopy.go b/pkg/proxy/apis/config/zz_generated.deepcopy.go index 82d3e3cafea9..89adbaae11bc 100644 --- a/pkg/proxy/apis/config/zz_generated.deepcopy.go +++ b/pkg/proxy/apis/config/zz_generated.deepcopy.go @@ -48,6 +48,22 @@ func (in ConfigurationMap) DeepCopy() ConfigurationMap { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DetectLocalConfiguration) DeepCopyInto(out *DetectLocalConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DetectLocalConfiguration. +func (in *DetectLocalConfiguration) DeepCopy() *DetectLocalConfiguration { + if in == nil { + return nil + } + out := new(DetectLocalConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { *out = *in @@ -76,6 +92,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { copy(*out, *in) } out.Winkernel = in.Winkernel + out.DetectLocal = in.DetectLocal return } diff --git a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go index 06d933d68111..ba02859006a0 100644 --- a/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-proxy/config/v1alpha1/types.go @@ -103,6 +103,18 @@ type KubeProxyWinkernelConfiguration struct { ForwardHealthCheckVip bool `json:"forwardHealthCheckVip"` } +// DetectLocalConfiguration contains optional settings related to DetectLocalMode option +type DetectLocalConfiguration struct { + // BridgeInterface is a string argument which represents a single bridge interface name. + // Kube-proxy considers traffic as local if originating from this given bridge. + // This argument should be set if DetectLocalMode is set to LocalModeBridgeInterface. + BridgeInterface string `json:"bridgeInterface"` + // InterfaceNamePrefix is a string argument which represents a single interface prefix name. + // Kube-proxy considers traffic as local if originating from one or more interfaces which match + // the given prefix. This argument should be set if DetectLocalMode is set to LocalModeInterfaceNamePrefix. + InterfaceNamePrefix string `json:"interfaceNamePrefix"` +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeProxyConfiguration contains everything necessary to configure the @@ -170,6 +182,8 @@ type KubeProxyConfiguration struct { ShowHiddenMetricsForVersion string `json:"showHiddenMetricsForVersion"` // DetectLocalMode determines mode to use for detecting local traffic, defaults to LocalModeClusterCIDR DetectLocalMode LocalMode `json:"detectLocalMode"` + // DetectLocal contains optional configuration settings related to DetectLocalMode. + DetectLocal DetectLocalConfiguration `json:"detectLocal"` } // ProxyMode represents modes used by the Kubernetes proxy server. diff --git a/staging/src/k8s.io/kube-proxy/config/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/kube-proxy/config/v1alpha1/zz_generated.deepcopy.go index dafffdaf89cf..c7c8b5f8e60c 100644 --- a/staging/src/k8s.io/kube-proxy/config/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kube-proxy/config/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DetectLocalConfiguration) DeepCopyInto(out *DetectLocalConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DetectLocalConfiguration. +func (in *DetectLocalConfiguration) DeepCopy() *DetectLocalConfiguration { + if in == nil { + return nil + } + out := new(DetectLocalConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { *out = *in @@ -54,6 +70,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) { copy(*out, *in) } out.Winkernel = in.Winkernel + out.DetectLocal = in.DetectLocal return } From 1ea5f9432cf4e4a351f115b2f56394ae947aad5d Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Sat, 29 Jan 2022 13:21:43 +0100 Subject: [PATCH 3/3] Add validation for bridge-interface and interface-name-prefix Co-authored-by: Will Daly Signed-off-by: Surya Seetharaman --- .../apis/config/validation/validation.go | 14 +++ .../apis/config/validation/validation_test.go | 98 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/pkg/proxy/apis/config/validation/validation.go b/pkg/proxy/apis/config/validation/validation.go index 062b231c7ec9..aacc00cfb4ec 100644 --- a/pkg/proxy/apis/config/validation/validation.go +++ b/pkg/proxy/apis/config/validation/validation.go @@ -102,6 +102,12 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { allErrs = append(allErrs, validateKubeProxyNodePortAddress(config.NodePortAddresses, newPath.Child("NodePortAddresses"))...) allErrs = append(allErrs, validateShowHiddenMetricsVersion(config.ShowHiddenMetricsForVersion, newPath.Child("ShowHiddenMetricsForVersion"))...) + if config.DetectLocalMode == kubeproxyconfig.LocalModeBridgeInterface { + allErrs = append(allErrs, validateInterface(config.DetectLocal.BridgeInterface, newPath.Child("InterfaceName"))...) + } + if config.DetectLocalMode == kubeproxyconfig.LocalModeInterfaceNamePrefix { + allErrs = append(allErrs, validateInterface(config.DetectLocal.InterfaceNamePrefix, newPath.Child("InterfacePrefix"))...) + } return allErrs } @@ -317,3 +323,11 @@ func validateShowHiddenMetricsVersion(version string, fldPath *field.Path) field return allErrs } + +func validateInterface(iface string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(iface) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath, iface, "must not be empty")) + } + return allErrs +} diff --git a/pkg/proxy/apis/config/validation/validation_test.go b/pkg/proxy/apis/config/validation/validation_test.go index 1cb0a92d9af8..23c56614bbe0 100644 --- a/pkg/proxy/apis/config/validation/validation_test.go +++ b/pkg/proxy/apis/config/validation/validation_test.go @@ -175,6 +175,52 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, }, }, + { + BindAddress: "10.10.12.11", + HealthzBindAddress: "0.0.0.0:12345", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32Ptr(1), + Min: pointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + DetectLocalMode: kubeproxyconfig.LocalModeInterfaceNamePrefix, + DetectLocal: kubeproxyconfig.DetectLocalConfiguration{ + InterfaceNamePrefix: "vethabcde", + }, + }, + { + BindAddress: "10.10.12.11", + HealthzBindAddress: "0.0.0.0:12345", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32Ptr(1), + Min: pointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + DetectLocalMode: kubeproxyconfig.LocalModeBridgeInterface, + DetectLocal: kubeproxyconfig.DetectLocalConfiguration{ + BridgeInterface: "avz", + }, + }, } for _, successCase := range successCases { @@ -366,6 +412,58 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { }, expectedErrs: field.ErrorList{field.Invalid(newPath.Child("KubeProxyIPVSConfiguration.SyncPeriod"), metav1.Duration{Duration: 0}, "must be greater than 0")}, }, + "interfacePrefix is empty": { + config: kubeproxyconfig.KubeProxyConfiguration{ + BindAddress: "10.10.12.11", + HealthzBindAddress: "0.0.0.0:12345", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32Ptr(1), + Min: pointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + DetectLocalMode: kubeproxyconfig.LocalModeInterfaceNamePrefix, + DetectLocal: kubeproxyconfig.DetectLocalConfiguration{ + InterfaceNamePrefix: "", + }, + }, + expectedErrs: field.ErrorList{field.Invalid(newPath.Child("InterfacePrefix"), "", "must not be empty")}, + }, + "bridgeInterfaceName is empty": { + config: kubeproxyconfig.KubeProxyConfiguration{ + BindAddress: "10.10.12.11", + HealthzBindAddress: "0.0.0.0:12345", + MetricsBindAddress: "127.0.0.1:10249", + ClusterCIDR: "192.168.59.0/24", + UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second}, + ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ + MaxPerCore: pointer.Int32Ptr(1), + Min: pointer.Int32Ptr(1), + TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, + TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, + }, + DetectLocalMode: kubeproxyconfig.LocalModeBridgeInterface, + DetectLocal: kubeproxyconfig.DetectLocalConfiguration{ + InterfaceNamePrefix: "eth0", // we won't care about prefix since mode is not prefix + }, + }, + expectedErrs: field.ErrorList{field.Invalid(newPath.Child("InterfaceName"), "", "must not be empty")}, + }, } for name, testCase := range testCases {