enable kubelet server to dynamically load tls certificate files

pkg/kubelet/certificate/kubelet.go

@@ -17,24 +17,30 @@ limitations under the License.
 package certificate
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
 	"math"
 	"net"
 	"sort"
+	"sync/atomic"
 	"time"
 
+	"k8s.io/klog/v2"
+
 	certificates "k8s.io/api/certificates/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/certificate"
 	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
+
 	netutils "k8s.io/utils/net"
 )
 
@@ -234,3 +240,70 @@ func NewKubeletClientCertificateManager(
 
 	return m, nil
 }
+
+// NewKubeletServerCertificateDynamicFileManager creates a certificate manager based on reading and watching certificate and key files.
+// The returned struct implements certificate.Manager interface, enabling using it like other CertificateManager in this package.
+// But the struct doesn't communicate with API server to perform certificate request at all.
+func NewKubeletServerCertificateDynamicFileManager(certFile, keyFile string) (certificate.Manager, error) {
+	c, err := dynamiccertificates.NewDynamicServingContentFromFiles("kubelet-server-cert-files", certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("no certificate available: %w", err)
+	}
+	m := &kubeletServerCertificateDynamicFileManager{
+		dynamicCertificateContent: c,
+		certFile:                  certFile,
+		keyFile:                   keyFile,
+	}
+	c.AddListener(m)
+	return m, nil
+}
+
+// kubeletServerCertificateDynamicFileManager uses a dynamic CertKeyContentProvider based on cert and key files.
+type kubeletServerCertificateDynamicFileManager struct {
+	ctx                       context.Context
+	cancelFn                  context.CancelFunc
+	certFile                  string
+	keyFile                   string
+	dynamicCertificateContent *dynamiccertificates.DynamicCertKeyPairContent
+	currentTLSCertificate     atomic.Pointer[tls.Certificate]
+}
+
+// Enqueue implements the functions to be notified when the serving cert content changes.
+func (m *kubeletServerCertificateDynamicFileManager) Enqueue() {
+	m.currentTLSCertificate.Store(nil)
+}
+
+// Current returns the last valid certificate key pair loaded from files.
+func (m *kubeletServerCertificateDynamicFileManager) Current() *tls.Certificate {
+	current := m.currentTLSCertificate.Load()
+	if current != nil {
+		return current
+	}
+
+	certContent, keyContent := m.dynamicCertificateContent.CurrentCertKeyContent()
+	cert, err := tls.X509KeyPair(certContent, keyContent)
+	if err != nil {
+		klog.ErrorS(err, "invalid certificate and key pair from file", "certFile", m.certFile, "keyFile", m.keyFile)
+		return nil
+	}
+	m.currentTLSCertificate.Store(&cert)
+	return &cert
+}
+
+// Start starts watching the certificate and key files
+func (m *kubeletServerCertificateDynamicFileManager) Start() {
+	m.ctx, m.cancelFn = context.WithCancel(context.Background())
+	go m.dynamicCertificateContent.Run(m.ctx, 1)
+}
+
+// Stop stops watching the certificate and key files
+func (m *kubeletServerCertificateDynamicFileManager) Stop() {
+	if m.cancelFn != nil {
+		m.cancelFn()
+	}
+}
+
+// ServerHealthy always return true since the file manager doesn't communicate with any server
+func (m *kubeletServerCertificateDynamicFileManager) ServerHealthy() bool {
+	return true
+}

pkg/kubelet/certificate/kubelet_test.go

@@ -17,9 +17,13 @@ limitations under the License.
 package certificate
 
 import (
+	"bytes"
 	"net"
+	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
 	netutils "k8s.io/utils/net"
@@ -100,3 +104,199 @@ func TestAddressesToHostnamesAndIPs(t *testing.T) {
 		})
 	}
 }
+
+const cert1 = `
+-----BEGIN CERTIFICATE-----
+MIIDOTCCAiGgAwIBAgIRAP6V9wv0uHGm/1vETx8fm6MwDQYJKoZIhvcNAQELBQAw
+EjEQMA4GA1UEAxMHdGVzdC1jYTAgFw0yNDA0MjYwMjEwMDZaGA8yMTI0MDQwMjAy
+MTAwNlowMjEUMBIGA1UEChMLc3lzdGVtOm5vZGUxGjAYBgNVBAMTEXN5c3RlbTpu
+b2RlOmhvc3QxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6gUPdv4o
+tUNyrNq1R/W4jCOUVb9V8qdVu0CRXeRQA1aQXIOofcYM6hLU4fFi+e7kF3OoPzhL
+CeRK+I6qUkahnclTHkIXA796U0zXl9IFI5c4BBQr2wtyY/LG4EOcKC844FCnG9X8
+LR4Qj8zpeuAg80hy0B+3jdIbqcoHLA15LOqBLPHyyqWXguAQgUfkNQbMZ5Qm+Jh/
+/TdkC4wsTiayx6bOBwIsspqDi+tx6aYd0p6lBmtCX5pZS1yfSfbn7wPO6dfD5eFQ
+BNqUKdMoYdguWmrjI3vhsJtCQ5HSWQyliXPnsZdaqRsFFZtrxkIKVjfd/sw/GUH7
+yBnUamj7Y9ypnwIDAQABo2gwZjAOBgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYI
+KwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUyo3AFYI2nr1+3zDo
+AaFTO0637hkwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAnOls
+zr503G78g1qzIDELYuC7BlrNk6IaACuCTZNSYW/QIXDHLraI907yVWa5P6owyVYP
+glcJ+erGY6placRUbyH3LDX9WucnPgqrPsFE6/XOY3VzzTE3XzDYm4WBzm/lUrai
+bwN2i7XHmvWBLP4aDBLYGhPqjar1vNV0sJV8vxY9oe09MydSan1gXmqOvqdVSL0a
+yhnYhbSzzxOu1WUijUq3oaYZDwdCdGNWZtCYPbGxaA19hacQkJpEPZujlCvTI1a9
+VuXQdkw656fF/frUJ+p6tLW6V3L1MSxB4jwgkG5pHkl81AYokgUi9PjZOiaMbVn5
+QdLGrYk97ZSGGYMa0Q==
+-----END CERTIFICATE-----
+`
+
+const key1 = `
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDqBQ92/ii1Q3Ks
+2rVH9biMI5RVv1Xyp1W7QJFd5FADVpBcg6h9xgzqEtTh8WL57uQXc6g/OEsJ5Er4
+jqpSRqGdyVMeQhcDv3pTTNeX0gUjlzgEFCvbC3Jj8sbgQ5woLzjgUKcb1fwtHhCP
+zOl64CDzSHLQH7eN0hupygcsDXks6oEs8fLKpZeC4BCBR+Q1BsxnlCb4mH/9N2QL
+jCxOJrLHps4HAiyymoOL63Hpph3SnqUGa0JfmllLXJ9J9ufvA87p18Pl4VAE2pQp
+0yhh2C5aauMje+Gwm0JDkdJZDKWJc+exl1qpGwUVm2vGQgpWN93+zD8ZQfvIGdRq
+aPtj3KmfAgMBAAECggEAKsZQBFUChddVP6keV4/fcqYSN/YoNJlHf1mW+I2B3opV
+CsP56TtpIuPcS4w+piZ3RJ4cU6nrdVxoI7SYBz/nzJp++dnksQevyUgTZCm8TLwY
+Pg6d0YTvHLvEhDt3cJTpFX9IfDsJxAlpx48R4ibTfwRwEACsIV0VN1y5IOE5k+7T
+uiNATGM9kLxAAlMMMrwcyX2XOCPNyXDK8GtZMGdwDfT0tPs5Kg8E+/XXHict+6wO
+aTLKkLSXzFtPujnMl2HAGJaui6jJEU5be51XTyCWyPeTyXYGBgnZsmd9SMU8udYG
+Yx8BG5jPDi7eRy6FtQbv9mtbsPdS9rgi3QVoC76kKQKBgQDuuf9/4AXSOY08JnI4
++3s4Ri2/3iQ1xVUGAvuquvhKI8IPruwagSrz8VkzSWeQMZzGAiYrEaN+KUA1DL3k
+W+477MlN7jgHNsSvG5xzk7SQxx2o+Hug+yOS/wLpMgmGU4xKciJ60KPIiO5JPMIx
+ukikmRgk7hrdV969SwoHR04p7QKBgQD68+CNYZVdNiL2657BmsaCCbaSEHPHUT0p
+13LJr2CT0nBSrf+AHj2uRW8cBuGxypgKoP+erXIBN4sijNw0bJaZtomufFhWgG/D
+YDeKlojfNuMKkxSb1RPYoteI//pkXVxFtdIWsP7tPi7o1cabZD/8A05KKQlqQsXs
+a6MlaR0AOwKBgQDTQ8t20TyVlNUGnEeNYhDj9kdaey60X1QlI9KwfxJoGkkNNBJC
+SnlGtRnpp1Z9Z0qEeTZp+wwjBEKMMCCEzU6BvcVQsDbpO3DIPrkwF1E+ptf6xxwx
+lM8gsYlT8jI3rAyFfYhCBA5N09B9A8Yf1+mdsNaLKCSiKbc28geH9XSY6QKBgQDg
+/qKHX8RQy+bRRzMRFbmAgUDk+Ec1nsqdpwLNfKW0Iup91m7K6VIX2zzg/fKAOsnS
+TcKg+5TJLolaMryDbBAiRJxwih+Rfpm1q6BgatLQfh9VLcU+ae3fPzDLLeXK8kF2
+ZquzmIEXJ7dbHb9xNpJ/Wl9o8h303WUisOaW9gUAXQKBgCZWPdu++XdAHA3Ec3vj
+rJBbjdLhp5vkqx0wzpYGQOTC45AXuaqqtf0UTGvdZ2AzXMVbXZhVwTYT8HNr+ERo
+uQIpKp7PdYtcKPhYncY+9yT+kTE6X30AtF8BolE7ZwKC8xWl7bLjGhMikk1hQoLJ
+HMdto22GHpusU1sG3ta7wbfV
+-----END PRIVATE KEY-----
+`
+
+const cert2 = `
+-----BEGIN CERTIFICATE-----
+MIIDOzCCAiOgAwIBAgIRAJtXnoadlUNlNmIyqh8ioTAwDQYJKoZIhvcNAQELBQAw
+FDESMBAGA1UEAxMJdGVzdC1jYS0yMCAXDTI0MDQyNjAyMTIyNVoYDzIxMjQwNDAy
+MDIxMjI1WjAyMRQwEgYDVQQKEwtzeXN0ZW06bm9kZTEaMBgGA1UEAxMRc3lzdGVt
+Om5vZGU6aG9zdDEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDLPd24
+HHWDKHOPDoSooyRHdUq5WEE3dWcZm/oH/EBMy5Zb3nR6Fh+wF+2qopyjujQnx3lb
+8OjuTO7iBW1iqH3Nrqml7z5d4i4+/zSf0gh3BiWDyAKDidtwhfQi+wuXCAtphDXE
+doAw5XOydWmLJyyUSQJekN53+9M0dw5lVig1HTxggF7q1k4aCL/S9NcvqFeIFXhX
+YmpXEvSsCdpF8EP2DtA7Eq/I1PjB5BcHoSToY/bWKfEKBAzatzOR28t6Bp4VPEsk
+XUFDeGwovy2KWlOoIplEyNaX0srQq8fIoRG3dttbmnR0ZG6d1Vo0lrru7STs66iB
+UOtEJUHZVj7IWb3ZAgMBAAGjaDBmMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAK
+BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQ2pFokSGOYqQ1e
+PucDrkUcv1rXYDAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBz
+no5lUJQ5bu1j84TGPSZN23o+rPyOdEmikUukaX1HLhwQIeI3VaNgHr+F/OGJ7JyJ
+hl7OE3zRC/OOZ3Etm1vTQn4fUbuTPC45k73OZ5GxG/orwlKx5m86QY+fKPYm9ul9
+q1RluDRmmerMj4QePiJSzsMy5xB8NFDLe0eSM/1HbwBAXAHjLxfFfIuPMpQ6ugll
+APtq+UjevEIjva5Qge+Kj5X/zPcJq9HZV98uVSPMOWSC7GLTkjdS6IwjxeFivU8x
+2BROZ6N+8btxlv8JlZMYHhIRmctM5nPI8bVC12qv4HXQ0wIEbrTgnoc+NiH6gtGY
+n8eN5Tw33FUpkEhqinjR
+-----END CERTIFICATE-----
+`
+
+const key2 = `
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDLPd24HHWDKHOP
+DoSooyRHdUq5WEE3dWcZm/oH/EBMy5Zb3nR6Fh+wF+2qopyjujQnx3lb8OjuTO7i
+BW1iqH3Nrqml7z5d4i4+/zSf0gh3BiWDyAKDidtwhfQi+wuXCAtphDXEdoAw5XOy
+dWmLJyyUSQJekN53+9M0dw5lVig1HTxggF7q1k4aCL/S9NcvqFeIFXhXYmpXEvSs
+CdpF8EP2DtA7Eq/I1PjB5BcHoSToY/bWKfEKBAzatzOR28t6Bp4VPEskXUFDeGwo
+vy2KWlOoIplEyNaX0srQq8fIoRG3dttbmnR0ZG6d1Vo0lrru7STs66iBUOtEJUHZ
+Vj7IWb3ZAgMBAAECggEBAKt1La88kwZrAdIV9WQu/VQrZzaldZ9LtAaux2glLjmb
+JuWp7alxMJpmFWJ2fJ7DX2yPo5okytz+miijW3x3mGoEh6otAvhA77LFqaeKkQmY
+bd2Wxkgh7LYoy6UXFNf9OWNy3ck4Dz0w1UIgO0HhcoJGdXFB1exyzeLc7ZAf9xuW
+ZZK/az4OmqtDd8vXElOkD6b56BbCtGVq4br4P533yesH4DDQpy7rkvCgHdro3wIH
+aeg4AE8LOMmZggBOLorKXvG3+9s5ZXEnHj00oAk0c27/4UgGeMI0DGAayRJfTLW7
+fZZRerVuGRC0KmLYjBT4Vj0Ih/2LSE1Y+ZE0ZUtTQmECgYEA5TtVTfHQWb3V+1N+
+Rxl0O7rg4O0PCKWDjnbqYLbBNKZIrar4ebfqSAYWuhXR6jasRDnBlMH5pt5LD0g4
+bcztKNSYajUaRBjNg6lHsvAp1AKgbLhpUtwXvNe4bjNJFM7yMY6o/ZhFVWwylLQI
+BX9RMWLF7bBxSx6QFko95eZz7P8CgYEA4vmVUC3DWVQ+KZb9hEtw4UKyWs+WMQTy
+KIE8ZJoAweFEqCvvk03DwAy5xlDwtOuyO/dDibMmoB7Jgkr4iB9MA0qJEjkjDihG
+mabUW2K3WTezEZT31EwjAIs8If7zKatSwHZOLTmAam70LSbAy4ZVL3yFf4UbT5Oh
+NsXQ77hoXScCgYBfkaQX8ff2YjHjLUUZaWBPQrNcsxiwdyjo7WT721WjmKv2U7By
+Np4jVv6EqHIy3oZlj1rIpTJrQoQyo9560JQTkMbWiLshpuGPwbSVwpD9xfaSPTQU
+CpSO87T9pL4UQc8xoBOOXryRR6Gy43fwqsrz9wUj+orRUbWqxVsXDURJiwKBgQCa
+VsSlZLj1QUeT2ExDbVkwk73b6lRiuM5BpL+AWQgyzg91m4qpS7PUH9Mje15yZ+Mm
+y5htRhj5wHWd14TwavexNTnH3npr4g8/5CV6jsHGNQ3a4sUy4yLZ99PH+ik3KHx4
+yvmV3wfnV9NJ8JQg0ROT2sScVdKgZe615AWTPH4a+QKBgDA7kyhEYoCltchrEsxk
+ahz4YVmnJVZp1UDy+bLUZYCKNCyGIAkTVNutrxHThx/3HapMjkd6EmefLyNZ6rd9
+iNByAPzy9m6/zVhMdy97RgDpbXR+iuHpCwjiTh7bF3l78fLgStB2UD9jgiNDaiBv
+KSUELgGVOfUpsLO/ZTQqdJEy
+-----END PRIVATE KEY-----
+`
+
+func TestKubeletServerCertificateFromFiles(t *testing.T) {
+	certDir, err := os.MkdirTemp("", "test-kubelet-cert-files")
+	if err != nil {
+		t.Fatalf("Unable to setup cert dir: %v", err)
+	}
+
+	rotateCertErrs := make(chan error, 10)
+
+	defer func() {
+		if err := os.RemoveAll(certDir); err != nil {
+			t.Errorf("Unable to clean up test directory %q: %v", certDir, err)
+		}
+		if len(rotateCertErrs) != 0 {
+			t.Errorf("got errors when rotating certificate files in the test")
+		}
+		close(rotateCertErrs)
+	}()
+
+	certPath := filepath.Join(certDir, "kubelet.cert")
+	keyPath := filepath.Join(certDir, "kubelet.key")
+	createCertFn := func(cert, key []byte) error {
+		if err := os.WriteFile(certPath, cert, os.FileMode(0644)); err != nil {
+			return err
+		}
+		if err := os.WriteFile(keyPath, key, os.FileMode(0600)); err != nil {
+			return err
+		}
+		return nil
+	}
+	err = createCertFn([]byte(cert1), []byte(key1))
+	if err != nil {
+		t.Fatalf("Unable to setup cert: %v", err)
+	}
+
+	// simulate certificate files update in the background
+	go func() {
+		time.Sleep(500 * time.Millisecond)
+		if err := os.Remove(certPath); err != nil {
+			rotateCertErrs <- err
+			return
+		}
+		if err := os.Remove(keyPath); err != nil {
+			rotateCertErrs <- err
+			return
+		}
+		if err := createCertFn([]byte(cert2), []byte(key2)); err != nil {
+			rotateCertErrs <- err
+		}
+	}()
+
+	m, err := NewKubeletServerCertificateDynamicFileManager(certPath, keyPath)
+	if err != nil {
+		t.Fatalf("Unable to create certificte provider: %v", err)
+	}
+
+	m.Start()
+	defer m.Stop()
+
+	c := m.Current()
+	if c == nil {
+		t.Fatal("failed to provide valid certificate")
+	}
+	time.Sleep(100 * time.Millisecond)
+	c2 := m.Current()
+	if c2 == nil {
+		t.Fatal("failed to provide valid certificate")
+	}
+	if c2 != c {
+		t.Errorf("expected the same loaded certificate object when there is no cert file change, got different")
+	}
+
+	time.Sleep(600 * time.Millisecond)
+	c3 := m.Current()
+	if c3 == nil {
+		t.Errorf("failed to provide valid certificate after file update")
+	} else if bytes.Equal(c.Certificate[0], c3.Certificate[0]) {
+		t.Errorf("failed to provide the updated certificate")
+	}
+
+	if err = os.Remove(certPath); err != nil {
+		t.Errorf("could not delete file in order to perform test")
+	}
+
+	if m.Current() == nil {
+		t.Errorf("expected the manager still provide cached content when certificate file was not available")
+	}
+}

pkg/kubelet/kubelet.go

@@ -774,17 +774,28 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	klet.imageManager = imageManager
 
-	if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
-		klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
-		if err != nil {
-			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
+	if kubeDeps.TLSOptions != nil {
+		if kubeCfg.ServerTLSBootstrap && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+			klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory)
+			if err != nil {
+				return nil, fmt.Errorf("failed to initialize certificate manager: %w", err)
+			}
+
+		} else if kubeDeps.TLSOptions.CertFile != "" && kubeDeps.TLSOptions.KeyFile != "" {
+			klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateDynamicFileManager(kubeDeps.TLSOptions.CertFile, kubeDeps.TLSOptions.KeyFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to initialize file based certificate manager: %w", err)
+			}
 		}
-		kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
-			cert := klet.serverCertificateManager.Current()
-			if cert == nil {
-				return nil, fmt.Errorf("no serving certificate available for the kubelet")
+
+		if klet.serverCertificateManager != nil {
+			kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+				cert := klet.serverCertificateManager.Current()
+				if cert == nil {
+					return nil, fmt.Errorf("no serving certificate available for the kubelet")
+				}
+				return cert, nil
 			}
-			return cert, nil
 		}
 	}