diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go
index 8ba056b04e..39b6e5f0cb 100644
--- a/pkg/webhook/server.go
+++ b/pkg/webhook/server.go
@@ -81,10 +81,6 @@ type Server struct {
 	// and to provide better panic messages on duplicate webhook registration.
 	webhooks map[string]http.Handler
 
-	// tlsMinVersion is the result of the conversion from human-readable TLS version (for example "1.1")
-	// to the values accepted by tls.Config (for example 0x301)
-	tlsMinVersion uint16
-
 	// setFields allows injecting dependencies from an external source
 	setFields inject.Func
 
@@ -117,17 +113,6 @@ func (s *Server) setDefaults() {
 	if len(s.KeyName) == 0 {
 		s.KeyName = "tls.key"
 	}
-
-	switch s.TLSMinVersion {
-	case "1.1":
-		s.tlsMinVersion = tls.VersionTLS11
-	case "1.2":
-		s.tlsMinVersion = tls.VersionTLS12
-	case "1.3":
-		s.tlsMinVersion = tls.VersionTLS13
-	default:
-		s.tlsMinVersion = tls.VersionTLS10
-	}
 }
 
 // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
@@ -194,6 +179,26 @@ func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) er
 	return s.Start(ctx)
 }
 
+// tlsMinVersion converts from human-readable TLS version (for example "1.1")
+// to the values accepted by tls.Config (for example 0x301)
+func tlsVersion(version string) (uint16, error) {
+	switch version {
+	// default is previous behaviour
+	case "":
+		return tls.VersionTLS10, nil
+	case "1.0":
+		return tls.VersionTLS10, nil
+	case "1.1":
+		return tls.VersionTLS11, nil
+	case "1.2":
+		return tls.VersionTLS12, nil
+	case "1.3":
+		return tls.VersionTLS13, nil
+	default:
+		return 0, fmt.Errorf("Invalid TLSMinVersion %v: expects 1.0, 1.1, 1.2, 1.3 or empty", version)
+	}
+}
+
 // Start runs the server.
 // It will install the webhook related resources depend on the server configuration.
 func (s *Server) Start(ctx context.Context) error {
@@ -216,10 +221,15 @@ func (s *Server) Start(ctx context.Context) error {
 		}
 	}()
 
+	tlsMinVersion, err := tlsVersion(s.TLSMinVersion)
+	if err != nil {
+		return err
+	}
+
 	cfg := &tls.Config{
 		NextProtos:     []string{"h2"},
 		GetCertificate: certWatcher.GetCertificate,
-		MinVersion:     s.tlsMinVersion,
+		MinVersion:     tlsMinVersion,
 	}
 
 	// load CA to verify client certificate