diff --git a/pkg/resourcemanager/webhook/systemcomponentsconfig/handler.go b/pkg/resourcemanager/webhook/systemcomponentsconfig/handler.go
index 50939f12c34..e5437ed70d5 100644
--- a/pkg/resourcemanager/webhook/systemcomponentsconfig/handler.go
+++ b/pkg/resourcemanager/webhook/systemcomponentsconfig/handler.go
@@ -19,15 +19,14 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/gardener/gardener/pkg/utils"
+	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilsets "k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
-
-	"github.com/gardener/gardener/pkg/utils"
-	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
-	setsutils "github.com/gardener/gardener/pkg/utils/sets"
 )
 
 // Handler contains required nodeSelector and tolerations information.
@@ -82,7 +81,7 @@ func (h *Handler) handleTolerations(ctx context.Context, log logr.Logger, pod *c
 	}
 
 	var (
-		tolerations = setsutils.New[corev1.Toleration]()
+		tolerations = utilsets.New[corev1.Toleration]()
 
 		// We need to use semantically equal tolerations, i.e. equality of underlying values of pointers,
 		// before they are added to the tolerations set.
diff --git a/pkg/utils/gardener/shoot.go b/pkg/utils/gardener/shoot.go
index e38cc1fcb4e..ab43fcf2baa 100644
--- a/pkg/utils/gardener/shoot.go
+++ b/pkg/utils/gardener/shoot.go
@@ -29,6 +29,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilsets "k8s.io/apimachinery/pkg/util/sets"
 	clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest"
 	clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
 	"k8s.io/component-base/version"
@@ -45,7 +46,6 @@ import (
 	"github.com/gardener/gardener/pkg/utils"
 	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/gardener/gardener/pkg/utils/secrets"
-	setsutils "github.com/gardener/gardener/pkg/utils/sets"
 	"github.com/gardener/gardener/pkg/utils/timewindow"
 )
 
@@ -496,7 +496,7 @@ func GetShootSeedNames(obj client.Object) (*string, *string) {
 // on the given workers. Tolerations are only considered for workers which have `SystemComponents.Allow: true`.
 func ExtractSystemComponentsTolerations(workers []gardencorev1beta1.Worker) []corev1.Toleration {
 	var (
-		tolerations = setsutils.New[corev1.Toleration]()
+		tolerations = utilsets.New[corev1.Toleration]()
 
 		// We need to use semantically equal tolerations, i.e. equality of underlying values of pointers,
 		// before they are added to the tolerations set.
diff --git a/pkg/utils/sets/ordered.go b/pkg/utils/sets/ordered.go
deleted file mode 100644
index e6828690762..00000000000
--- a/pkg/utils/sets/ordered.go
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
-Copyright 2022 The Kubernetes Authors.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// TODO(timuthy): Remove this file after sources have been vendored to Kubernetes 1.26, see https://github.com/kubernetes/kubernetes/pull/112377.
-
-package sets
-
-// ordered is a constraint that permits any ordered type: any type
-// that supports the operators < <= >= >.
-// If future releases of Go add new ordered types,
-// this constraint will be modified to include them.
-type ordered interface {
-	integer | float | ~string
-}
-
-// integer is a constraint that permits any integer type.
-// If future releases of Go add new predeclared integer types,
-// this constraint will be modified to include them.
-type integer interface {
-	signed | unsigned
-}
-
-// float is a constraint that permits any floating-point type.
-// If future releases of Go add new predeclared floating-point types,
-// this constraint will be modified to include them.
-type float interface {
-	~float32 | ~float64
-}
-
-// signed is a constraint that permits any signed integer type.
-// If future releases of Go add new predeclared signed integer types,
-// this constraint will be modified to include them.
-type signed interface {
-	~int | ~int8 | ~int16 | ~int32 | ~int64
-}
-
-// unsigned is a constraint that permits any unsigned integer type.
-// If future releases of Go add new predeclared unsigned integer types,
-// this constraint will be modified to include them.
-type unsigned interface {
-	~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr
-}
diff --git a/pkg/utils/sets/set.go b/pkg/utils/sets/set.go
deleted file mode 100644
index 6ea61cafc3f..00000000000
--- a/pkg/utils/sets/set.go
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
-Copyright 2022 The Kubernetes Authors.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package sets
-
-import (
-	"sort"
-
-	"k8s.io/apimachinery/pkg/util/sets"
-)
-
-// TODO(timuthy): Remove this file after sources have been vendored to Kubernetes 1.26, see https://github.com/kubernetes/kubernetes/pull/112377.
-
-// Set is a set of the same type elements, implemented via map[comparable]struct{} for minimal memory consumption.
-type Set[T comparable] map[T]sets.Empty
-
-// cast transforms specified set to generic Set[T].
-func cast[T comparable](s map[T]sets.Empty) Set[T] { return s }
-
-// New creates a Set from a list of values.
-// NOTE: type param must be explicitly instantiated if given items are empty.
-func New[T comparable](items ...T) Set[T] {
-	ss := make(Set[T], len(items))
-	ss.Insert(items...)
-	return ss
-}
-
-// KeySet creates a Set from a keys of a map[comparable](? extends interface{}).
-// If the value passed in is not actually a map, this will panic.
-func KeySet[T comparable, V any](theMap map[T]V) Set[T] {
-	ret := Set[T]{}
-	for keyValue := range theMap {
-		ret.Insert(keyValue)
-	}
-	return ret
-}
-
-// Insert adds items to the set.
-func (s Set[T]) Insert(items ...T) Set[T] {
-	for _, item := range items {
-		s[item] = sets.Empty{}
-	}
-	return s
-}
-
-// Insert adds items to the given set.
-func Insert[T comparable](set Set[T], items ...T) Set[T] {
-	return set.Insert(items...)
-}
-
-// Delete removes all items from the set.
-func (s Set[T]) Delete(items ...T) Set[T] {
-	for _, item := range items {
-		delete(s, item)
-	}
-	return s
-}
-
-// Has returns true if and only if item is contained in the set.
-func (s Set[T]) Has(item T) bool {
-	_, contained := s[item]
-	return contained
-}
-
-// HasAll returns true if and only if all items are contained in the set.
-func (s Set[T]) HasAll(items ...T) bool {
-	for _, item := range items {
-		if !s.Has(item) {
-			return false
-		}
-	}
-	return true
-}
-
-// HasAny returns true if any items are contained in the set.
-func (s Set[T]) HasAny(items ...T) bool {
-	for _, item := range items {
-		if s.Has(item) {
-			return true
-		}
-	}
-	return false
-}
-
-// Clone returns a new set which is a copy of the current set.
-func (s Set[T]) Clone() Set[T] {
-	result := make(Set[T], len(s))
-	for key := range s {
-		result.Insert(key)
-	}
-	return result
-}
-
-// Difference returns a set of objects that are not in s2.
-// For example:
-// s1 = {a1, a2, a3}
-// s2 = {a1, a2, a4, a5}
-// s1.Difference(s2) = {a3}
-// s2.Difference(s1) = {a4, a5}
-func (s Set[T]) Difference(s2 Set[T]) Set[T] {
-	result := New[T]()
-	for key := range s {
-		if !s2.Has(key) {
-			result.Insert(key)
-		}
-	}
-	return result
-}
-
-// SymmetricDifference returns a set of elements which are in either of the sets, but not in their intersection.
-// For example:
-// s1 = {a1, a2, a3}
-// s2 = {a1, a2, a4, a5}
-// s1.SymmetricDifference(s2) = {a3, a4, a5}
-// s2.SymmetricDifference(s1) = {a3, a4, a5}
-func (s Set[T]) SymmetricDifference(s2 Set[T]) Set[T] {
-	return s.Difference(s2).Union(s2.Difference(s))
-}
-
-// Union returns a new set which includes items in either s1 or s2.
-// For example:
-// s1 = {a1, a2}
-// s2 = {a3, a4}
-// s1.Union(s2) = {a1, a2, a3, a4}
-// s2.Union(s1) = {a1, a2, a3, a4}
-func (s Set[T]) Union(s2 Set[T]) Set[T] {
-	result := s.Clone()
-	for key := range s2 {
-		result.Insert(key)
-	}
-	return result
-}
-
-// Intersection returns a new set which includes the item in BOTH s1 and s2
-// For example:
-// s1 = {a1, a2}
-// s2 = {a2, a3}
-// s1.Intersection(s2) = {a2}
-func (s Set[T]) Intersection(s2 Set[T]) Set[T] {
-	var walk, other Set[T]
-	result := New[T]()
-	if s.Len() < s2.Len() {
-		walk = s
-		other = s2
-	} else {
-		walk = s2
-		other = s
-	}
-	for key := range walk {
-		if other.Has(key) {
-			result.Insert(key)
-		}
-	}
-	return result
-}
-
-// IsSuperset returns true if and only if s is a superset of s2.
-func (s Set[T]) IsSuperset(s2 Set[T]) bool {
-	for item := range s2 {
-		if !s.Has(item) {
-			return false
-		}
-	}
-	return true
-}
-
-// Equal returns true if and only if s is equal (as a set) to s2.
-// Two sets are equal if their membership is identical.
-// (In practice, this means same elements, order doesn't matter)
-func (s Set[T]) Equal(s2 Set[T]) bool {
-	return len(s) == len(s2) && s.IsSuperset(s2)
-}
-
-type sortableSliceOfGeneric[T ordered] []T
-
-func (g sortableSliceOfGeneric[T]) Len() int           { return len(g) }
-func (g sortableSliceOfGeneric[T]) Less(i, j int) bool { return less[T](g[i], g[j]) }
-func (g sortableSliceOfGeneric[T]) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
-
-// List returns the contents as a sorted T slice.
-//
-// This is a separate function and not a method because not all types supported
-// by Generic are ordered and only those can be sorted.
-func List[T ordered](s Set[T]) []T {
-	res := make(sortableSliceOfGeneric[T], 0, len(s))
-	for key := range s {
-		res = append(res, key)
-	}
-	sort.Sort(res)
-	return res
-}
-
-// UnsortedList returns the slice with contents in random order.
-func (s Set[T]) UnsortedList() []T {
-	res := make([]T, 0, len(s))
-	for key := range s {
-		res = append(res, key)
-	}
-	return res
-}
-
-// PopAny returns a single element from the set.
-func (s Set[T]) PopAny() (T, bool) {
-	for key := range s {
-		s.Delete(key)
-		return key, true
-	}
-	var zeroValue T
-	return zeroValue, false
-}
-
-// Len returns the size of the set.
-func (s Set[T]) Len() int {
-	return len(s)
-}
-
-func less[T ordered](lhs, rhs T) bool {
-	return lhs < rhs
-}
diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml
index 4e387a71b8e..036531359bf 100644
--- a/skaffold-operator.yaml
+++ b/skaffold-operator.yaml
@@ -114,7 +114,6 @@ build:
         - pkg/utils/managedresources/builder
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/secrets/manager
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
@@ -214,7 +213,6 @@ build:
         - pkg/utils/kubernetes/unstructured
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
         - pkg/utils/validation/cidr
diff --git a/skaffold.yaml b/skaffold.yaml
index aa09d6ba315..784e80b953b 100644
--- a/skaffold.yaml
+++ b/skaffold.yaml
@@ -167,7 +167,6 @@ build:
         - pkg/utils/retry
         - pkg/utils/secrets
         - pkg/utils/secrets/manager
-        - pkg/utils/sets
         - pkg/utils/time
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
@@ -373,7 +372,6 @@ build:
         - pkg/utils/managedresources/builder
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/secrets/manager
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
@@ -439,7 +437,6 @@ build:
         - pkg/utils/gardener
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/timewindow
         - pkg/utils/validation/cidr
         - pkg/utils/version
@@ -514,7 +511,6 @@ build:
         - pkg/utils/kubernetes/bootstraptoken
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/timewindow
         - pkg/utils/version
         - third_party/apiserver/pkg/apis/audit/v1alpha1
@@ -721,7 +717,6 @@ build:
         - pkg/utils/retry
         - pkg/utils/secrets
         - pkg/utils/secrets/manager
-        - pkg/utils/sets
         - pkg/utils/timewindow
         - pkg/utils/version
         - vendor
@@ -1005,7 +1000,6 @@ build:
         - pkg/utils/retry
         - pkg/utils/secrets
         - pkg/utils/secrets/manager
-        - pkg/utils/sets
         - pkg/utils/time
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
@@ -1105,7 +1099,6 @@ build:
         - pkg/utils/kubernetes/unstructured
         - pkg/utils/retry
         - pkg/utils/secrets
-        - pkg/utils/sets
         - pkg/utils/timewindow
         - pkg/utils/validation/admissionplugins
         - pkg/utils/validation/cidr