diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
index 965e4936e7..8e8d889dd7 100644
--- a/pkg/cache/cache.go
+++ b/pkg/cache/cache.go
@@ -272,7 +272,7 @@ func combineSelectors(inherited, options Options, scheme *runtime.Scheme) (Selec
 	//
 	// There is a bunch of complexity here because we need to convert to SelectorsByGVK
 	// to be able to match keys between options and inherited and then convert back to SelectorsByObject
-	optionsSelectorsByGVK, err := convertToByGVK(options.SelectorsByObject, options.DefaultSelector, options.Scheme)
+	optionsSelectorsByGVK, err := convertToByGVK(options.SelectorsByObject, options.DefaultSelector, scheme)
 	if err != nil {
 		return nil, ObjectSelector{}, err
 	}
diff --git a/pkg/cache/cache_unit_test.go b/pkg/cache/cache_unit_test.go
index a675cc341c..c5e2ab7763 100644
--- a/pkg/cache/cache_unit_test.go
+++ b/pkg/cache/cache_unit_test.go
@@ -177,6 +177,28 @@ var _ = Describe("cache.inheritFrom", func() {
 			Expect(selector.Field.Matches(fields.Set{"metadata.name": "other", "metadata.namespace": "inherited"})).To(BeFalse())
 			Expect(selector.Field.Matches(fields.Set{"metadata.name": "specified", "metadata.namespace": "inherited"})).To(BeTrue())
 		})
+		It("uses inherited scheme for inherited selectors", func() {
+			inherited.Scheme = coreScheme
+			inherited.SelectorsByObject = map[client.Object]ObjectSelector{&corev1.ConfigMap{}: {}}
+			Expect(checkError(specified.inheritFrom(inherited)).SelectorsByObject).To(HaveLen(1))
+		})
+		It("does not use specified scheme for inherited selectors", func() {
+			inherited.Scheme = runtime.NewScheme()
+			specified.Scheme = coreScheme
+			inherited.SelectorsByObject = map[client.Object]ObjectSelector{&corev1.ConfigMap{}: {}}
+			_, err := specified.inheritFrom(inherited)
+			Expect(err).To(WithTransform(runtime.IsNotRegisteredError, BeTrue()))
+		})
+		It("uses inherited scheme for specified selectors", func() {
+			inherited.Scheme = coreScheme
+			specified.SelectorsByObject = map[client.Object]ObjectSelector{&corev1.ConfigMap{}: {}}
+			Expect(checkError(specified.inheritFrom(inherited)).SelectorsByObject).To(HaveLen(1))
+		})
+		It("uses specified scheme for specified selectors", func() {
+			specified.Scheme = coreScheme
+			specified.SelectorsByObject = map[client.Object]ObjectSelector{&corev1.ConfigMap{}: {}}
+			Expect(checkError(specified.inheritFrom(inherited)).SelectorsByObject).To(HaveLen(1))
+		})
 	})
 	Context("DefaultSelector", func() {
 		It("is unchanged when specified and inherited are unset", func() {