repro upgrade issue

bootstrap/kubeadm/config/manager/manager.yaml

@@ -33,6 +33,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

config/manager/manager.yaml

@@ -34,6 +34,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

controlplane/kubeadm/config/manager/manager.yaml

@@ -33,6 +33,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

go.mod

@@ -41,3 +41,5 @@ require (
 	sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318
 	sigs.k8s.io/yaml v1.3.0
 )
+
+replace sigs.k8s.io/controller-runtime => github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00

go.sum

@@ -578,6 +578,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00 h1:qSL3qMYkXUwE2ddpQRyztLxRQjlLhwWLWx5vs2wuyQU=
+github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
@@ -1134,8 +1136,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8=
 sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
 sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs=

internal/envtest/environment.go

@@ -161,17 +161,17 @@ func new(uncachedObjs ...client.Object) *Environment {
 			filepath.Join(root, "controlplane", "kubeadm", "config", "crd", "bases"),
 			filepath.Join(root, "bootstrap", "kubeadm", "config", "crd", "bases"),
 		},
-		CRDs: []apiextensionsv1.CustomResourceDefinition{
-			*builder.GenericBootstrapConfigCRD.DeepCopy(),
-			*builder.GenericBootstrapConfigTemplateCRD.DeepCopy(),
-			*builder.GenericControlPlaneCRD.DeepCopy(),
-			*builder.GenericControlPlaneTemplateCRD.DeepCopy(),
-			*builder.GenericInfrastructureMachineCRD.DeepCopy(),
-			*builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(),
-			*builder.GenericInfrastructureClusterCRD.DeepCopy(),
-			*builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(),
-			*builder.GenericRemediationCRD.DeepCopy(),
-			*builder.GenericRemediationTemplateCRD.DeepCopy(),
+		CRDs: []*apiextensionsv1.CustomResourceDefinition{
+			builder.GenericBootstrapConfigCRD.DeepCopy(),
+			builder.GenericBootstrapConfigTemplateCRD.DeepCopy(),
+			builder.GenericControlPlaneCRD.DeepCopy(),
+			builder.GenericControlPlaneTemplateCRD.DeepCopy(),
+			builder.GenericInfrastructureMachineCRD.DeepCopy(),
+			builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(),
+			builder.GenericInfrastructureClusterCRD.DeepCopy(),
+			builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(),
+			builder.GenericRemediationCRD.DeepCopy(),
+			builder.GenericRemediationTemplateCRD.DeepCopy(),
 		},
 		// initialize webhook here to be able to test the envtest install via webhookOptions
 		// This should set LocalServingCertDir and LocalServingPort that are used below.

internal/envtest/webhooks.go

@@ -39,8 +39,8 @@ const (
 )
 
 func initWebhookInstallOptions() envtest.WebhookInstallOptions {
-	validatingWebhooks := []admissionv1.ValidatingWebhookConfiguration{}
-	mutatingWebhooks := []admissionv1.MutatingWebhookConfiguration{}
+	validatingWebhooks := []*admissionv1.ValidatingWebhookConfiguration{}
+	mutatingWebhooks := []*admissionv1.MutatingWebhookConfiguration{}
 
 	// Get the root of the current file to use in CRD paths.
 	_, filename, _, _ := goruntime.Caller(0) //nolint
@@ -86,7 +86,7 @@ func initWebhookInstallOptions() envtest.WebhookInstallOptions {
 
 // Mutate the name of each webhook, because kubebuilder generates the same name for all controllers.
 // In normal usage, kustomize will prefix the controller name, which we have to do manually here.
-func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookConfiguration, validatingWebhooks []admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]admissionv1.MutatingWebhookConfiguration, []admissionv1.ValidatingWebhookConfiguration, error) {
+func appendWebhookConfiguration(mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) {
 	objs, err := utilyaml.ToUnstructured(configyamlFile)
 	if err != nil {
 		klog.Fatalf("failed to parse yaml")
@@ -104,7 +104,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo
 					klog.Fatalf("failed to convert MutatingWebhookConfiguration %s", o.GetName())
 				}
 
-				mutatingWebhooks = append(mutatingWebhooks, *webhook)
+				mutatingWebhooks = append(mutatingWebhooks, webhook)
 			}
 		}
 		if o.GetKind() == validatingWebhookKind {
@@ -117,7 +117,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo
 					klog.Fatalf("failed to convert ValidatingWebhookConfiguration %s", o.GetName())
 				}
 
-				validatingWebhooks = append(validatingWebhooks, *webhook)
+				validatingWebhooks = append(validatingWebhooks, webhook)
 			}
 		}
 	}

test/e2e/config/docker.yaml

@@ -34,6 +34,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
     files:
@@ -43,13 +45,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -63,6 +69,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -72,13 +80,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../bootstrap/kubeadm/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -92,6 +104,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -101,13 +115,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../controlplane/kubeadm/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -121,6 +139,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -131,6 +151,8 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -139,6 +161,8 @@ providers:
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../test/infrastructure/docker/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -180,9 +204,9 @@ variables:
   CLUSTER_TOPOLOGY: "true"
   # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
   # the management cluster to be upgraded.
-  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
-  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4"
-  INIT_WITH_KUBERNETES_VERSION: "v1.22.0"
+  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
+  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
+  INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

test/framework/alltypes_helpers.go

@@ -26,7 +26,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -72,6 +71,36 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr
 		}
 	}
 
+	// cluster-wide
+	types = []metav1.TypeMeta{
+		{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+	}
+
+	for i := range types {
+		typeMeta := types[i]
+		typeList := new(unstructured.UnstructuredList)
+		typeList.SetAPIVersion(typeMeta.APIVersion)
+		typeList.SetKind(typeMeta.Kind)
+
+		if err := input.Lister.List(ctx, typeList); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
+		}
+		for i := range typeList.Items {
+			obj := typeList.Items[i]
+			objList = append(objList, &obj)
+		}
+	}
+
 	return objList
 }
 

test/framework/clusterctl/clusterctl_helpers.go

@@ -99,6 +99,9 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
 		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
 			Getter:     client,
 			Deployment: deployment,
+			GetLister:  client,
+			ClientSet:  input.ClusterProxy.GetClientSet(),
+			LogPath:    filepath.Join(input.LogFolder, "controllers"),
 		}, intervals...)
 
 		// Start streaming logs from all controller providers
@@ -155,6 +158,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
 			Getter:     client,
 			Deployment: deployment,
+			GetLister:  client,
+			ClientSet:  input.ClusterProxy.GetClientSet(),
+			LogPath:    filepath.Join(input.LogFolder, "controllers"),
 		}, intervals...)
 
 		// Start streaming logs from all controller providers