repro upgrade issue

bootstrap/kubeadm/config/manager/manager.yaml

@@ -33,6 +33,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

config/manager/manager.yaml

@@ -34,6 +34,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

controlplane/kubeadm/config/manager/manager.yaml

@@ -33,6 +33,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz

go.mod

@@ -41,3 +41,5 @@ require (
 	sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318
 	sigs.k8s.io/yaml v1.3.0
 )
+
+replace sigs.k8s.io/controller-runtime => github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3

go.sum

@@ -165,6 +165,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWc
 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 h1:MMRZoNdbvMkVb5d+opdF6nqAmQVPsOA+GWd3Q2fQRo0=
+github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -1134,8 +1136,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8=
 sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
 sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs=

test/e2e/config/docker.yaml

@@ -34,6 +34,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
     files:
@@ -43,13 +45,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -63,6 +69,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -72,13 +80,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../bootstrap/kubeadm/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -92,6 +104,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -101,13 +115,17 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
       - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../controlplane/kubeadm/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -121,6 +139,8 @@ providers:
     type: "url"
     contract: v1alpha3
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -131,6 +151,8 @@ providers:
     type: "url"
     contract: v1alpha4
     replacements:
+      - old: "--leader-elect"
+        new: "--leader-elect=false"
       - old: --metrics-addr=127.0.0.1:8080
         new: --metrics-addr=:8080
     files:
@@ -139,6 +161,8 @@ providers:
   - name: v1.0.99 # next; use manifest from source files
     value: ../../../test/infrastructure/docker/config/default
     replacements:
+    - old: "--leader-elect"
+      new: "--leader-elect=false"
     - old: --metrics-bind-addr=localhost:8080
       new: --metrics-bind-addr=:8080
     files:
@@ -180,9 +204,9 @@ variables:
   CLUSTER_TOPOLOGY: "true"
   # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
   # the management cluster to be upgraded.
-  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
-  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4"
-  INIT_WITH_KUBERNETES_VERSION: "v1.22.0"
+  INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
+  INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
+  INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

test/framework/alltypes_helpers.go

@@ -26,7 +26,6 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -72,6 +71,36 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr
 		}
 	}
 
+	// cluster-wide
+	types = []metav1.TypeMeta{
+		{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+	}
+
+	for i := range types {
+		typeMeta := types[i]
+		typeList := new(unstructured.UnstructuredList)
+		typeList.SetAPIVersion(typeMeta.APIVersion)
+		typeList.SetKind(typeMeta.Kind)
+
+		if err := input.Lister.List(ctx, typeList); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
+		}
+		for i := range typeList.Items {
+			obj := typeList.Items[i]
+			objList = append(objList, &obj)
+		}
+	}
+
 	return objList
 }
 

test/framework/clusterctl/clusterctl_helpers.go

@@ -99,6 +99,9 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
 		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
 			Getter:     client,
 			Deployment: deployment,
+			GetLister:  client,
+			ClientSet:  input.ClusterProxy.GetClientSet(),
+			LogPath:    filepath.Join(input.LogFolder, "controllers"),
 		}, intervals...)
 
 		// Start streaming logs from all controller providers
@@ -155,6 +158,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
 			Getter:     client,
 			Deployment: deployment,
+			GetLister:  client,
+			ClientSet:  input.ClusterProxy.GetClientSet(),
+			LogPath:    filepath.Join(input.LogFolder, "controllers"),
 		}, intervals...)
 
 		// Start streaming logs from all controller providers

test/framework/deployment_helpers.go

@@ -47,6 +47,9 @@ import (
 type WaitForDeploymentsAvailableInput struct {
 	Getter     Getter
 	Deployment *appsv1.Deployment
+	GetLister  GetLister
+	ClientSet  *kubernetes.Clientset
+	LogPath    string
 }
 
 // WaitForDeploymentsAvailable waits until the Deployment has status.Available = True, that signals that
@@ -69,7 +72,17 @@ func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAv
 			}
 		}
 		return false
-	}, intervals...).Should(BeTrue(), func() string { return DescribeFailedDeployment(input, deployment) })
+	}, intervals...).Should(BeTrue(), func() string {
+		if input.GetLister != nil && input.ClientSet != nil && input.LogPath != "" {
+			GetDeploymentLogs(ctx, GetDeploymentLogsInput{
+				Deployment: deployment,
+				GetLister:  input.GetLister,
+				ClientSet:  input.ClientSet,
+				LogPath:    input.LogPath,
+			})
+		}
+		return DescribeFailedDeployment(input, deployment)
+	})
 }
 
 // DescribeFailedDeployment returns detailed output to help debug a deployment failure in e2e.
@@ -152,6 +165,68 @@ func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) {
 	}
 }
 
+type GetDeploymentLogsInput struct {
+	GetLister  GetLister
+	ClientSet  *kubernetes.Clientset
+	Deployment *appsv1.Deployment
+	LogPath    string
+}
+
+func GetDeploymentLogs(ctx context.Context, input GetDeploymentLogsInput) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for GetDeploymentLogs")
+	Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for GetDeploymentLogs")
+	Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for GetDeploymentLogs")
+
+	deployment := &appsv1.Deployment{}
+	key := client.ObjectKeyFromObject(input.Deployment)
+	Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+
+	selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
+	Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+
+	pods := &corev1.PodList{}
+	Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+
+	for _, pod := range pods.Items {
+		for _, container := range deployment.Spec.Template.Spec.Containers {
+			log.Logf("Creating log watcher for controller %s/%s, pod %s, container %s", input.Deployment.Namespace, input.Deployment.Name, pod.Name, container.Name)
+
+			// Watch each container's logs in a goroutine so we can stream them all concurrently.
+			go func(pod corev1.Pod, container corev1.Container) {
+				defer GinkgoRecover()
+
+				logFile := filepath.Clean(path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+".log"))
+				Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed())
+
+				f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+				Expect(err).NotTo(HaveOccurred())
+				defer f.Close()
+
+				opts := &corev1.PodLogOptions{
+					Container: container.Name,
+					Follow:    false,
+				}
+
+				podLogs, err := input.ClientSet.CoreV1().Pods(input.Deployment.Namespace).GetLogs(pod.Name, opts).Stream(ctx)
+				if err != nil {
+					// Failing to stream logs should not cause the test to fail
+					log.Logf("Error starting logs stream for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err)
+					return
+				}
+				defer podLogs.Close()
+
+				out := bufio.NewWriter(f)
+				defer out.Flush()
+				_, err = out.ReadFrom(podLogs)
+				if err != nil && err != io.ErrUnexpectedEOF {
+					// Failing to stream logs should not cause the test to fail
+					log.Logf("Got error while streaming logs for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err)
+				}
+			}(pod, container)
+		}
+	}
+}
+
 type WatchPodMetricsInput struct {
 	GetLister   GetLister
 	ClientSet   *kubernetes.Clientset

test/go.mod

@@ -28,3 +28,5 @@ require (
 	sigs.k8s.io/kind v0.11.1
 	sigs.k8s.io/yaml v1.3.0
 )
+
+replace sigs.k8s.io/controller-runtime => github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3

test/go.sum

@@ -315,6 +315,8 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
 github.com/evanphx/json-patch/v5 v5.2.0 h1:8ozOH5xxoMYDt5/u+yMTsVXydVCbTORFnOOoq2lumco=
 github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3 h1:MMRZoNdbvMkVb5d+opdF6nqAmQVPsOA+GWd3Q2fQRo0=
+github.com/fabriziopandini/controller-runtime v0.2.0-beta.1.0.20211105090959-708d5397c4f3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -1477,8 +1479,6 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g=
-sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
 sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=
 sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA=
 sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=

test/infrastructure/docker/config/manager/manager.yaml

@@ -31,6 +31,8 @@ spec:
             path: /readyz
             port: healthz
         livenessProbe:
+          failureThreshold: 12
+          periodSeconds: 10
           httpGet:
             path: /healthz
             port: healthz