From 4bdada9a37acab0f1effa53e4bbca0c4752f7ead Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Thu, 7 Oct 2021 14:43:29 +0200 Subject: [PATCH] repro upgrade issue --- bootstrap/kubeadm/config/manager/manager.yaml | 2 + config/manager/manager.yaml | 2 + .../kubeadm/config/manager/manager.yaml | 2 + go.mod | 2 + go.sum | 4 +- internal/envtest/environment.go | 22 +++--- internal/envtest/webhooks.go | 10 +-- test/e2e/config/docker.yaml | 30 +++++++- test/framework/alltypes_helpers.go | 31 +++++++- .../clusterctl/clusterctl_helpers.go | 6 ++ test/framework/deployment_helpers.go | 77 ++++++++++++++++++- .../docker/config/manager/manager.yaml | 2 + 12 files changed, 167 insertions(+), 23 deletions(-) diff --git a/bootstrap/kubeadm/config/manager/manager.yaml b/bootstrap/kubeadm/config/manager/manager.yaml index 233ba3d5fd90..a01ea2e1308f 100644 --- a/bootstrap/kubeadm/config/manager/manager.yaml +++ b/bootstrap/kubeadm/config/manager/manager.yaml @@ -33,6 +33,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6e4d23e7ad32..e3b2a9c166b0 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,6 +34,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml index b5e31734e031..2363bd55d3ba 100644 --- a/controlplane/kubeadm/config/manager/manager.yaml +++ b/controlplane/kubeadm/config/manager/manager.yaml @@ -33,6 +33,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz diff --git a/go.mod b/go.mod index 8709e0227229..3b0f14f897ff 100644 --- a/go.mod +++ b/go.mod @@ -41,3 +41,5 @@ require ( sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 sigs.k8s.io/yaml v1.3.0 ) + +replace sigs.k8s.io/controller-runtime => github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00 diff --git a/go.sum b/go.sum index f795c572bce9..596aa6c2e677 100644 --- a/go.sum +++ b/go.sum @@ -578,6 +578,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00 h1:qSL3qMYkXUwE2ddpQRyztLxRQjlLhwWLWx5vs2wuyQU= +github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -1134,8 +1136,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318 h1:MxgxYdJH4JJcEkyzFwYeH2o8WAlRlWE4cuIfsMvqK5g= -sigs.k8s.io/controller-runtime v0.10.3-0.20211011182302-43ea648ec318/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= diff --git a/internal/envtest/environment.go b/internal/envtest/environment.go index e65036d07147..fb06481be647 100644 --- a/internal/envtest/environment.go +++ b/internal/envtest/environment.go @@ -161,17 +161,17 @@ func new(uncachedObjs ...client.Object) *Environment { filepath.Join(root, "controlplane", "kubeadm", "config", "crd", "bases"), filepath.Join(root, "bootstrap", "kubeadm", "config", "crd", "bases"), }, - CRDs: []apiextensionsv1.CustomResourceDefinition{ - *builder.GenericBootstrapConfigCRD.DeepCopy(), - *builder.GenericBootstrapConfigTemplateCRD.DeepCopy(), - *builder.GenericControlPlaneCRD.DeepCopy(), - *builder.GenericControlPlaneTemplateCRD.DeepCopy(), - *builder.GenericInfrastructureMachineCRD.DeepCopy(), - *builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(), - *builder.GenericInfrastructureClusterCRD.DeepCopy(), - *builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(), - *builder.GenericRemediationCRD.DeepCopy(), - *builder.GenericRemediationTemplateCRD.DeepCopy(), + CRDs: []*apiextensionsv1.CustomResourceDefinition{ + builder.GenericBootstrapConfigCRD.DeepCopy(), + builder.GenericBootstrapConfigTemplateCRD.DeepCopy(), + builder.GenericControlPlaneCRD.DeepCopy(), + builder.GenericControlPlaneTemplateCRD.DeepCopy(), + builder.GenericInfrastructureMachineCRD.DeepCopy(), + builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(), + builder.GenericInfrastructureClusterCRD.DeepCopy(), + builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(), + builder.GenericRemediationCRD.DeepCopy(), + builder.GenericRemediationTemplateCRD.DeepCopy(), }, // initialize webhook here to be able to test the envtest install via webhookOptions // This should set LocalServingCertDir and LocalServingPort that are used below. diff --git a/internal/envtest/webhooks.go b/internal/envtest/webhooks.go index c7ab03c61d94..08c6a44cba45 100644 --- a/internal/envtest/webhooks.go +++ b/internal/envtest/webhooks.go @@ -39,8 +39,8 @@ const ( ) func initWebhookInstallOptions() envtest.WebhookInstallOptions { - validatingWebhooks := []admissionv1.ValidatingWebhookConfiguration{} - mutatingWebhooks := []admissionv1.MutatingWebhookConfiguration{} + validatingWebhooks := []*admissionv1.ValidatingWebhookConfiguration{} + mutatingWebhooks := []*admissionv1.MutatingWebhookConfiguration{} // Get the root of the current file to use in CRD paths. _, filename, _, _ := goruntime.Caller(0) //nolint @@ -86,7 +86,7 @@ func initWebhookInstallOptions() envtest.WebhookInstallOptions { // Mutate the name of each webhook, because kubebuilder generates the same name for all controllers. // In normal usage, kustomize will prefix the controller name, which we have to do manually here. -func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookConfiguration, validatingWebhooks []admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]admissionv1.MutatingWebhookConfiguration, []admissionv1.ValidatingWebhookConfiguration, error) { +func appendWebhookConfiguration(mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) { objs, err := utilyaml.ToUnstructured(configyamlFile) if err != nil { klog.Fatalf("failed to parse yaml") @@ -104,7 +104,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo klog.Fatalf("failed to convert MutatingWebhookConfiguration %s", o.GetName()) } - mutatingWebhooks = append(mutatingWebhooks, *webhook) + mutatingWebhooks = append(mutatingWebhooks, webhook) } } if o.GetKind() == validatingWebhookKind { @@ -117,7 +117,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo klog.Fatalf("failed to convert ValidatingWebhookConfiguration %s", o.GetName()) } - validatingWebhooks = append(validatingWebhooks, *webhook) + validatingWebhooks = append(validatingWebhooks, webhook) } } } diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index fcb7fc2d8e65..20e902ef8bef 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -34,6 +34,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -43,6 +45,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -50,6 +54,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -63,6 +69,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -72,6 +80,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -79,6 +89,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../bootstrap/kubeadm/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -92,6 +104,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -101,6 +115,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -108,6 +124,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../controlplane/kubeadm/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -121,6 +139,8 @@ providers: type: "url" contract: v1alpha3 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -131,6 +151,8 @@ providers: type: "url" contract: v1alpha4 replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 files: @@ -139,6 +161,8 @@ providers: - name: v1.0.99 # next; use manifest from source files value: ../../../test/infrastructure/docker/config/default replacements: + - old: "--leader-elect" + new: "--leader-elect=false" - old: --metrics-bind-addr=localhost:8080 new: --metrics-bind-addr=:8080 files: @@ -180,9 +204,9 @@ variables: CLUSTER_TOPOLOGY: "true" # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize # the management cluster to be upgraded. - INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}" - INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4" - INIT_WITH_KUBERNETES_VERSION: "v1.22.0" + INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}" + INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3" + INIT_WITH_KUBERNETES_VERSION: "v1.21.2" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index 4f38d18f6216..30b7a6ffce83 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -72,6 +71,36 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr } } + // cluster-wide + types = []metav1.TypeMeta{ + { + Kind: "Deployment", + APIVersion: "apps/v1", + }, + { + Kind: "Pod", + APIVersion: "v1", + }, + } + + for i := range types { + typeMeta := types[i] + typeList := new(unstructured.UnstructuredList) + typeList.SetAPIVersion(typeMeta.APIVersion) + typeList.SetKind(typeMeta.Kind) + + if err := input.Lister.List(ctx, typeList); err != nil { + if apierrors.IsNotFound(err) { + continue + } + Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err)) + } + for i := range typeList.Items { + obj := typeList.Items[i] + objList = append(objList, &obj) + } + } + return objList } diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index f016172ec7c0..5ed6618c6957 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -99,6 +99,9 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ Getter: client, Deployment: deployment, + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + LogPath: filepath.Join(input.LogFolder, "controllers"), }, intervals...) // Start streaming logs from all controller providers @@ -155,6 +158,9 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ Getter: client, Deployment: deployment, + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + LogPath: filepath.Join(input.LogFolder, "controllers"), }, intervals...) // Start streaming logs from all controller providers diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 2652e56d10c9..f7c5b19befdc 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -47,6 +47,9 @@ import ( type WaitForDeploymentsAvailableInput struct { Getter Getter Deployment *appsv1.Deployment + GetLister GetLister + ClientSet *kubernetes.Clientset + LogPath string } // WaitForDeploymentsAvailable waits until the Deployment has status.Available = True, that signals that @@ -69,7 +72,17 @@ func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAv } } return false - }, intervals...).Should(BeTrue(), func() string { return DescribeFailedDeployment(input, deployment) }) + }, intervals...).Should(BeTrue(), func() string { + if input.GetLister != nil && input.ClientSet != nil && input.LogPath != "" { + GetDeploymentLogs(ctx, GetDeploymentLogsInput{ + Deployment: deployment, + GetLister: input.GetLister, + ClientSet: input.ClientSet, + LogPath: input.LogPath, + }) + } + return DescribeFailedDeployment(input, deployment) + }) } // DescribeFailedDeployment returns detailed output to help debug a deployment failure in e2e. @@ -152,6 +165,68 @@ func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) { } } +type GetDeploymentLogsInput struct { + GetLister GetLister + ClientSet *kubernetes.Clientset + Deployment *appsv1.Deployment + LogPath string +} + +func GetDeploymentLogs(ctx context.Context, input GetDeploymentLogsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetDeploymentLogs") + Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for GetDeploymentLogs") + Expect(input.Deployment).NotTo(BeNil(), "input.Deployment is required for GetDeploymentLogs") + + deployment := &appsv1.Deployment{} + key := client.ObjectKeyFromObject(input.Deployment) + Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) + Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + pods := &corev1.PodList{} + Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + for _, pod := range pods.Items { + for _, container := range deployment.Spec.Template.Spec.Containers { + log.Logf("Creating log watcher for controller %s/%s, pod %s, container %s", input.Deployment.Namespace, input.Deployment.Name, pod.Name, container.Name) + + // Watch each container's logs in a goroutine so we can stream them all concurrently. + go func(pod corev1.Pod, container corev1.Container) { + defer GinkgoRecover() + + logFile := filepath.Clean(path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+".log")) + Expect(os.MkdirAll(filepath.Dir(logFile), 0750)).To(Succeed()) + + f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + Expect(err).NotTo(HaveOccurred()) + defer f.Close() + + opts := &corev1.PodLogOptions{ + Container: container.Name, + Follow: false, + } + + podLogs, err := input.ClientSet.CoreV1().Pods(input.Deployment.Namespace).GetLogs(pod.Name, opts).Stream(ctx) + if err != nil { + // Failing to stream logs should not cause the test to fail + log.Logf("Error starting logs stream for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err) + return + } + defer podLogs.Close() + + out := bufio.NewWriter(f) + defer out.Flush() + _, err = out.ReadFrom(podLogs) + if err != nil && err != io.ErrUnexpectedEOF { + // Failing to stream logs should not cause the test to fail + log.Logf("Got error while streaming logs for pod %s/%s, container %s: %v", input.Deployment.Namespace, pod.Name, container.Name, err) + } + }(pod, container) + } + } +} + type WatchPodMetricsInput struct { GetLister GetLister ClientSet *kubernetes.Clientset diff --git a/test/infrastructure/docker/config/manager/manager.yaml b/test/infrastructure/docker/config/manager/manager.yaml index 4454c900483b..b27cfa51aacc 100644 --- a/test/infrastructure/docker/config/manager/manager.yaml +++ b/test/infrastructure/docker/config/manager/manager.yaml @@ -31,6 +31,8 @@ spec: path: /readyz port: healthz livenessProbe: + failureThreshold: 12 + periodSeconds: 10 httpGet: path: /healthz port: healthz