Reference Secrets for per stack versioning (#565)

Following the [Reference ConfigMap for per stack versioning](#520) and [Revised adaptation of ConfigMap per Stack](#550), this Pull Request implements **Secrets versioned by reference**. The expected behaviour is that given a **Secret** named in the `<stack-name>-<secret-name>` pattern is defined on `stackTemplate.configurationResources.secretRef`, Stackset Controller identifies the deployed **Secret** and updates it with the **Stack** info on `ownerReferences`.
zalando-incubator · Feb 5, 2024 · ed1440e · ed1440e
1 parent 1512402
commit ed1440e
Show file tree

Hide file tree

Showing 19 changed files with 707 additions and 88 deletions.
diff --git a/cmd/e2e/README.md b/cmd/e2e/README.md
@@ -32,7 +32,7 @@ watch -n 10 "kubectl get -n foo stackset,stack,ing,ep,deployment"
 kubectl delete namespace foo; kubectl create namespace foo
 make
 ./build/stackset-controller --apiserver=http://127.0.0.1:8001 \
---enable-configmap-support --enable-routegroup-support \
+--enable-configmap-support --enable-secret-support --enable-routegroup-support \
 --enable-traffic-segments --annotated-traffic-segments \
 --sync-ingress-annotation=example.org/i-haz-synchronize \
 --sync-ingress-annotation=teapot.org/the-best --controller-id=foo \

diff --git a/cmd/e2e/basic_test.go b/cmd/e2e/basic_test.go
@@ -39,6 +39,7 @@ type TestStacksetSpecFactory struct {
 	routegroup                    bool
 	externalIngress               bool
 	configMap                     bool
+	secret                        bool
 	limit                         int32
 	scaleDownTTL                  int64
 	replicas                      int32
@@ -56,6 +57,7 @@ func NewTestStacksetSpecFactory(stacksetName string) *TestStacksetSpecFactory {
 	return &TestStacksetSpecFactory{
 		stacksetName:           stacksetName,
 		configMap:              false,
+		secret:                 false,
 		ingress:                false,
 		externalIngress:        false,
 		limit:                  4,
@@ -72,6 +74,11 @@ func (f *TestStacksetSpecFactory) ConfigMap() *TestStacksetSpecFactory {
 	return f
 }
 
+func (f *TestStacksetSpecFactory) Secret() *TestStacksetSpecFactory {
+	f.secret = true
+	return f
+}
+
 func (f *TestStacksetSpecFactory) Behavior(stabilizationWindowSeconds int32) *TestStacksetSpecFactory {
 	f.hpaBehavior = true
 	f.hpaStabilizationWindowSeconds = stabilizationWindowSeconds
@@ -166,7 +173,7 @@ func (f *TestStacksetSpecFactory) Create(t *testing.T, stackVersion string) zv1.
 			},
 		}
 
-		result.StackTemplate.Spec.StackSpec.PodTemplate.Spec.Volumes = []corev1.Volume{
+		result.StackTemplate.Spec.PodTemplate.Spec.Volumes = []corev1.Volume{
 			{
 				Name: "config-volume",
 				VolumeSource: corev1.VolumeSource{
@@ -180,6 +187,40 @@ func (f *TestStacksetSpecFactory) Create(t *testing.T, stackVersion string) zv1.
 		}
 	}
 
+	if f.secret {
+		secretName := fmt.Sprintf("%s-%s-secret", f.stacksetName, stackVersion)
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: secretName,
+			},
+			Data: map[string][]byte{
+				"key": []byte("value"),
+			},
+		}
+
+		_, err := secretInterface().Create(context.Background(), secret, metav1.CreateOptions{})
+		require.NoError(t, err)
+
+		result.StackTemplate.Spec.ConfigurationResources = []zv1.ConfigurationResourcesSpec{
+			{
+				SecretRef: corev1.LocalObjectReference{
+					Name: secretName,
+				},
+			},
+		}
+
+		result.StackTemplate.Spec.PodTemplate.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "secret-volume",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: secretName,
+					},
+				},
+			},
+		}
+	}
+
 	if f.autoscaler {
 		result.StackTemplate.Spec.Autoscaler = &zv1.Autoscaler{
 			MaxReplicas: f.hpaMaxReplicas,
@@ -333,21 +374,40 @@ func verifyStack(t *testing.T, stacksetName, currentVersion string, stacksetSpec
 		require.EqualValues(t, expectedRef, hpa.Spec.ScaleTargetRef)
 	}
 
-	// Verify ConfigMaps
 	for _, rsc := range stacksetSpec.StackTemplate.Spec.ConfigurationResources {
-		configMap, err := waitForConfigMap(t, rsc.ConfigMapRef.Name)
-		require.NoError(t, err)
-		assert.EqualValues(t, stackResourceLabels, configMap.Labels)
-		assert.Contains(t, configMap.Name, stack.Name)
-		assert.Equal(t, map[string]string{"key": "value"}, configMap.Data)
-		assert.Equal(t, []metav1.OwnerReference{
-			{
-				APIVersion: "zalando.org/v1",
-				Kind:       "Stack",
-				Name:       stack.Name,
-				UID:        stack.UID,
-			},
-		}, configMap.OwnerReferences)
+		// Verify ConfigMaps
+		if rsc.ConfigMapRef.Name != "" {
+			configMap, err := waitForConfigMap(t, rsc.ConfigMapRef.Name)
+			require.NoError(t, err)
+			assert.EqualValues(t, stackResourceLabels, configMap.Labels)
+			assert.Contains(t, configMap.Name, stack.Name)
+			assert.Equal(t, map[string]string{"key": "value"}, configMap.Data)
+			assert.Equal(t, []metav1.OwnerReference{
+				{
+					APIVersion: "zalando.org/v1",
+					Kind:       "Stack",
+					Name:       stack.Name,
+					UID:        stack.UID,
+				},
+			}, configMap.OwnerReferences)
+		}
+
+		// Verify Secrets
+		if rsc.SecretRef.Name != "" {
+			secret, err := waitForSecret(t, rsc.SecretRef.Name)
+			require.NoError(t, err)
+			assert.EqualValues(t, stackResourceLabels, secret.Labels)
+			assert.Contains(t, secret.Name, stack.Name)
+			assert.Equal(t, map[string][]byte{"key": []byte("value")}, secret.Data)
+			assert.Equal(t, []metav1.OwnerReference{
+				{
+					APIVersion: "zalando.org/v1",
+					Kind:       "Stack",
+					Name:       stack.Name,
+					UID:        stack.UID,
+				},
+			}, secret.OwnerReferences)
+		}
 	}
 
 	verifyStackIngressSources(
@@ -640,6 +700,7 @@ func testStacksetCreate(
 	t *testing.T,
 	testName string,
 	configmap bool,
+	secret bool,
 	hpa,
 	ingress,
 	routegroup,
@@ -656,6 +717,9 @@ func testStacksetCreate(
 		if configmap {
 			stacksetSpecFactory.ConfigMap()
 		}
+		if secret {
+			stacksetSpecFactory.Secret()
+		}
 		if hpa {
 			stacksetSpecFactory.Autoscaler(
 				1,
@@ -1008,31 +1072,35 @@ func testStacksetUpdate(
 }
 
 func TestStacksetCreateBasic(t *testing.T) {
-	testStacksetCreate(t, "basic", false, false, false, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "basic", false, false, false, false, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateConfigMap(t *testing.T) {
-	testStacksetCreate(t, "configmap", true, false, false, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "configmap", true, false, false, false, false, false, false, testAnnotationsCreate)
+}
+
+func TestStacksetCreateSecret(t *testing.T) {
+	testStacksetCreate(t, "secret", false, true, false, false, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateHPA(t *testing.T) {
-	testStacksetCreate(t, "hpa", false, true, false, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "hpa", false, false, true, false, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateIngress(t *testing.T) {
-	testStacksetCreate(t, "ingress", false, false, true, false, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "ingress", false, false, false, true, false, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateRouteGroup(t *testing.T) {
-	testStacksetCreate(t, "routegroup", false, false, false, true, false, false, testAnnotationsCreate)
+	testStacksetCreate(t, "routegroup", false, false, false, false, true, false, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateExternalIngress(t *testing.T) {
-	testStacksetCreate(t, "externalingress", false, false, false, false, true, false, testAnnotationsCreate)
+	testStacksetCreate(t, "externalingress", false, false, false, false, false, true, false, testAnnotationsCreate)
 }
 
 func TestStacksetCreateUpdateStrategy(t *testing.T) {
-	testStacksetCreate(t, "updatestrategy", false, false, false, false, false, true, testAnnotationsCreate)
+	testStacksetCreate(t, "updatestrategy", false, false, false, false, false, false, true, testAnnotationsCreate)
 }
 
 func TestStacksetUpdateBasic(t *testing.T) {

diff --git a/cmd/e2e/broken_stack_test.go b/cmd/e2e/broken_stack_test.go
@@ -171,3 +171,83 @@ func TestBrokenStackWithConfigMaps(t *testing.T) {
 	}
 
 }
+
+func TestBrokenStackWithSecrets(t *testing.T) {
+	t.Parallel()
+
+	stacksetName := "stackset-broken-stacks-with-secret"
+	factory := NewTestStacksetSpecFactory(stacksetName).Ingress().Secret().StackGC(1, 30)
+
+	firstVersion := "v1"
+	firstStack := fmt.Sprintf("%s-%s", stacksetName, firstVersion)
+	spec := factory.Create(t, firstVersion)
+	err := createStackSet(stacksetName, 0, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, firstVersion)
+	require.NoError(t, err)
+
+	unhealthyVersion := "v2"
+	unhealthyStack := fmt.Sprintf("%s-%s", stacksetName, unhealthyVersion)
+	spec = factory.Create(t, unhealthyVersion)
+	for _, cr := range spec.StackTemplate.Spec.ConfigurationResources {
+		if cr.SecretRef.Name != "" {
+			err := secretInterface().Delete(context.Background(), cr.SecretRef.Name, metav1.DeleteOptions{})
+			require.NoError(t, err)
+		}
+	}
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, unhealthyVersion)
+	require.NoError(t, err)
+
+	_, err = waitForIngress(t, stacksetName)
+	require.NoError(t, err)
+
+	initialWeights := map[string]float64{firstStack: 100}
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, initialWeights, nil).await()
+	require.NoError(t, err)
+
+	// Switch traffic to the second stack, this should fail
+	desiredWeights := map[string]float64{unhealthyStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, desiredWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, desiredWeights, nil).await()
+	require.Error(t, err)
+
+	// Create a healthy stack
+	healthyVersion := "v3"
+	healthyStack := fmt.Sprintf("%s-%s", stacksetName, healthyVersion)
+	spec = factory.Create(t, healthyVersion)
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, healthyVersion)
+	require.NoError(t, err)
+
+	healthyWeights := map[string]float64{healthyStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, healthyWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, healthyWeights, nil).await()
+	require.NoError(t, err)
+
+	// Create another healthy stack so we can test GC
+	finalVersion := "v4"
+	finalStack := fmt.Sprintf("%s-%s", stacksetName, finalVersion)
+	spec = factory.Create(t, finalVersion)
+	err = updateStackset(stacksetName, spec)
+	require.NoError(t, err)
+	_, err = waitForStack(t, stacksetName, finalVersion)
+	require.NoError(t, err)
+
+	finalWeights := map[string]float64{finalStack: 100}
+	err = setDesiredTrafficWeightsStackset(stacksetName, finalWeights)
+	require.NoError(t, err)
+	err = trafficWeightsUpdatedStackset(t, stacksetName, weightKindActual, finalWeights, nil).await()
+	require.NoError(t, err)
+
+	// Check that the unhealthy stack was deleted
+	for _, stack := range []string{unhealthyStack, firstStack} {
+		err := resourceDeleted(t, "stack", stack, stackInterface()).withTimeout(time.Second * 60).await()
+		require.NoError(t, err)
+	}
+
+}
diff --git a/cmd/e2e/test_environment.go b/cmd/e2e/test_environment.go
@@ -101,6 +101,10 @@ func configMapInterface() corev1typed.ConfigMapInterface {
 	return kubernetesClient.CoreV1().ConfigMaps(namespace)
 }
 
+func secretInterface() corev1typed.SecretInterface {
+	return kubernetesClient.CoreV1().Secrets(namespace)
+}
+
 func requiredEnvar(envar string) string {
 	namespace := os.Getenv(envar)
 	if namespace == "" {

diff --git a/cmd/e2e/test_utils.go b/cmd/e2e/test_utils.go
@@ -488,6 +488,14 @@ func waitForConfigMap(t *testing.T, configMapName string) (*corev1.ConfigMap, er
 	return configMapInterface().Get(context.Background(), configMapName, metav1.GetOptions{})
 }
 
+func waitForSecret(t *testing.T, secretName string) (*corev1.Secret, error) {
+	err := resourceCreated(t, "secret", secretName, secretInterface()).await()
+	if err != nil {
+		return nil, err
+	}
+	return secretInterface().Get(context.Background(), secretName, metav1.GetOptions{})
+}
+
 func waitForUpdatedRouteGroup(t *testing.T, name string, oldTimestamp string) (*rgv1.RouteGroup, error) {
 	err := newAwaiter(t, fmt.Sprintf("updated RouteGroup %s", name)).withPoll(func() (bool, error) {
 		rg, err := routegroupInterface().Get(context.Background(), name, metav1.GetOptions{})

diff --git a/cmd/stackset-controller/main.go b/cmd/stackset-controller/main.go
@@ -49,6 +49,7 @@ var (
 		IngressSourceSwitchTTL      time.Duration
 		ReconcileWorkers            int
 		ConfigMapSupportEnabled     bool
+		SecretSupportEnabled        bool
 	}
 )
 
@@ -84,6 +85,7 @@ func main() {
 	kingpin.Flag("ingress-source-switch-ttl", "The ttl before an ingress source is deleted when replaced with another one e.g. switching from RouteGroup to Ingress or vice versa.").
 		Default(defaultIngressSourceSwitchTTL).DurationVar(&config.IngressSourceSwitchTTL)
 	kingpin.Flag("enable-configmap-support", "Enable support for ConfigMaps on StackSets.").Default("false").BoolVar(&config.ConfigMapSupportEnabled)
+	kingpin.Flag("enable-secret-support", "Enable support for Secrets on StackSets.").Default("false").BoolVar(&config.SecretSupportEnabled)
 	kingpin.Parse()
 
 	if config.Debug {
@@ -115,6 +117,7 @@ func main() {
 		config.AnnotatedTrafficSegments,
 		config.SyncIngressAnnotations,
 		config.ConfigMapSupportEnabled,
+		config.SecretSupportEnabled,
 		config.IngressSourceSwitchTTL,
 	)
 	if err != nil {