feat: allow interpolation of cluster generator values (#9254)

* feat: allow interpolation of generator values

Allow the interpolation of `values` found in the cluster generator.
This allows interpolation of `{{name}}`, `{{server}}`,
`{{metadata.labels.*}}` and `{{metadata.annotations.*}}`. See
argoproj/applicationset#371.

This interpolation could potentially be extended to the list and
duck-type generators if desired.

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

* docs: add values interpolation usage instructions

Add a basic example of how values interpolation can be used with the
cluster generator.

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

* fix: remove billion-laughs attack vector

The previous implementation was vulnerable to a billion-laughs attack,
where someone could interpolate values based upon other values,
something like:

```yaml

values:
  lol1: lol
  lol2: '{{values.lol1}}{{values.lol1}}' #
  lol3: '{{values.lol2}}{{values.lol2}}{{values.lol2}}{{values.lol2}}'
```

To counteract that, instead of directly manipulating the `params` map,
we create a map to keep track of the interpolated values, and only
template the values which have been previously whitelisted. Once we go
through all the values, we then merge the interpolated values map back
to the `params` map.

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

applicationset/generators/cluster.go

@@ -3,6 +3,7 @@ package generators
 import (
 	"context"
 	"fmt"
+	"github.com/valyala/fasttemplate"
 	"regexp"
 	"strings"
 	"time"
@@ -37,6 +38,8 @@ type ClusterGenerator struct {
 	settingsManager *settings.SettingsManager
 }
 
+var render = &utils.Render{}
+
 func NewClusterGenerator(c client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator {
 
 	settingsManager := settings.NewSettingsManager(ctx, clientset, namespace)
@@ -106,19 +109,21 @@ func (g *ClusterGenerator) GenerateParams(
 			params["name"] = cluster.Name
 			params["server"] = cluster.Server
 
-			for key, value := range appSetGenerator.Clusters.Values {
-				params[fmt.Sprintf("values.%s", key)] = value
+			err = appendTemplatedValues(appSetGenerator.Clusters.Values, params)
+			if err != nil {
+				return nil, err
 			}
 
-			log.WithField("cluster", "local cluster").Info("matched local cluster")
-
 			res = append(res, params)
+
+			log.WithField("cluster", "local cluster").Info("matched local cluster")
 		}
 	}
 
 	// For each matching cluster secret (non-local clusters only)
 	for _, cluster := range secretsFound {
 		params := map[string]string{}
+
 		params["name"] = string(cluster.Data["name"])
 		params["nameNormalized"] = sanitizeName(string(cluster.Data["name"]))
 		params["server"] = string(cluster.Data["server"])
@@ -128,17 +133,52 @@ func (g *ClusterGenerator) GenerateParams(
 		for key, value := range cluster.ObjectMeta.Labels {
 			params[fmt.Sprintf("metadata.labels.%s", key)] = value
 		}
-		for key, value := range appSetGenerator.Clusters.Values {
-			params[fmt.Sprintf("values.%s", key)] = value
+
+		err = appendTemplatedValues(appSetGenerator.Clusters.Values, params)
+		if err != nil {
+			return nil, err
 		}
-		log.WithField("cluster", cluster.Name).Info("matched cluster secret")
 
 		res = append(res, params)
+
+		log.WithField("cluster", cluster.Name).Info("matched cluster secret")
 	}
 
 	return res, nil
 }
 
+func appendTemplatedValues(clusterValues map[string]string, params map[string]string) error {
+	// We create a local map to ensure that we do not fall victim to a billion-laughs attack. We iterate through the
+	// cluster values map and only replace values in said map if it has already been whitelisted in the params map.
+	// Once we iterate through all the cluster values we can then safely merge the `tmp` map into the main params map.
+	tmp := map[string]string{}
+
+	for key, value := range clusterValues {
+		result, err := replaceTemplatedString(value, params)
+
+		if err != nil {
+			return err
+		}
+
+		tmp[fmt.Sprintf("values.%s", key)] = result
+	}
+
+	for key, value := range tmp {
+		params[key] = value
+	}
+
+	return nil
+}
+
+func replaceTemplatedString(value string, params map[string]string) (string, error) {
+	fstTmpl := fasttemplate.New(value, "{{", "}}")
+	replacedTmplStr, err := render.Replace(fstTmpl, params, true)
+	if err != nil {
+		return "", err
+	}
+	return replacedTmplStr, nil
+}
+
 func (g *ClusterGenerator) getSecretsByClusterName(appSetGenerator *argoappsetv1alpha1.ApplicationSetGenerator) (map[string]corev1.Secret, error) {
 	// List all Clusters:
 	clusterSecretList := &corev1.SecretList{}

applicationset/generators/cluster_test.go

@@ -94,15 +94,23 @@ func TestGenerateParams(t *testing.T) {
 		{
 			name:     "no label selector",
 			selector: metav1.LabelSelector{},
-			values:   nil,
-			expected: []map[string]string{
-				{"name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar",
+			values: map[string]string{
+				"lol1":  "lol",
+				"lol2":  "{{values.lol1}}{{values.lol1}}",
+				"lol3":  "{{values.lol2}}{{values.lol2}}{{values.lol2}}",
+				"foo":   "bar",
+				"bar":   "{{ metadata.annotations.foo.argoproj.io }}",
+				"bat":   "{{ metadata.labels.environment }}",
+				"aaa":   "{{ server }}",
+				"no-op": "{{ this-does-not-exist }}",
+			}, expected: []map[string]string{
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "production", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "production", "values.aaa": "https://production-01.example.com", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar",
 					"metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"},
 
-				{"name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo",
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "staging", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "staging", "values.aaa": "https://staging-01.example.com", "name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo",
 					"metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"},
 
-				{"name": "in-cluster", "server": "https://kubernetes.default.svc"},
+				{"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "{{ metadata.annotations.foo.argoproj.io }}", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "{{ metadata.labels.environment }}", "values.aaa": "https://kubernetes.default.svc", "name": "in-cluster", "server": "https://kubernetes.default.svc"},
 			},
 			clientError:   false,
 			expectedError: nil,

applicationset/utils/utils.go

@@ -64,7 +64,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *
 }
 
 // Replace executes basic string substitution of a template with replacement values.
-// 'allowUnresolved' indicates whether or not it is acceptable to have unresolved variables
+// 'allowUnresolved' indicates whether it is acceptable to have unresolved variables
 // remaining in the substituted template.
 func (r *Render) Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) {
 	var unresolvedErr error

docs/operator-manual/applicationset/Generators-Cluster.md

@@ -159,3 +159,49 @@ In this example the `revision` value from the `generators.clusters` fields is pa
 
 !!! note
     The `values.` prefix is always prepended to values provided via `generators.clusters.values` field. Ensure you include this prefix in the parameter name within the `template` when using it.
+
+In `values` we can also interpolate the following parameter values (i.e. the same values as presented in the beginning of this page)
+
+- `name`
+- `nameNormalized` *('name' but normalized to contain only lowercase alphanumeric characters, '-' or '.')*
+- `server`
+- `metadata.labels.<key>` *(for each label in the Secret)*
+- `metadata.annotations.<key>` *(for each annotation in the Secret)*
+
+Extending the example above, we could do something like this:
+
+```yaml
+spec:
+  generators:
+  - clusters:
+      selector:
+        matchLabels:
+          type: 'staging'
+      # A key-value map for arbitrary parameters
+      values:
+        # If `my-custom-annotation` is in your cluster secret, `revision` will be substituted with it.
+        revision: '{{metadata.annotations.my-custom-annotation}}' 
+        clusterName: '{{name}}'
+  - clusters:
+      selector:
+        matchLabels:
+          type: 'production'
+      values:
+        # production uses a different revision value, for 'stable' branch
+        revision: stable
+        clusterName: '{{name}}'
+  template:
+    metadata:
+      name: '{{name}}-guestbook'
+    spec:
+      project: "my-project"
+      source:
+        repoURL: https://github.com/argoproj/argocd-example-apps/
+        # The cluster values field for each generator will be substituted here:
+        targetRevision: '{{values.revision}}'
+        path: guestbook
+      destination:
+        # In this case this is equivalent to just using {{name}}
+        server: '{{values.clusterName}}'
+        namespace: guestbook
+```