From a041bf80fc7709b804c01925ec78a1b09e003875 Mon Sep 17 00:00:00 2001 From: Blake Pettersson Date: Mon, 27 Jun 2022 15:35:59 +0200 Subject: [PATCH] feat: allow interpolation of cluster generator values (#9254) * feat: allow interpolation of generator values Allow the interpolation of `values` found in the cluster generator. This allows interpolation of `{{name}}`, `{{server}}`, `{{metadata.labels.*}}` and `{{metadata.annotations.*}}`. See argoproj/applicationset#371. This interpolation could potentially be extended to the list and duck-type generators if desired. Signed-off-by: Blake Pettersson * docs: add values interpolation usage instructions Add a basic example of how values interpolation can be used with the cluster generator. Signed-off-by: Blake Pettersson * fix: remove billion-laughs attack vector The previous implementation was vulnerable to a billion-laughs attack, where someone could interpolate values based upon other values, something like: ```yaml values: lol1: lol lol2: '{{values.lol1}}{{values.lol1}}' # lol3: '{{values.lol2}}{{values.lol2}}{{values.lol2}}{{values.lol2}}' ``` To counteract that, instead of directly manipulating the `params` map, we create a map to keep track of the interpolated values, and only template the values which have been previously whitelisted. Once we go through all the values, we then merge the interpolated values map back to the `params` map. Signed-off-by: Blake Pettersson --- applicationset/generators/cluster.go | 54 ++++++++++++++++--- applicationset/generators/cluster_test.go | 18 +++++-- applicationset/utils/utils.go | 2 +- .../applicationset/Generators-Cluster.md | 46 ++++++++++++++++ 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/applicationset/generators/cluster.go b/applicationset/generators/cluster.go index cc291e40bed9..60109ee28ec1 100644 --- a/applicationset/generators/cluster.go +++ b/applicationset/generators/cluster.go @@ -3,6 +3,7 @@ package generators import ( "context" "fmt" + "github.com/valyala/fasttemplate" "regexp" "strings" "time" @@ -37,6 +38,8 @@ type ClusterGenerator struct { settingsManager *settings.SettingsManager } +var render = &utils.Render{} + func NewClusterGenerator(c client.Client, ctx context.Context, clientset kubernetes.Interface, namespace string) Generator { settingsManager := settings.NewSettingsManager(ctx, clientset, namespace) @@ -106,19 +109,21 @@ func (g *ClusterGenerator) GenerateParams( params["name"] = cluster.Name params["server"] = cluster.Server - for key, value := range appSetGenerator.Clusters.Values { - params[fmt.Sprintf("values.%s", key)] = value + err = appendTemplatedValues(appSetGenerator.Clusters.Values, params) + if err != nil { + return nil, err } - log.WithField("cluster", "local cluster").Info("matched local cluster") - res = append(res, params) + + log.WithField("cluster", "local cluster").Info("matched local cluster") } } // For each matching cluster secret (non-local clusters only) for _, cluster := range secretsFound { params := map[string]string{} + params["name"] = string(cluster.Data["name"]) params["nameNormalized"] = sanitizeName(string(cluster.Data["name"])) params["server"] = string(cluster.Data["server"]) @@ -128,17 +133,52 @@ func (g *ClusterGenerator) GenerateParams( for key, value := range cluster.ObjectMeta.Labels { params[fmt.Sprintf("metadata.labels.%s", key)] = value } - for key, value := range appSetGenerator.Clusters.Values { - params[fmt.Sprintf("values.%s", key)] = value + + err = appendTemplatedValues(appSetGenerator.Clusters.Values, params) + if err != nil { + return nil, err } - log.WithField("cluster", cluster.Name).Info("matched cluster secret") res = append(res, params) + + log.WithField("cluster", cluster.Name).Info("matched cluster secret") } return res, nil } +func appendTemplatedValues(clusterValues map[string]string, params map[string]string) error { + // We create a local map to ensure that we do not fall victim to a billion-laughs attack. We iterate through the + // cluster values map and only replace values in said map if it has already been whitelisted in the params map. + // Once we iterate through all the cluster values we can then safely merge the `tmp` map into the main params map. + tmp := map[string]string{} + + for key, value := range clusterValues { + result, err := replaceTemplatedString(value, params) + + if err != nil { + return err + } + + tmp[fmt.Sprintf("values.%s", key)] = result + } + + for key, value := range tmp { + params[key] = value + } + + return nil +} + +func replaceTemplatedString(value string, params map[string]string) (string, error) { + fstTmpl := fasttemplate.New(value, "{{", "}}") + replacedTmplStr, err := render.Replace(fstTmpl, params, true) + if err != nil { + return "", err + } + return replacedTmplStr, nil +} + func (g *ClusterGenerator) getSecretsByClusterName(appSetGenerator *argoappsetv1alpha1.ApplicationSetGenerator) (map[string]corev1.Secret, error) { // List all Clusters: clusterSecretList := &corev1.SecretList{} diff --git a/applicationset/generators/cluster_test.go b/applicationset/generators/cluster_test.go index a519a4073ab1..b798150348a0 100644 --- a/applicationset/generators/cluster_test.go +++ b/applicationset/generators/cluster_test.go @@ -94,15 +94,23 @@ func TestGenerateParams(t *testing.T) { { name: "no label selector", selector: metav1.LabelSelector{}, - values: nil, - expected: []map[string]string{ - {"name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", + values: map[string]string{ + "lol1": "lol", + "lol2": "{{values.lol1}}{{values.lol1}}", + "lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", + "foo": "bar", + "bar": "{{ metadata.annotations.foo.argoproj.io }}", + "bat": "{{ metadata.labels.environment }}", + "aaa": "{{ server }}", + "no-op": "{{ this-does-not-exist }}", + }, expected: []map[string]string{ + {"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "production", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "production", "values.aaa": "https://production-01.example.com", "name": "production_01/west", "nameNormalized": "production-01-west", "server": "https://production-01.example.com", "metadata.labels.environment": "production", "metadata.labels.org": "bar", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "production"}, - {"name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", + {"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "staging", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "staging", "values.aaa": "https://staging-01.example.com", "name": "staging-01", "nameNormalized": "staging-01", "server": "https://staging-01.example.com", "metadata.labels.environment": "staging", "metadata.labels.org": "foo", "metadata.labels.argocd.argoproj.io/secret-type": "cluster", "metadata.annotations.foo.argoproj.io": "staging"}, - {"name": "in-cluster", "server": "https://kubernetes.default.svc"}, + {"values.lol1": "lol", "values.lol2": "{{values.lol1}}{{values.lol1}}", "values.lol3": "{{values.lol2}}{{values.lol2}}{{values.lol2}}", "values.foo": "bar", "values.bar": "{{ metadata.annotations.foo.argoproj.io }}", "values.no-op": "{{ this-does-not-exist }}", "values.bat": "{{ metadata.labels.environment }}", "values.aaa": "https://kubernetes.default.svc", "name": "in-cluster", "server": "https://kubernetes.default.svc"}, }, clientError: false, expectedError: nil, diff --git a/applicationset/utils/utils.go b/applicationset/utils/utils.go index f3586ffdad29..b2b144502277 100644 --- a/applicationset/utils/utils.go +++ b/applicationset/utils/utils.go @@ -64,7 +64,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * } // Replace executes basic string substitution of a template with replacement values. -// 'allowUnresolved' indicates whether or not it is acceptable to have unresolved variables +// 'allowUnresolved' indicates whether it is acceptable to have unresolved variables // remaining in the substituted template. func (r *Render) Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { var unresolvedErr error diff --git a/docs/operator-manual/applicationset/Generators-Cluster.md b/docs/operator-manual/applicationset/Generators-Cluster.md index ad2316d119d6..04546278a7c3 100644 --- a/docs/operator-manual/applicationset/Generators-Cluster.md +++ b/docs/operator-manual/applicationset/Generators-Cluster.md @@ -159,3 +159,49 @@ In this example the `revision` value from the `generators.clusters` fields is pa !!! note The `values.` prefix is always prepended to values provided via `generators.clusters.values` field. Ensure you include this prefix in the parameter name within the `template` when using it. + +In `values` we can also interpolate the following parameter values (i.e. the same values as presented in the beginning of this page) + +- `name` +- `nameNormalized` *('name' but normalized to contain only lowercase alphanumeric characters, '-' or '.')* +- `server` +- `metadata.labels.` *(for each label in the Secret)* +- `metadata.annotations.` *(for each annotation in the Secret)* + +Extending the example above, we could do something like this: + +```yaml +spec: + generators: + - clusters: + selector: + matchLabels: + type: 'staging' + # A key-value map for arbitrary parameters + values: + # If `my-custom-annotation` is in your cluster secret, `revision` will be substituted with it. + revision: '{{metadata.annotations.my-custom-annotation}}' + clusterName: '{{name}}' + - clusters: + selector: + matchLabels: + type: 'production' + values: + # production uses a different revision value, for 'stable' branch + revision: stable + clusterName: '{{name}}' + template: + metadata: + name: '{{name}}-guestbook' + spec: + project: "my-project" + source: + repoURL: https://github.com/argoproj/argocd-example-apps/ + # The cluster values field for each generator will be substituted here: + targetRevision: '{{values.revision}}' + path: guestbook + destination: + # In this case this is equivalent to just using {{name}} + server: '{{values.clusterName}}' + namespace: guestbook +``` \ No newline at end of file