From 26b9af03790443ddb294e5976611446eb0b4d0e0 Mon Sep 17 00:00:00 2001 From: koba1t Date: Thu, 27 Jan 2022 06:17:23 +0900 Subject: [PATCH 1/4] Allow setting every array element in replacements --- kyaml/yaml/fns.go | 6 ++++++ kyaml/yaml/match.go | 32 ++++++++++++++++++++++++++++++++ kyaml/yaml/match_test.go | 6 ++++++ 3 files changed, 44 insertions(+) diff --git a/kyaml/yaml/fns.go b/kyaml/yaml/fns.go index 43e54ce36c..4be2dba47e 100644 --- a/kyaml/yaml/fns.go +++ b/kyaml/yaml/fns.go @@ -796,6 +796,12 @@ func SplitIndexNameValue(p string) (string, string, error) { return parts[0], parts[1], nil } +// IsMatchEveryIndex returns true if p is matching every elements. +// e.g. "*" +func IsMatchEveryIndex(p string) bool { + return p == "*" +} + // IncrementFieldIndex increments i to point to the next field name element in // a slice of Contents. func IncrementFieldIndex(i int) int { diff --git a/kyaml/yaml/match.go b/kyaml/yaml/match.go index 149716063c..3b80f8b7be 100644 --- a/kyaml/yaml/match.go +++ b/kyaml/yaml/match.go @@ -83,10 +83,42 @@ func (p *PathMatcher) filter(rn *RNode) (*RNode, error) { // match seq elements return p.doSeq(rn) } + + if IsMatchEveryIndex(p.Path[0]) { + // match every elements (*) + return p.doMatchEvery(rn) + } // match a field return p.doField(rn) } +func (p *PathMatcher) doMatchEvery(rn *RNode) (*RNode, error) { + + if err := rn.VisitElements(p.visitEveryElem); err != nil { + return nil, err + } + + // fmt.Println(p.val.String()) + return p.val, nil +} + +func (p *PathMatcher) visitEveryElem(elem *RNode) error { + + fieldName := p.Path[0] + // recurse on the matching element + pm := &PathMatcher{Path: p.Path[1:]} + add, err := pm.filter(elem) + for k, v := range pm.Matches { + p.Matches[k] = v + } + if err != nil || add == nil { + return err + } + p.append(fieldName, add.Content()...) + + return nil +} + func (p *PathMatcher) doField(rn *RNode) (*RNode, error) { // lookup the field field, err := rn.Pipe(Get(p.Path[0])) diff --git a/kyaml/yaml/match_test.go b/kyaml/yaml/match_test.go index 70852398b3..abe51c6019 100644 --- a/kyaml/yaml/match_test.go +++ b/kyaml/yaml/match_test.go @@ -77,6 +77,12 @@ spec: {[]string{ "spec", "template", "spec", "containers", "[name=s.*]", "ports", "[containerPort=.*2]"}, ""}, + {[]string{ + "spec", "template", "spec", "containers", "*", "image"}, + "- nginx:1.7.9\n- sidecar:1.0.0\n"}, + {[]string{ + "spec", "template", "spec", "containers", "*", "ports", "*"}, + "- containerPort: 80\n- containerPort: 8081\n- containerPort: 9090\n"}, } for i, u := range updates { result, err := node.Pipe(&PathMatcher{Path: u.path}) From b79d77a8a79f796250247876a415afa059fb70b6 Mon Sep 17 00:00:00 2001 From: koba1t Date: Sun, 30 Jan 2022 18:50:20 +0900 Subject: [PATCH 2/4] add replacements every element match test --- api/filters/replacement/replacement_test.go | 183 +++++++++++++++++--- 1 file changed, 156 insertions(+), 27 deletions(-) diff --git a/api/filters/replacement/replacement_test.go b/api/filters/replacement/replacement_test.go index b06b074f4a..00efd72a9e 100644 --- a/api/filters/replacement/replacement_test.go +++ b/api/filters/replacement/replacement_test.go @@ -42,7 +42,7 @@ spec: - select: kind: Deployment name: deploy - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -95,7 +95,7 @@ spec: targets: - select: kind: Deployment - fieldPaths: + fieldPaths: - spec.template.spec.containers `, expected: `apiVersion: v1 @@ -328,7 +328,7 @@ spec: - select: kind: Deployment name: deploy1 - fieldPaths: + fieldPaths: - spec.template.spec.containers.[name=postgresdb].image `, expected: `apiVersion: v1 @@ -405,7 +405,7 @@ spec: targets: - select: version: v3 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: my-group-1/v1 @@ -492,7 +492,7 @@ spec: targets: - select: name: my-name-2 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `spec: @@ -582,7 +582,7 @@ spec: reject: - name: deploy2 - name: deploy3 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -662,7 +662,7 @@ spec: reject: - kind: Deployment name: my-name - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -731,7 +731,7 @@ spec: reject: - kind: Deployment - name: my-name - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -799,7 +799,7 @@ spec: - select: kind: Deployment name: deploy1 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image options: delimiter: ':' @@ -872,7 +872,7 @@ spec: - select: kind: Pod name: pod2 - fieldPaths: + fieldPaths: - spec.volumes.0.projected.sources.0.configMap.items.0.path options: delimiter: '/' @@ -948,7 +948,7 @@ spec: - select: kind: Pod name: pod1 - fieldPaths: + fieldPaths: - spec.volumes.0.projected.sources.0.configMap.items.0.path options: delimiter: '/' @@ -1024,7 +1024,7 @@ spec: - select: kind: Pod name: pod1 - fieldPaths: + fieldPaths: - spec.volumes.0.projected.sources.0.configMap.items.0.path options: delimiter: '/' @@ -1100,7 +1100,7 @@ spec: - select: kind: Pod name: pod1 - fieldPaths: + fieldPaths: - spec.volumes.0.projected.sources.0.configMap.items.0.path options: delimiter: '/' @@ -1176,7 +1176,7 @@ spec: - select: kind: Pod name: pod1 - fieldPaths: + fieldPaths: - spec.volumes.0.projected.sources.0.configMap.items.0.path options: delimiter: '/' @@ -1212,7 +1212,7 @@ metadata: targets: - select: name: deploy1 - fieldPaths: + fieldPaths: - spec.template.spec.containers options: create: true @@ -1223,7 +1223,7 @@ metadata: targets: - select: name: deploy2 - fieldPaths: + fieldPaths: - spec.template.spec.containers `, expected: `apiVersion: v1 @@ -1285,12 +1285,12 @@ spec: kind: Pod name: pod fieldPath: spec.containers - options: + options: delimiter: "/" targets: - select: kind: Deployment - fieldPaths: + fieldPaths: - spec.template.spec.containers `, expectedErr: "delimiter option can only be used with scalar nodes", @@ -1331,9 +1331,9 @@ spec: targets: - select: kind: Deployment - fieldPaths: + fieldPaths: - spec.template.spec.containers - options: + options: delimiter: "/" `, expectedErr: "delimiter option can only be used with scalar nodes", @@ -1354,7 +1354,7 @@ metadata: targets: - select: name: custom - fieldPaths: + fieldPaths: - metadata.annotations.[f.g.h/i-j] `, expected: `apiVersion: v1 @@ -1431,6 +1431,136 @@ spec: name: second version: latest property: second`, + }, + "index contains '*' character": { + input: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: XXXXX +`, + replacements: `replacements: +- source: + kind: Deployment + name: sample-deploy + fieldPath: metadata.name + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.*.env.[name=deployment-name].value +`, + expected: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: sample-deploy`, + }, + "list index contains '*' character": { + input: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: XXXXX + - name: foo + value: bar + - image: nginx + name: sidecar + env: + - name: deployment-name + value: YYYYY +`, + replacements: `replacements: +- source: + kind: Deployment + name: sample-deploy + fieldPath: metadata.name + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.*.env.[name=deployment-name].value +`, + expected: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: sample-deploy + - name: foo + value: bar + - image: nginx + name: sidecar + env: + - name: deployment-name + value: sample-deploy`, }, "multiple field paths in target": { input: `apiVersion: v1 @@ -1513,7 +1643,7 @@ spec: kind: Deployment metadata: name: pre-deploy - annotations: + annotations: internal.config.kubernetes.io/previousNames: deploy,deploy internal.config.kubernetes.io/previousKinds: CronJob,Deployment internal.config.kubernetes.io/previousNamespaces: default,default @@ -1535,7 +1665,7 @@ spec: - select: kind: Deployment name: deploy - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -1556,7 +1686,6 @@ spec: name: postgresdb `, }, - "replacement source.fieldPath does not exist": { input: `apiVersion: v1 kind: ConfigMap @@ -1628,7 +1757,7 @@ spec: targets: - select: annotationSelector: foo=bar-1 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -1702,7 +1831,7 @@ spec: targets: - select: labelSelector: foo=bar-1 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 @@ -1778,7 +1907,7 @@ spec: kind: Deployment reject: - labelSelector: foo=bar-2 - fieldPaths: + fieldPaths: - spec.template.spec.containers.1.image `, expected: `apiVersion: v1 From 22f9daa3ab2fa8ea47a4702d6f52757e56867fba Mon Sep 17 00:00:00 2001 From: koba1t Date: Thu, 3 Feb 2022 03:56:16 +0900 Subject: [PATCH 3/4] replacements allow to replace multi values --- api/filters/replacement/replacement.go | 28 +++++++++++++++-- kyaml/yaml/fns.go | 7 +++++ kyaml/yaml/match.go | 43 +++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/api/filters/replacement/replacement.go b/api/filters/replacement/replacement.go index bff8bba411..edc40e9123 100644 --- a/api/filters/replacement/replacement.go +++ b/api/filters/replacement/replacement.go @@ -119,13 +119,14 @@ func applyToNode(node *yaml.RNode, value *yaml.RNode, target *types.TargetSelect if target.Options != nil && target.Options.Create { t, err = node.Pipe(yaml.LookupCreate(value.YNode().Kind, fieldPath...)) } else { - t, err = node.Pipe(yaml.Lookup(fieldPath...)) + // t, err = node.Pipe(yaml.Lookup(fieldPath...)) + t, err = node.Pipe(&yaml.PathMatcher{Path: fieldPath}) } if err != nil { return err } if t != nil { - if err = setTargetValue(target.Options, t, value); err != nil { + if err = applyToOneNode(target.Options, t, value); err != nil { return err } } @@ -133,6 +134,27 @@ func applyToNode(node *yaml.RNode, value *yaml.RNode, target *types.TargetSelect return nil } +func applyToOneNode(options *types.FieldOptions, t *yaml.RNode, value *yaml.RNode) error { + if len(t.YNode().Content) == 0 { + if err := setTargetValue(options, t, value); err != nil { + return err + } + return nil + } + + for _, scalarNode := range t.YNode().Content { + if options != nil && options.Create { + return fmt.Errorf("cannot use create option in a multi-value target") + } + rn := yaml.NewRNode(scalarNode) + if err := setTargetValue(options, rn, value); err != nil { + return err + } + } + + return nil +} + func setTargetValue(options *types.FieldOptions, t *yaml.RNode, value *yaml.RNode) error { value = value.Copy() if options != nil && options.Delimiter != "" { @@ -152,7 +174,9 @@ func setTargetValue(options *types.FieldOptions, t *yaml.RNode, value *yaml.RNod } value.YNode().Value = strings.Join(tv, options.Delimiter) } + t.SetYNode(value.YNode()) + return nil } diff --git a/kyaml/yaml/fns.go b/kyaml/yaml/fns.go index 4be2dba47e..999ef53e02 100644 --- a/kyaml/yaml/fns.go +++ b/kyaml/yaml/fns.go @@ -782,6 +782,13 @@ func IsListIndex(p string) bool { return strings.HasPrefix(p, "[") && strings.HasSuffix(p, "]") } +// IsIdxNumber returns true if p is an index number. +// e.g. 1 +func IsIdxNumber(p string) bool { + idx, err := strconv.Atoi(p) + return err == nil && idx >= 0 +} + // SplitIndexNameValue splits a lookup part Val index into the field name // and field value to match. // e.g. splits [name=nginx] into (name, nginx) diff --git a/kyaml/yaml/match.go b/kyaml/yaml/match.go index 3b80f8b7be..713dd25438 100644 --- a/kyaml/yaml/match.go +++ b/kyaml/yaml/match.go @@ -5,6 +5,7 @@ package yaml import ( "regexp" + "strconv" "strings" ) @@ -42,9 +43,10 @@ type PathMatcher struct { // This is useful for if the nodes are to be printed in FlowStyle. StripComments bool - val *RNode - field string - matchRegex string + val *RNode + field string + matchRegex string + indexNumber int } func (p *PathMatcher) stripComments(n *Node) { @@ -79,6 +81,10 @@ func (p *PathMatcher) filter(rn *RNode) (*RNode, error) { return p.val, nil } + if IsIdxNumber(p.Path[0]) { + return p.doIndexSeq(rn) + } + if IsListIndex(p.Path[0]) { // match seq elements return p.doSeq(rn) @@ -98,7 +104,6 @@ func (p *PathMatcher) doMatchEvery(rn *RNode) (*RNode, error) { return nil, err } - // fmt.Println(p.val.String()) return p.val, nil } @@ -134,6 +139,36 @@ func (p *PathMatcher) doField(rn *RNode) (*RNode, error) { return p.val, err } +// doIndexSeq iterates over a sequence and appends elements matching the index p.Val +func (p *PathMatcher) doIndexSeq(rn *RNode) (*RNode, error) { + // parse to index number + idx, err := strconv.Atoi(p.Path[0]) + if err != nil { + return nil, err + } + p.indexNumber = idx + + elements, err := rn.Elements() + if err != nil { + return nil, err + } + + // get target element + element := elements[idx] + + // recurse on the matching element + pm := &PathMatcher{Path: p.Path[1:]} + add, err := pm.filter(element) + for k, v := range pm.Matches { + p.Matches[k] = v + } + if err != nil || add == nil { + return nil, err + } + p.append("", add.Content()...) + return p.val, nil +} + // doSeq iterates over a sequence and appends elements matching the path regex to p.Val func (p *PathMatcher) doSeq(rn *RNode) (*RNode, error) { // parse the field + match pair From 5ed96a34d7ae5fad7887980a3b1b86f350058cad Mon Sep 17 00:00:00 2001 From: koba1t Date: Thu, 10 Feb 2022 05:13:29 +0900 Subject: [PATCH 4/4] rename IsMatchEveryIndex to IsWildcard --- api/filters/replacement/replacement.go | 1 - api/filters/replacement/replacement_test.go | 72 +++++++++++++++++++++ kyaml/yaml/fns.go | 12 ++-- kyaml/yaml/match.go | 2 +- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/api/filters/replacement/replacement.go b/api/filters/replacement/replacement.go index edc40e9123..0cf269419f 100644 --- a/api/filters/replacement/replacement.go +++ b/api/filters/replacement/replacement.go @@ -119,7 +119,6 @@ func applyToNode(node *yaml.RNode, value *yaml.RNode, target *types.TargetSelect if target.Options != nil && target.Options.Create { t, err = node.Pipe(yaml.LookupCreate(value.YNode().Kind, fieldPath...)) } else { - // t, err = node.Pipe(yaml.Lookup(fieldPath...)) t, err = node.Pipe(&yaml.PathMatcher{Path: fieldPath}) } if err != nil { diff --git a/api/filters/replacement/replacement_test.go b/api/filters/replacement/replacement_test.go index 00efd72a9e..e5077b93f6 100644 --- a/api/filters/replacement/replacement_test.go +++ b/api/filters/replacement/replacement_test.go @@ -1431,6 +1431,78 @@ spec: name: second version: latest property: second`, + }, + "one replacements target has multiple value": { + input: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: XXXXX + - name: foo + value: bar + - image: nginx + name: sidecar + env: + - name: deployment-name + value: YYYYY +`, + replacements: `replacements: +- source: + kind: Deployment + name: sample-deploy + fieldPath: metadata.name + targets: + - select: + kind: Deployment + fieldPaths: + - spec.template.spec.containers.[image=nginx].env.[name=deployment-name].value +`, + expected: `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: sample-deploy + name: sample-deploy +spec: + replicas: 1 + selector: + matchLabels: + app: sample-deploy + template: + metadata: + labels: + app: sample-deploy + spec: + containers: + - image: nginx + name: main + env: + - name: deployment-name + value: sample-deploy + - name: foo + value: bar + - image: nginx + name: sidecar + env: + - name: deployment-name + value: sample-deploy`, }, "index contains '*' character": { input: `apiVersion: apps/v1 diff --git a/kyaml/yaml/fns.go b/kyaml/yaml/fns.go index 999ef53e02..22a9c14d9d 100644 --- a/kyaml/yaml/fns.go +++ b/kyaml/yaml/fns.go @@ -789,6 +789,12 @@ func IsIdxNumber(p string) bool { return err == nil && idx >= 0 } +// IsWildcard returns true if p is matching every elements. +// e.g. "*" +func IsWildcard(p string) bool { + return p == "*" +} + // SplitIndexNameValue splits a lookup part Val index into the field name // and field value to match. // e.g. splits [name=nginx] into (name, nginx) @@ -803,12 +809,6 @@ func SplitIndexNameValue(p string) (string, string, error) { return parts[0], parts[1], nil } -// IsMatchEveryIndex returns true if p is matching every elements. -// e.g. "*" -func IsMatchEveryIndex(p string) bool { - return p == "*" -} - // IncrementFieldIndex increments i to point to the next field name element in // a slice of Contents. func IncrementFieldIndex(i int) int { diff --git a/kyaml/yaml/match.go b/kyaml/yaml/match.go index 713dd25438..a7cdf83d82 100644 --- a/kyaml/yaml/match.go +++ b/kyaml/yaml/match.go @@ -90,7 +90,7 @@ func (p *PathMatcher) filter(rn *RNode) (*RNode, error) { return p.doSeq(rn) } - if IsMatchEveryIndex(p.Path[0]) { + if IsWildcard(p.Path[0]) { // match every elements (*) return p.doMatchEvery(rn) }