/
kio.go
376 lines (330 loc) · 11.8 KB
/
kio.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package kio contains low-level libraries for reading, modifying and writing
// Resource Configuration and packages.
package kio
import (
"fmt"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Reader reads ResourceNodes. Analogous to io.Reader.
type Reader interface {
Read() ([]*yaml.RNode, error)
}
// ResourceNodeSlice is a collection of ResourceNodes.
// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
// Readers, Filters or Writers may have constraints.
type ResourceNodeSlice []*yaml.RNode
var _ Reader = ResourceNodeSlice{}
func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
return o, nil
}
// Writer writes ResourceNodes. Analogous to io.Writer.
type Writer interface {
Write([]*yaml.RNode) error
}
// WriterFunc implements a Writer as a function.
type WriterFunc func([]*yaml.RNode) error
func (fn WriterFunc) Write(o []*yaml.RNode) error {
return fn(o)
}
// ReaderWriter implements both Reader and Writer interfaces
type ReaderWriter interface {
Reader
Writer
}
// Filter modifies a collection of Resource Configuration by returning the modified slice.
// When possible, Filters should be serializable to yaml so that they can be described
// as either data or code.
//
// Analogous to http://www.linfo.org/filters.html
type Filter interface {
Filter([]*yaml.RNode) ([]*yaml.RNode, error)
}
// FilterFunc implements a Filter as a function.
type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
return fn(o)
}
// Pipeline reads Resource Configuration from a set of Inputs, applies some
// transformation filters, and writes the results to a set of Outputs.
//
// Analogous to http://www.linfo.org/pipes.html
type Pipeline struct {
// Inputs provide sources for Resource Configuration to be read.
Inputs []Reader `yaml:"inputs,omitempty"`
// Filters are transformations applied to the Resource Configuration.
// They are applied in the order they are specified.
// Analogous to http://www.linfo.org/filters.html
Filters []Filter `yaml:"filters,omitempty"`
// Outputs are where the transformed Resource Configuration is written.
Outputs []Writer `yaml:"outputs,omitempty"`
// ContinueOnEmptyResult configures what happens when a filter in the pipeline
// returns an empty result.
// If it is false (default), subsequent filters will be skipped and the result
// will be returned immediately. This is useful as an optimization when you
// know that subsequent filters will not alter the empty result.
// If it is true, the empty result will be provided as input to the next
// filter in the list. This is useful when subsequent functions in the
// pipeline may generate new resources.
ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
}
// Execute executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline.
func (p Pipeline) Execute() error {
return p.ExecuteWithCallback(nil)
}
// PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
type PipelineExecuteCallbackFunc = func(op Filter)
// ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline. The callback will be called each time a step succeeds.
func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
var result []*yaml.RNode
// read from the inputs
for _, i := range p.Inputs {
nodes, err := i.Read()
if err != nil {
return errors.Wrap(err)
}
result = append(result, nodes...)
}
// apply operations
var err error
for i := range p.Filters {
// Not all RNodes passed through kio.Pipeline have metadata nor should
// they all be required to.
var nodeAnnos map[string]map[string]string
nodeAnnos, err = storeInternalAnnotations(result)
if err != nil && err != yaml.ErrMissingMetadata {
return err
}
op := p.Filters[i]
if callback != nil {
callback(op)
}
result, err = op.Filter(result)
// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
// handled by outputs. However currently some writer like LocalPackageReadWriter
// will clear the output directory and which will cause unpredictable results
if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
return errors.Wrap(err)
}
// If either the internal annotations for path, index, and id OR the legacy
// annotations for path, index, and id are changed, we have to update the other.
err = reconcileInternalAnnotations(result, nodeAnnos)
if err != nil && err != yaml.ErrMissingMetadata {
return err
}
}
// write to the outputs
for _, o := range p.Outputs {
if err := o.Write(result); err != nil {
return errors.Wrap(err)
}
}
return nil
}
// FilterAll runs the yaml.Filter against all inputs
func FilterAll(filter yaml.Filter) Filter {
return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
for i := range nodes {
_, err := filter.Filter(nodes[i])
if err != nil {
return nil, errors.Wrap(err)
}
}
return nodes, nil
})
}
// Store the original path, index, and id annotations so that we can reconcile
// it later. This is necessary because currently both internal-prefixed annotations
// and legacy annotations are currently supported, and a change to one must be
// reflected in the other.
func storeInternalAnnotations(result []*yaml.RNode) (map[string]map[string]string, error) {
nodeAnnosMap := make(map[string]map[string]string)
for i := range result {
if err := kioutil.CopyLegacyAnnotations(result[i]); err != nil {
return nil, err
}
meta, err := result[i].GetMeta()
if err != nil {
return nil, err
}
path := meta.Annotations[kioutil.PathAnnotation]
index := meta.Annotations[kioutil.IndexAnnotation]
id := meta.Annotations[kioutil.IdAnnotation]
legacyPath := meta.Annotations[kioutil.LegacyPathAnnotation]
legacyIndex := meta.Annotations[kioutil.LegacyIndexAnnotation]
legacyId := meta.Annotations[kioutil.LegacyIdAnnotation]
// if prior to running the functions, the legacy and internal annotations differ,
// throw an error as we cannot infer the user's intent.
if path != legacyPath {
return nil, fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
}
if index!= legacyIndex {
return nil, fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
}
if id != legacyId {
return nil, fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
}
if _, ok := nodeAnnosMap[path]; !ok {
nodeAnnosMap[path] = make(map[string]string)
}
nodeAnnosMap[path][index] = id
}
return nodeAnnosMap, nil
}
type nodeAnnotations struct {
path string
index string
id string
}
func reconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
for _, node := range result {
meta, err := node.GetMeta()
if err != nil {
return err
}
// if only one annotation is set, set the other.
err = missingInternalOrLegacyAnnotations(node, meta)
if err != nil {
return err
}
// we must check to see if the function changed either the new internal annotations
// or the old legacy annotations. If one is changed, the change must be reflected
// in the other.
err = checkAnnotationsAltered(node, meta, nodeAnnosMap)
if err != nil {
return err
}
// if the annotations are still somehow out of sync, prefer the internal annotations
// and copy them to the legacy ones
err = kioutil.CopyLegacyAnnotations(node)
if err != nil {
return err
}
}
return nil
}
func missingInternalOrLegacyAnnotations(rn *yaml.RNode, meta yaml.ResourceMeta) error {
if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
return err
}
if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
return err
}
if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
return err
}
return nil
}
func missingInternalOrLegacyAnnotation(rn *yaml.RNode, meta yaml.ResourceMeta, newKey string, legacyKey string) error {
value := meta.Annotations[newKey]
legacyValue := meta.Annotations[legacyKey]
if value == "" && legacyValue == "" {
// do nothing
return nil
}
if value == "" {
// new key is not set, copy from legacy key
if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
return err
}
} else if legacyValue == "" {
// legacy key is not set, copy from new key
if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
return err
}
}
return nil
}
func checkAnnotationsAltered(rn *yaml.RNode, meta yaml.ResourceMeta, nodeAnnosMap map[string]map[string]string) error {
// get the resource's current path, index, and ids from the new annotations
internal := nodeAnnotations{
path: meta.Annotations[kioutil.PathAnnotation],
index: meta.Annotations[kioutil.IndexAnnotation],
id: meta.Annotations[kioutil.IdAnnotation],
}
// get the resource's current path, index, and ids from the legacy annotations
legacy := nodeAnnotations{
path: meta.Annotations[kioutil.LegacyPathAnnotation],
index: meta.Annotations[kioutil.LegacyIndexAnnotation],
id: meta.Annotations[kioutil.LegacyIdAnnotation],
}
if internal.path == legacy.path &&
internal.index == legacy.index &&
internal.id == legacy.id {
// none of the annotations differ, so no reconciliation is needed
return nil
}
// nodeAnnosMap is a map of structure path -> index -> id that stores
// all of the resources' path/index/id annotations prior to the functions
// being run. We use that to check whether the legacy or new internal
// annotations have been changed, and make sure the change is reflected
// in the other.
// first, check if the internal annotations are found in nodeAnnosMap
if indexIdMap, ok := nodeAnnosMap[internal.path]; ok {
if id, ok := indexIdMap[internal.index]; ok {
if id == internal.id {
// the internal annotations of the resource match the ones stored in
// nodeAnnosMap, so we should copy the legacy annotations to the
// internal ones
if err := updateAnnotations(rn, meta,
[]string{
kioutil.PathAnnotation,
kioutil.IndexAnnotation,
kioutil.IdAnnotation,
},
[]string{
legacy.path,
legacy.index,
legacy.id,
}); err != nil {
return err
}
}
}
}
// check the opposite, to see if the legacy annotations are in nodeAnnosMap
if indexIdMap, ok := nodeAnnosMap[legacy.path]; ok {
if id, ok := indexIdMap[legacy.index]; ok {
if id == legacy.id {
// the legacy annotations of the resource match the ones stored in
// nodeAnnosMap, so we should copy the internal annotations to the
// legacy ones
if err := updateAnnotations(rn, meta,
[]string{
kioutil.LegacyPathAnnotation,
kioutil.LegacyIndexAnnotation,
kioutil.LegacyIdAnnotation,
},
[]string{
internal.path,
internal.index,
internal.id,
}); err != nil {
return err
}
}
}
}
return nil
}
func updateAnnotations(rn *yaml.RNode, meta yaml.ResourceMeta, keys []string, values []string) error {
if len(keys) != len(values) {
return fmt.Errorf("keys is not same length as values")
}
for i := range keys {
_, ok := meta.Annotations[keys[i]]
if values[i] == "" && !ok {
// don't set "" if annotation is not already there
continue
}
if err := rn.PipeE(yaml.SetAnnotation(keys[i], values[i])); err != nil {
return err
}
}
return nil
}