From ff690b9803a30863d0c96031567c9941ec67f178 Mon Sep 17 00:00:00 2001 From: incubator4 Date: Wed, 22 Dec 2021 18:13:13 +0800 Subject: [PATCH] hclwrite: Various new "TokensFor..." functions The "TokensFor..." family of functions all aim to construct valid raw token sequences representing particular syntax constructs. Previously we had only "leaf" functions TokensForValue and TokensForTraversal, but nothing to help with constructing compound structures. Here we add TokensForTuple, TokensForObject, and TokensForFunctionCall which together cover all of the constructs that HCL allows static analysis of, and thus constructs where it's likely that someone would want to generate an expression that is interpreted purely by its syntax and not resolved into a value. What all of these have in common is that they take other Tokens values as arguments and include them verbatim as part of their result, with the caller being responsible for making sure these smaller units are themselves valid expression tokens. This also adds TokensForIdentifier as a convenient shorthand for generating single-identifier tokens, which is particularly useful for populating the attribute names in TokensForObject. --- hclwrite/generate.go | 140 ++++++++++++++++ hclwrite/generate_test.go | 330 ++++++++++++++++++++++++++++++++++++++ hclwrite/tokens.go | 10 ++ 3 files changed, 480 insertions(+) diff --git a/hclwrite/generate.go b/hclwrite/generate.go index 48e13121..6f6a2e63 100644 --- a/hclwrite/generate.go +++ b/hclwrite/generate.go @@ -39,6 +39,146 @@ func TokensForTraversal(traversal hcl.Traversal) Tokens { return toks } +// TokensForIdentifier returns a sequence of tokens representing just the +// given identifier. +// +// In practice this function can only ever generate exactly one token, because +// an identifier is always a leaf token in the syntax tree. +// +// This is similar to calling TokensForTraversal with a single-step absolute +// traversal, but avoids the need to construct a separate traversal object +// for this simple common case. If you need to generate a multi-step traversal, +// use TokensForTraversal instead. +func TokensForIdentifier(name string) Tokens { + return Tokens{ + newIdentToken(name), + } +} + +// TokensForTuple returns a sequence of tokens that represents a tuple +// constructor, with element expressions populated from the given list +// of tokens. +// +// TokensForTuple includes the given elements verbatim into the element +// positions in the resulting tuple expression, without any validation to +// ensure that they represent valid expressions. Use TokensForValue or +// TokensForTraversal to generate valid leaf expression values, or use +// TokensForTuple, TokensForObject, and TokensForFunctionCall to +// generate other nested compound expressions. +func TokensForTuple(elems []Tokens) Tokens { + var toks Tokens + toks = append(toks, &Token{ + Type: hclsyntax.TokenOBrack, + Bytes: []byte{'['}, + }) + for index, elem := range elems { + if index > 0 { + toks = append(toks, &Token{ + Type: hclsyntax.TokenComma, + Bytes: []byte{','}, + }) + } + toks = append(toks, elem...) + } + + toks = append(toks, &Token{ + Type: hclsyntax.TokenCBrack, + Bytes: []byte{']'}, + }) + + format(toks) // fiddle with the SpacesBefore field to get canonical spacing + return toks +} + +// TokensForObject returns a sequence of tokens that represents an object +// constructor, with attribute name/value pairs populated from the given +// list of attribute token objects. +// +// TokensForObject includes the given tokens verbatim into the name and +// value positions in the resulting object expression, without any validation +// to ensure that they represent valid expressions. Use TokensForValue or +// TokensForTraversal to generate valid leaf expression values, or use +// TokensForTuple, TokensForObject, and TokensForFunctionCall to +// generate other nested compound expressions. +// +// Note that HCL requires placing a traversal expression in parentheses if +// you intend to use it as an attribute name expression, because otherwise +// the parser will interpret it as a literal attribute name. TokensForObject +// does not handle that situation automatically, so a caller must add the +// necessary `TokenOParen` and TokenCParen` manually if needed. +func TokensForObject(attrs []ObjectAttrTokens) Tokens { + var toks Tokens + toks = append(toks, &Token{ + Type: hclsyntax.TokenOBrace, + Bytes: []byte{'{'}, + }) + if len(attrs) > 0 { + toks = append(toks, &Token{ + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + }) + } + for _, attr := range attrs { + toks = append(toks, attr.Name...) + toks = append(toks, &Token{ + Type: hclsyntax.TokenEqual, + Bytes: []byte{'='}, + }) + toks = append(toks, attr.Value...) + toks = append(toks, &Token{ + Type: hclsyntax.TokenNewline, + Bytes: []byte{'\n'}, + }) + } + toks = append(toks, &Token{ + Type: hclsyntax.TokenCBrace, + Bytes: []byte{'}'}, + }) + + format(toks) // fiddle with the SpacesBefore field to get canonical spacing + return toks +} + +// TokensForFunctionCall returns a sequence of tokens that represents call +// to the function with the given name, using the argument tokens to +// populate the argument expressions. +// +// TokensForFunctionCall includes the given argument tokens verbatim into the +// positions in the resulting call expression, without any validation +// to ensure that they represent valid expressions. Use TokensForValue or +// TokensForTraversal to generate valid leaf expression values, or use +// TokensForTuple, TokensForObject, and TokensForFunctionCall to +// generate other nested compound expressions. +// +// This function doesn't include an explicit way to generate the expansion +// symbol "..." on the final argument. Currently, generating that requires +// manually appending a TokenEllipsis with the bytes "..." to the tokens for +// the final argument. +func TokensForFunctionCall(funcName string, args ...Tokens) Tokens { + var toks Tokens + toks = append(toks, TokensForIdentifier(funcName)...) + toks = append(toks, &Token{ + Type: hclsyntax.TokenOParen, + Bytes: []byte{'('}, + }) + for index, arg := range args { + if index > 0 { + toks = append(toks, &Token{ + Type: hclsyntax.TokenComma, + Bytes: []byte{','}, + }) + } + toks = append(toks, arg...) + } + toks = append(toks, &Token{ + Type: hclsyntax.TokenCParen, + Bytes: []byte{')'}, + }) + + format(toks) // fiddle with the SpacesBefore field to get canonical spacing + return toks +} + func appendTokensForValue(val cty.Value, toks Tokens) Tokens { switch { diff --git a/hclwrite/generate_test.go b/hclwrite/generate_test.go index d74a086f..a3df8442 100644 --- a/hclwrite/generate_test.go +++ b/hclwrite/generate_test.go @@ -536,3 +536,333 @@ func TestTokensForTraversal(t *testing.T) { } } } + +func TestTokensForTuple(t *testing.T) { + tests := map[string]struct { + Val []Tokens + Want Tokens + }{ + "no elements": { + nil, + Tokens{ + {Type: hclsyntax.TokenOBrack, Bytes: []byte{'['}}, + {Type: hclsyntax.TokenCBrack, Bytes: []byte{']'}}, + }, + }, + "one element": { + []Tokens{ + TokensForValue(cty.StringVal("foo")), + }, + Tokens{ + {Type: hclsyntax.TokenOBrack, Bytes: []byte{'['}}, + {Type: hclsyntax.TokenOQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenQuotedLit, Bytes: []byte("foo")}, + {Type: hclsyntax.TokenCQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenCBrack, Bytes: []byte{']'}}, + }, + }, + "two elements": { + []Tokens{ + TokensForTraversal(hcl.Traversal{ + hcl.TraverseRoot{Name: "root"}, + hcl.TraverseAttr{Name: "attr"}, + }), + TokensForValue(cty.StringVal("foo")), + }, + Tokens{ + {Type: hclsyntax.TokenOBrack, Bytes: []byte{'['}}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("root")}, + {Type: hclsyntax.TokenDot, Bytes: []byte(".")}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("attr")}, + {Type: hclsyntax.TokenComma, Bytes: []byte{','}}, + {Type: hclsyntax.TokenOQuote, Bytes: []byte(`"`), SpacesBefore: 1}, + {Type: hclsyntax.TokenQuotedLit, Bytes: []byte("foo")}, + {Type: hclsyntax.TokenCQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenCBrack, Bytes: []byte{']'}}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + got := TokensForTuple(test.Val) + + if !cmp.Equal(got, test.Want) { + diff := cmp.Diff(got, test.Want, cmp.Comparer(func(a, b []byte) bool { + return bytes.Equal(a, b) + })) + var gotBuf, wantBuf bytes.Buffer + got.WriteTo(&gotBuf) + test.Want.WriteTo(&wantBuf) + t.Errorf( + "wrong result\nvalue: %#v\ngot: %s\nwant: %s\ndiff: %s", + test.Val, gotBuf.String(), wantBuf.String(), diff, + ) + } + }) + } +} + +func TestTokensForObject(t *testing.T) { + tests := map[string]struct { + Val []ObjectAttrTokens + Want Tokens + }{ + "no attributes": { + nil, + Tokens{ + {Type: hclsyntax.TokenOBrace, Bytes: []byte{'{'}}, + {Type: hclsyntax.TokenCBrace, Bytes: []byte{'}'}}, + }, + }, + "one attribute": { + []ObjectAttrTokens{ + { + Name: TokensForTraversal(hcl.Traversal{ + hcl.TraverseRoot{Name: "bar"}, + }), + Value: TokensForValue(cty.StringVal("baz")), + }, + }, + Tokens{ + {Type: hclsyntax.TokenOBrace, Bytes: []byte{'{'}}, + {Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("bar"), SpacesBefore: 2}, + {Type: hclsyntax.TokenEqual, Bytes: []byte{'='}, SpacesBefore: 1}, + {Type: hclsyntax.TokenOQuote, Bytes: []byte(`"`), SpacesBefore: 1}, + {Type: hclsyntax.TokenQuotedLit, Bytes: []byte("baz")}, + {Type: hclsyntax.TokenCQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}, + {Type: hclsyntax.TokenCBrace, Bytes: []byte{'}'}}, + }, + }, + "two attributes": { + []ObjectAttrTokens{ + { + Name: TokensForTraversal(hcl.Traversal{ + hcl.TraverseRoot{Name: "foo"}, + }), + Value: TokensForTraversal(hcl.Traversal{ + hcl.TraverseRoot{Name: "root"}, + hcl.TraverseAttr{Name: "attr"}, + }), + }, + { + Name: TokensForTraversal(hcl.Traversal{ + hcl.TraverseRoot{Name: "bar"}, + }), + Value: TokensForValue(cty.StringVal("baz")), + }, + }, + Tokens{ + {Type: hclsyntax.TokenOBrace, Bytes: []byte{'{'}}, + {Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("foo"), SpacesBefore: 2}, + {Type: hclsyntax.TokenEqual, Bytes: []byte{'='}, SpacesBefore: 1}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("root"), SpacesBefore: 1}, + {Type: hclsyntax.TokenDot, Bytes: []byte(".")}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("attr")}, + {Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("bar"), SpacesBefore: 2}, + {Type: hclsyntax.TokenEqual, Bytes: []byte{'='}, SpacesBefore: 1}, + {Type: hclsyntax.TokenOQuote, Bytes: []byte(`"`), SpacesBefore: 1}, + {Type: hclsyntax.TokenQuotedLit, Bytes: []byte("baz")}, + {Type: hclsyntax.TokenCQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenNewline, Bytes: []byte{'\n'}}, + {Type: hclsyntax.TokenCBrace, Bytes: []byte{'}'}}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + got := TokensForObject(test.Val) + + if !cmp.Equal(got, test.Want) { + diff := cmp.Diff(got, test.Want, cmp.Comparer(func(a, b []byte) bool { + return bytes.Equal(a, b) + })) + var gotBuf, wantBuf bytes.Buffer + got.WriteTo(&gotBuf) + test.Want.WriteTo(&wantBuf) + t.Errorf( + "wrong result\nvalue: %#v\ngot: %s\nwant: %s\ndiff: %s", + test.Val, gotBuf.String(), wantBuf.String(), diff, + ) + } + }) + } +} + +func TestTokensForFunctionCall(t *testing.T) { + tests := map[string]struct { + FuncName string + Val []Tokens + Want Tokens + }{ + "no arguments": { + "uuid", + nil, + Tokens{ + {Type: hclsyntax.TokenIdent, Bytes: []byte("uuid")}, + {Type: hclsyntax.TokenOParen, Bytes: []byte{'('}}, + {Type: hclsyntax.TokenCParen, Bytes: []byte(")")}, + }, + }, + "one argument": { + "strlen", + []Tokens{ + TokensForValue(cty.StringVal("hello")), + }, + Tokens{ + {Type: hclsyntax.TokenIdent, Bytes: []byte("strlen")}, + {Type: hclsyntax.TokenOParen, Bytes: []byte{'('}}, + {Type: hclsyntax.TokenOQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenQuotedLit, Bytes: []byte("hello")}, + {Type: hclsyntax.TokenCQuote, Bytes: []byte(`"`)}, + {Type: hclsyntax.TokenCParen, Bytes: []byte(")")}, + }, + }, + "two arguments": { + "list", + []Tokens{ + TokensForIdentifier("string"), + TokensForIdentifier("int"), + }, + Tokens{ + {Type: hclsyntax.TokenIdent, Bytes: []byte("list")}, + {Type: hclsyntax.TokenOParen, Bytes: []byte{'('}}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("string")}, + {Type: hclsyntax.TokenComma, Bytes: []byte(",")}, + {Type: hclsyntax.TokenIdent, Bytes: []byte("int"), SpacesBefore: 1}, + {Type: hclsyntax.TokenCParen, Bytes: []byte(")")}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + got := TokensForFunctionCall(test.FuncName, test.Val...) + + if !cmp.Equal(got, test.Want) { + diff := cmp.Diff(got, test.Want, cmp.Comparer(func(a, b []byte) bool { + return bytes.Equal(a, b) + })) + var gotBuf, wantBuf bytes.Buffer + got.WriteTo(&gotBuf) + test.Want.WriteTo(&wantBuf) + t.Errorf( + "wrong result\nvalue: %#v\ngot: %s\nwant: %s\ndiff: %s", + test.Val, gotBuf.String(), wantBuf.String(), diff, + ) + } + }) + } +} + +func TestTokenGenerateConsistency(t *testing.T) { + + bytesComparer := cmp.Comparer(func(a, b []byte) bool { + return bytes.Equal(a, b) + }) + + // This test verifies that different ways of generating equivalent token + // sequences all generate identical tokens, to help us keep them all in + // sync under future maintanence. + + t.Run("tuple constructor", func(t *testing.T) { + tests := map[string]struct { + elems []cty.Value + }{ + "no elements": { + nil, + }, + "one element": { + []cty.Value{ + cty.StringVal("hello"), + }, + }, + "two elements": { + []cty.Value{ + cty.StringVal("hello"), + cty.StringVal("world"), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var listVal cty.Value + if len(test.elems) > 0 { + listVal = cty.ListVal(test.elems) + } else { + listVal = cty.ListValEmpty(cty.DynamicPseudoType) + } + fromListValue := TokensForValue(listVal) + fromTupleValue := TokensForValue(cty.TupleVal(test.elems)) + elemTokens := make([]Tokens, len(test.elems)) + for i, v := range test.elems { + elemTokens[i] = TokensForValue(v) + } + fromTupleTokens := TokensForTuple(elemTokens) + + if diff := cmp.Diff(fromListValue, fromTupleTokens, bytesComparer); diff != "" { + t.Errorf("inconsistency between TokensForValue(list) and TokensForTuple\n%s", diff) + } + if diff := cmp.Diff(fromTupleValue, fromTupleTokens, bytesComparer); diff != "" { + t.Errorf("inconsistency between TokensForValue(tuple) and TokensForTuple\n%s", diff) + } + + }) + } + }) + + t.Run("object constructor", func(t *testing.T) { + tests := map[string]struct { + attrs map[string]cty.Value + }{ + "no elements": { + nil, + }, + "one element": { + map[string]cty.Value{ + "greeting": cty.StringVal("hello"), + }, + }, + "two elements": { + map[string]cty.Value{ + "greeting1": cty.StringVal("hello"), + "greeting2": cty.StringVal("world"), + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var mapVal cty.Value + if len(test.attrs) > 0 { + mapVal = cty.MapVal(test.attrs) + } else { + mapVal = cty.MapValEmpty(cty.DynamicPseudoType) + } + fromMapValue := TokensForValue(mapVal) + fromObjectValue := TokensForValue(cty.ObjectVal(test.attrs)) + attrTokens := make([]ObjectAttrTokens, 0, len(test.attrs)) + for k, v := range test.attrs { + attrTokens = append(attrTokens, ObjectAttrTokens{ + Name: TokensForIdentifier(k), + Value: TokensForValue(v), + }) + } + fromObjectTokens := TokensForObject(attrTokens) + + if diff := cmp.Diff(fromMapValue, fromObjectTokens, bytesComparer); diff != "" { + t.Errorf("inconsistency between TokensForValue(map) and TokensForObject\n%s", diff) + } + if diff := cmp.Diff(fromObjectValue, fromObjectTokens, bytesComparer); diff != "" { + t.Errorf("inconsistency between TokensForValue(object) and TokensForObject\n%s", diff) + } + }) + } + }) +} diff --git a/hclwrite/tokens.go b/hclwrite/tokens.go index edb6483d..57a5fd2b 100644 --- a/hclwrite/tokens.go +++ b/hclwrite/tokens.go @@ -114,6 +114,16 @@ func (ts Tokens) BuildTokens(to Tokens) Tokens { return append(to, ts...) } +// ObjectAttrTokens represents the raw tokens for the name and value of +// one attribute in an object constructor expression. +// +// This is defined primarily for use with function TokensForObject. See +// that function's documentation for more information. +type ObjectAttrTokens struct { + Name Tokens + Value Tokens +} + func newIdentToken(name string) *Token { return &Token{ Type: hclsyntax.TokenIdent,