From fdcd784d5f4ece22b69d077abd7b1ca7f3f90a3a Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 16 Feb 2024 10:41:09 -0500 Subject: [PATCH] rpc: allow using both gob/protobuf for structs As follow-up to the introduction of the protobufs for HCLSpec, we introduce a new environment variable and code to use those structures, so we don't use gob for serialising HCLSpecs. This should make the plugins and packer able to transmit data over-the-wire without using gob for the most part (the communicators still use it, and will probably need some work to replace). --- Makefile | 2 +- go.mod | 11 +++-- go.sum | 22 ++++++++-- plugin/server.go | 2 +- rpc/common.go | 104 ++++++++++++++++++++++++++++++++++++++++++---- rpc/datasource.go | 81 +++++++++++++++++++++++++++++------- 6 files changed, 191 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 2e86ca800..82d27908d 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ generate: install-gen-deps ## Generate dynamically generated code @find ./ -type f | xargs grep -l '^// Code generated' | xargs rm -f PROJECT_ROOT="$(CURDIR)" go generate ./... go fmt bootcommand/boot_command.go -# go run ./cmd/generate-fixer-deprecations + protoc rpc/hcl_spec.proto --go_out=. --go_opt=paths=source_relative generate-check: generate ## Check go code generation is on par @echo "==> Checking that auto-generated code is not changed..." diff --git a/go.mod b/go.mod index 70f3f37be..a0ba47790 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/agext/levenshtein v1.2.3 github.com/antchfx/xpath v1.1.11 // indirect github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go v1.44.114 + github.com/aws/aws-sdk-go v1.45.6 github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/dylanmei/iso8601 v0.1.0 // indirect github.com/dylanmei/winrmtest v0.0.0-20210303004826-fbc9ae56efb6 @@ -74,6 +74,12 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) +require ( + github.com/vmihailenco/msgpack/v5 v5.3.5 + google.golang.org/grpc v1.50.1 + google.golang.org/protobuf v1.28.1 +) + require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.1.1 // indirect @@ -110,14 +116,13 @@ require ( github.com/posener/complete v1.2.3 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/oauth2 v0.1.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect - google.golang.org/grpc v1.50.1 // indirect - google.golang.org/protobuf v1.28.1 // indirect ) go 1.20 diff --git a/go.sum b/go.sum index 366761cea..6d6495cbb 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.44.114 h1:plIkWc/RsHr3DXBj4MEw9sEW4CcL/e2ryokc+CKyq1I= -github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI= +github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -349,7 +349,12 @@ github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zclconf/go-cty v1.13.3 h1:m+b9q3YDbg6Bec5rr+KGy1MzEVzY/jC2X+YX4yqKtHI= github.com/zclconf/go-cty v1.13.3/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= @@ -366,6 +371,7 @@ golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -382,6 +388,7 @@ golang.org/x/mobile v0.0.0-20210901025245-1fde1d6c3ca1 h1:t3ZHqovedSY8DEAUmZA99f golang.org/x/mobile v0.0.0-20210901025245-1fde1d6c3ca1/go.mod h1:jFTmtFYCV0MFtXBU+J5V/+5AUeVS0ON/0WkE/KSrl6E= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -399,7 +406,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -411,6 +419,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -437,14 +446,17 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -452,6 +464,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= @@ -465,6 +478,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/plugin/server.go b/plugin/server.go index 5c657efab..57e66aed9 100644 --- a/plugin/server.go +++ b/plugin/server.go @@ -54,7 +54,7 @@ const ( // // Api version 4 did not have the notion of a minor version, so Packer will // error with a weird error message. - APIVersionMajor, APIVersionMinor = "5", "0" + APIVersionMajor, APIVersionMinor = "5", "1" ) var ErrManuallyStartedPlugin = errors.New( diff --git a/rpc/common.go b/rpc/common.go index 2a0dadc34..2f98606a3 100644 --- a/rpc/common.go +++ b/rpc/common.go @@ -7,12 +7,30 @@ import ( "bytes" "encoding/gob" "fmt" + "log" "net/rpc" + "os" + "reflect" "github.com/hashicorp/hcl/v2/hcldec" "github.com/zclconf/go-cty/cty" + "google.golang.org/protobuf/proto" ) +const PackerUsePBEnvVar = "PACKER_RPC_PB" + +// useProtobuf is used by Packer/plugins to determine if structs like +// hclspec.Spec should be serialised to protobuf instead of gob. +func useProtobuf() bool { + useProto := os.Getenv(PackerUsePBEnvVar) + switch useProto { + case "", "0", "no", "false": + return false + } + + return true +} + // commonClient allows to rpc call funcs that can be defined on the different // build blocks of Packer. type commonClient struct { @@ -48,21 +66,91 @@ func (p *commonClient) ConfigSpec() hcldec.ObjectSpec { panic(err.Error()) } - res := hcldec.ObjectSpec{} - err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res) + // Legacy: this will need to be removed when we discontinue gob-encoding + // + // This is required for backwards compatibility for now, but using + // gob to encode the spec objects will fail against the upstream cty + // library, since they removed support for it. + // + // This will be a breaking change, as older plugins won't be able to + // communicate with Packer any longer. + if !useProtobuf() { + log.Printf("[DEBUG] - common: receiving ConfigSpec as gob") + res := hcldec.ObjectSpec{} + err := gob.NewDecoder(bytes.NewReader(resp.ConfigSpec)).Decode(&res) + if err != nil { + panic(fmt.Errorf("failed to decode HCL spec from gob: %s", err)) + } + return res + } + + log.Printf("[DEBUG] - common: receiving ConfigSpec as protobuf") + spec, err := protobufToHCL2Spec(resp.ConfigSpec) if err != nil { - panic("ici:" + err.Error()) + panic(err) } - return res + + return spec } func (s *commonServer) ConfigSpec(_ interface{}, reply *ConfigSpecResponse) error { spec := s.selfConfigurable.ConfigSpec() - b := bytes.NewBuffer(nil) - err := gob.NewEncoder(b).Encode(spec) - reply.ConfigSpec = b.Bytes() - return err + if !useProtobuf() { + log.Printf("[DEBUG] - common: sending ConfigSpec as gob") + b := &bytes.Buffer{} + err := gob.NewEncoder(b).Encode(spec) + if err != nil { + return fmt.Errorf("failed to encode spec from gob: %s", err) + } + reply.ConfigSpec = b.Bytes() + + return nil + } + + log.Printf("[DEBUG] - common: sending ConfigSpec as protobuf") + rawBytes, err := hcl2SpecToProtobuf(spec) + if err != nil { + return fmt.Errorf("failed to encode HCL spec from protobuf: %s", err) + } + reply.ConfigSpec = rawBytes + + return nil +} + +// hcl2SpecToProtobuf converts a hcldec.ObjectSpec to a protobuf-serialised +// byte array so it can then be used to send to a Plugin/Packer. +func hcl2SpecToProtobuf(spec hcldec.ObjectSpec) ([]byte, error) { + ret, err := ToProto(spec) + if err != nil { + return nil, fmt.Errorf("failed to convert hcldec.Spec to hclspec.Spec: %s", err) + } + rawBytes, err := proto.Marshal(ret) + if err != nil { + return nil, fmt.Errorf("failed to serialise hclspec.Spec to protobuf: %s", err) + } + + return rawBytes, nil +} + +// protobufToHCL2Spec converts a protobuf-encoded spec to a usable hcldec.Spec. +func protobufToHCL2Spec(serData []byte) (hcldec.ObjectSpec, error) { + confSpec := &Spec{} + err := proto.Unmarshal(serData, confSpec) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal hclspec.Spec from raw protobuf: %q", err) + } + spec, err := confSpec.FromProto() + if err != nil { + return nil, fmt.Errorf("failed to decode HCL spec: %q", err) + } + + obj, ok := spec.(*hcldec.ObjectSpec) + if !ok { + return nil, fmt.Errorf("decoded HCL spec is not an object spec: %s", reflect.TypeOf(spec).String()) + } + + return *obj, nil } func init() { diff --git a/rpc/datasource.go b/rpc/datasource.go index 443eeee85..d7cab6f85 100644 --- a/rpc/datasource.go +++ b/rpc/datasource.go @@ -7,10 +7,12 @@ import ( "bytes" "encoding/gob" "fmt" + "log" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/msgpack" ) // An implementation of packer.Datasource where the data source is actually @@ -52,10 +54,21 @@ func (d *datasource) OutputSpec() hcldec.ObjectSpec { err := fmt.Errorf("Datasource.OutputSpec failed: %v", err) panic(err.Error()) } - res := hcldec.ObjectSpec{} - err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res) + + if !useProtobuf() { + log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob") + res := hcldec.ObjectSpec{} + err := gob.NewDecoder(bytes.NewReader(resp.OutputSpec)).Decode(&res) + if err != nil { + panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from gob: %s", err)) + } + return res + } + + log.Printf("[DEBUG] - datasource: receiving OutputSpec as gob") + res, err := protobufToHCL2Spec(resp.OutputSpec) if err != nil { - panic("ici:" + err.Error()) + panic(fmt.Sprintf("datasource: failed to deserialise HCL spec from protobuf: %s", err)) } return res } @@ -66,20 +79,35 @@ type ExecuteResponse struct { } func (d *datasource) Execute() (cty.Value, error) { - res := new(cty.Value) resp := new(ExecuteResponse) if err := d.client.Call(d.endpoint+".Execute", new(interface{}), resp); err != nil { err := fmt.Errorf("Datasource.Execute failed: %v", err) - return *res, err + return cty.NilVal, err + } + + if !useProtobuf() { + log.Printf("[DEBUG] - datasource: receiving Execute as gob") + res := cty.Value{} + err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res) + if err != nil { + return res, fmt.Errorf("failed to unmarshal cty.Value from gob blob: %s", err) + } + if resp.Error != nil { + err = resp.Error + } + return res, err } - err := gob.NewDecoder(bytes.NewReader(resp.Value)).Decode(&res) + + log.Printf("[DEBUG] - datasource: receiving Execute as msgpack") + res, err := msgpack.Unmarshal(resp.Value, cty.DynamicPseudoType) if err != nil { - return *res, err + return cty.NilVal, fmt.Errorf("failed to unmarshal cty.Value from msgpack blob: %s", err) } + if resp.Error != nil { err = resp.Error } - return *res, err + return res, err } // DatasourceServer wraps a packer.Datasource implementation and makes it @@ -103,18 +131,43 @@ func (d *DatasourceServer) Configure(args *DatasourceConfigureArgs, reply *Datas func (d *DatasourceServer) OutputSpec(args *DatasourceConfigureArgs, reply *OutputSpecResponse) error { spec := d.d.OutputSpec() - b := bytes.NewBuffer(nil) - err := gob.NewEncoder(b).Encode(spec) - reply.OutputSpec = b.Bytes() + + if !useProtobuf() { + log.Printf("[DEBUG] - datasource: sending OutputSpec as gob") + b := &bytes.Buffer{} + err := gob.NewEncoder(b).Encode(spec) + reply.OutputSpec = b.Bytes() + return err + } + + log.Printf("[DEBUG] - datasource: sending OutputSpec as protobuf") + ret, err := hcl2SpecToProtobuf(spec) + if err != nil { + return err + } + reply.OutputSpec = ret + return err } func (d *DatasourceServer) Execute(args *interface{}, reply *ExecuteResponse) error { spec, err := d.d.Execute() reply.Error = NewBasicError(err) - b := bytes.NewBuffer(nil) - err = gob.NewEncoder(b).Encode(spec) - reply.Value = b.Bytes() + + if !useProtobuf() { + log.Printf("[DEBUG] - datasource: sending Execute as gob") + b := &bytes.Buffer{} + err = gob.NewEncoder(b).Encode(spec) + reply.Value = b.Bytes() + if reply.Error != nil { + err = reply.Error + } + return err + } + + log.Printf("[DEBUG] - datasource: sending Execute as msgpack") + raw, err := msgpack.Marshal(spec, cty.DynamicPseudoType) + reply.Value = raw if reply.Error != nil { err = reply.Error }