Merge #11510

11510: Phase 3 of the convert mapper r=Frassle a=Frassle

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Phase 3 of the new convert mapper interface. 
Pulls in the latest version of tfbridge.
Declares the GetMapping methods on the grpc and go interfaces.
Cleans up the commented out calls to GetMapping in this code base.
Adds some more comments documenting assumptions in the mapper class.

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [ ] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Fraser Waters <fraser@pulumi.com>

pkg/codegen/convert/mapper.go

@@ -22,10 +22,15 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
 )
 
+// An interface to map provider names (N.B. These aren't Pulumi provider names, but the names of "providers"
+// in the source language being converted from) to plugin specific mapping data.
 type Mapper interface {
+	// Returns plugin specific mapping data for the given provider name. Returns an empty result if no mapping
+	// information was available.
 	GetMapping(provider string) ([]byte, error)
 }
 
@@ -36,14 +41,20 @@ type pluginMapper struct {
 func NewPluginMapper(host plugin.Host, key string, mappings []string) (Mapper, error) {
 	entries := map[string][]byte{}
 
-	// Enumerate _all_ our installed plugins to ask for any mappings they provider. This allows users to
+	// Enumerate _all_ our installed plugins to ask for any mappings they provide. This allows users to
 	// convert aws terraform code for example by just having 'pulumi-aws' plugin locally, without needing to
-	// specify it anywhere on the command line, and without tf2pulumi needing to know about every possible plugin.
+	// specify it anywhere on the command line, and without tf2pulumi needing to know about every possible
+	// plugin.
 	plugins, err := workspace.GetPlugins()
 	if err != nil {
 		return nil, fmt.Errorf("could not get plugins: %w", err)
 	}
-	// We only care about the latest version of each plugin
+
+	// First assumption we only care about the latest version of each plugin. If we add support to get a
+	// mapping for plugin version 1, it seems unlikely that we would remove support for that mapping in v2, so
+	// the latest version should in most cases be fine. If a user case comes up where this is not fine we can
+	// provide the manual workaround that this is based on what is locally installed, not what is published
+	// and so the user can just delete the higher version plugins from their cache.
 	latestVersions := make(map[string]semver.Version)
 	for _, plugin := range plugins {
 		if plugin.Kind != workspace.ResourcePlugin {
@@ -58,28 +69,32 @@ func NewPluginMapper(host plugin.Host, key string, mappings []string) (Mapper, e
 			latestVersions[plugin.Name] = *plugin.Version
 		}
 	}
-	// Now go through each of those plugins and ask for any conversion data they have for the given key we're
-	// looking for.
-	//for pkg, version := range latestVersions {
-	// TODO: We have to do a dance here where first we publish a version of pulumi with these RPC structures
-	// then add methods to terraform-bridge to implement this method as if it did exist, and then actually add
-	// the RPC method and uncomment out the code below. This is all because we currently build these in a loop
-	// (pulumi include terraform-bridge, which includes pulumi).
 
-	//provider, err := host.Provider(tokens.Package(pkg), &version)
-	//if err != nil {
-	//	return nil, fmt.Errorf("could not create provider '%s': %w", pkg, err)
-	//}
+	// Now go through each of those plugins and ask for any conversion data they have for the given key we're
+	// looking for. Second assumption is that only one pulumi provider will provide a mapping for each source
+	// mapping. This _might_ change in the future if we for example add support to convert terraform to
+	// azure/aws-native, or multiple community members bridge the same terraform provider. But as above the
+	// decisions here are based on what's locally installed so the user can manually edit their plugin cache
+	// to be the set of plugins they want to use.
+	for pkg, version := range latestVersions {
+		version := version
+		provider, err := host.Provider(tokens.Package(pkg), &version)
+		if err != nil {
+			return nil, fmt.Errorf("could not create provider '%s': %w", pkg, err)
+		}
 
-	//data, mappedProvider, err := provider.GetMapping(key)
-	//if err != nil {
-	//	return nil, fmt.Errorf("could not get mapping for provider '%s': %w", pkg, err)
-	//}
-	//entries[mappedProvider] = data
-	//}
+		data, mappedProvider, err := provider.GetMapping(key)
+		if err != nil {
+			return nil, fmt.Errorf("could not get mapping for provider '%s': %w", pkg, err)
+		}
+		// A provider returns empty if it didn't have a mapping
+		if mappedProvider != "" && len(data) != 0 {
+			entries[mappedProvider] = data
+		}
+	}
 
 	// These take precedence over any plugin returned mappings so we do them last and just overwrite the
-	// entries
+	// entries.
 	for _, path := range mappings {
 		data, err := os.ReadFile(path)
 		if err != nil {
@@ -102,9 +117,5 @@ func NewPluginMapper(host plugin.Host, key string, mappings []string) (Mapper, e
 }
 
 func (l *pluginMapper) GetMapping(provider string) ([]byte, error) {
-	entry, ok := l.entries[provider]
-	if ok {
-		return entry, nil
-	}
-	return nil, fmt.Errorf("could not find any conversion mapping for %s", provider)
+	return l.entries[provider], nil
 }

pkg/go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pgavlin/goldmark v1.1.33-0.20200616210433-b5eb04559386
-	github.com/pulumi/pulumi/sdk/v3 v3.48.0
+	github.com/pulumi/pulumi/sdk/v3 v3.48.1-0.20221129095827-e43e98e5a7ab
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
 	github.com/sergi/go-diff v1.2.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
@@ -72,7 +72,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2
 	github.com/natefinch/atomic v1.0.1
 	github.com/pulumi/pulumi-java/pkg v0.6.0
-	github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221104131001-330791e758b8
+	github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221130185552-52ef9a815fb9
 	github.com/pulumi/pulumi-yaml v1.0.3
 	github.com/rivo/uniseg v0.2.0
 	github.com/segmentio/encoding v0.3.5

pkg/go.sum

@@ -1527,8 +1527,8 @@ github.com/prometheus/prometheus v0.37.0/go.mod h1:egARUgz+K93zwqsVIAneFlLZefyGO
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/pulumi/pulumi-java/pkg v0.6.0 h1:haiSQJlhrQIBBcR0r0aQCIF8i69e4znzRnHpaNQUchE=
 github.com/pulumi/pulumi-java/pkg v0.6.0/go.mod h1:xSK2B792P8zjwYZTHYapMM1RJdue2BpRFQNYObWO0C8=
-github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221104131001-330791e758b8 h1:cgw714pj+8Fq6RvJtRB5teKFj7MSyWpGMs6UdIK+1xk=
-github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221104131001-330791e758b8/go.mod h1:J3Jy0quNlS1bNZIFQK3dxUz34AR4FyWlTvdp20gD74A=
+github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221130185552-52ef9a815fb9 h1:d5x2w09F3jYUGI+dbkxKh9venuzh7b7osa1mb/aHWBY=
+github.com/pulumi/pulumi-terraform-bridge/v3 v3.33.1-0.20221130185552-52ef9a815fb9/go.mod h1:KwuxFDO7k2h3ZH5eTmd/zs3eQ5KlRTliI8vbAIdLcLI=
 github.com/pulumi/pulumi-yaml v1.0.3 h1:JDRQfGcQYxwJETIepa7T3B7wTV7Bk2P7uApmWvQZeL4=
 github.com/pulumi/pulumi-yaml v1.0.3/go.mod h1:KLFTyADnoNoPU9djMyRvXymrTH2SCBg9wnXaapuggEA=
 github.com/pulumi/ssh-agent v0.5.1 h1:7DT4FcZNHWBAp9BFI+k0+HeBYGWbJvilJ29ra/4FlRM=

pkg/resource/deploy/deploytest/provider.go

@@ -66,6 +66,8 @@ type Provider struct {
 		options plugin.CallOptions) (plugin.CallResult, error)
 
 	CancelF func() error
+
+	GetMappingF func(key string) ([]byte, string, error)
 }
 
 func (prov *Provider) SignalCancellation() error {
@@ -216,3 +218,10 @@ func (prov *Provider) Call(tok tokens.ModuleMember, args resource.PropertyMap, i
 	}
 	return prov.CallF(monitor, tok, args, info, options)
 }
+
+func (prov *Provider) GetMapping(key string) ([]byte, string, error) {
+	if prov.GetMappingF == nil {
+		return nil, "", nil
+	}
+	return prov.GetMappingF(key)
+}

pkg/resource/deploy/providers/registry_test.go

@@ -178,6 +178,9 @@ func (prov *testProvider) GetPluginInfo() (workspace.PluginInfo, error) {
 		Version: &prov.version,
 	}, nil
 }
+func (prov *testProvider) GetMapping(key string) ([]byte, string, error) {
+	return nil, "", nil
+}
 
 type providerLoader struct {
 	pkg     tokens.Package

proto/.checksum.txt

@@ -13,5 +13,5 @@
 3421371250 793 proto/pulumi/errors.proto
 3818289820 5711 proto/pulumi/language.proto
 2700626499 1743 proto/pulumi/plugin.proto
-3998073491 20713 proto/pulumi/provider.proto
+211074615 20823 proto/pulumi/provider.proto
 1325776472 11014 proto/pulumi/resource.proto

proto/pulumi/provider.proto

@@ -79,8 +79,9 @@ service ResourceProvider {
     // Attach sends the engine address to an already running plugin.
     rpc Attach(PluginAttach) returns (google.protobuf.Empty)  {}
 
-    // GetMapping fetches the mapping for this resource provider, if any.
-    // rpc GetMapping(GetMappingRequest) returns (GetMappingResponse) {}
+    // GetMapping fetches the mapping for this resource provider, if any. A provider should return an empty
+    // response (not an error) if it doesn't have a mapping for the given key.
+    rpc GetMapping(GetMappingRequest) returns (GetMappingResponse) {}
 }
 
 message GetSchemaRequest {

sdk/go/common/resource/plugin/provider.go

@@ -102,8 +102,9 @@ type Provider interface {
 	// called before (e.g.) hard-closing any gRPC connection.
 	SignalCancellation() error
 
-	// GetMapping returns the mapping (if any) for the provider.
-	// GetMapping(key string) ([]byte, string, error)
+	// GetMapping returns the mapping (if any) for the provider. A provider should return an empty response
+	// (not an error) if it doesn't have a mapping for the given key.
+	GetMapping(key string) ([]byte, string, error)
 }
 
 type GrpcProvider interface {

sdk/go/common/resource/plugin/provider_plugin.go

@@ -1736,17 +1736,18 @@ func decorateProviderSpans(span opentracing.Span, method string, req, resp inter
 
 // GetMapping fetches the conversion mapping (if any) for this resource provider.
 func (p *provider) GetMapping(key string) ([]byte, string, error) {
-	// TODO: We have to do a dance here where first we publish a version of pulumi with these RPC structures
-	// then add methods to terraform-bridge to implement this method as if it did exist, and then actually add
-	// the RPC method and uncomment out the code below. This is all because we currently build these in a loop
-	// (pulumi include terraform-bridge, which includes pulumi).
-	return nil, "", nil
-
-	//resp, err := p.clientRaw.GetMapping(p.requestContext(), &pulumirpc.GetMappingRequest{
-	//	Key: key,
-	//})
-	//if err != nil {
-	//	return nil, "", err
-	//}
-	//return resp.Data, resp.Provider, nil
+	resp, err := p.clientRaw.GetMapping(p.requestContext(), &pulumirpc.GetMappingRequest{
+		Key: key,
+	})
+	if err != nil {
+		rpcError := rpcerror.Convert(err)
+		code := rpcError.Code()
+		if code == codes.Unimplemented {
+			// For backwards compatibility, just return nothing as if the provider didn't have a mapping for
+			// the given key
+			return nil, "", nil
+		}
+		return nil, "", err
+	}
+	return resp.Data, resp.Provider, nil
 }

sdk/nodejs/proto/provider_grpc_pb.js

@@ -176,6 +176,28 @@ function deserialize_pulumirpc_DiffResponse(buffer_arg) {
   return pulumi_provider_pb.DiffResponse.deserializeBinary(new Uint8Array(buffer_arg));
 }
 
+function serialize_pulumirpc_GetMappingRequest(arg) {
+  if (!(arg instanceof pulumi_provider_pb.GetMappingRequest)) {
+    throw new Error('Expected argument of type pulumirpc.GetMappingRequest');
+  }
+  return Buffer.from(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_GetMappingRequest(buffer_arg) {
+  return pulumi_provider_pb.GetMappingRequest.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
+function serialize_pulumirpc_GetMappingResponse(arg) {
+  if (!(arg instanceof pulumi_provider_pb.GetMappingResponse)) {
+    throw new Error('Expected argument of type pulumirpc.GetMappingResponse');
+  }
+  return Buffer.from(arg.serializeBinary());
+}
+
+function deserialize_pulumirpc_GetMappingResponse(buffer_arg) {
+  return pulumi_provider_pb.GetMappingResponse.deserializeBinary(new Uint8Array(buffer_arg));
+}
+
 function serialize_pulumirpc_GetSchemaRequest(arg) {
   if (!(arg instanceof pulumi_provider_pb.GetSchemaRequest)) {
     throw new Error('Expected argument of type pulumirpc.GetSchemaRequest');
@@ -505,6 +527,19 @@ attach: {
     responseSerialize: serialize_google_protobuf_Empty,
     responseDeserialize: deserialize_google_protobuf_Empty,
   },
+  // GetMapping fetches the mapping for this resource provider, if any. A provider should return an empty
+// response (not an error) if it doesn't have a mapping for the given key.
+getMapping: {
+    path: '/pulumirpc.ResourceProvider/GetMapping',
+    requestStream: false,
+    responseStream: false,
+    requestType: pulumi_provider_pb.GetMappingRequest,
+    responseType: pulumi_provider_pb.GetMappingResponse,
+    requestSerialize: serialize_pulumirpc_GetMappingRequest,
+    requestDeserialize: deserialize_pulumirpc_GetMappingRequest,
+    responseSerialize: serialize_pulumirpc_GetMappingResponse,
+    responseDeserialize: deserialize_pulumirpc_GetMappingResponse,
+  },
 };
 
 exports.ResourceProviderClient = grpc.makeGenericClientConstructor(ResourceProviderService);