Skip to content

Commit

Permalink
First pass of adding the convert mapper
Browse files Browse the repository at this point in the history
  • Loading branch information
Frassle committed Nov 29, 2022
1 parent 9086bf9 commit 14d8b59
Show file tree
Hide file tree
Showing 15 changed files with 935 additions and 186 deletions.
18 changes: 16 additions & 2 deletions pkg/cmd/pulumi/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
javagen "github.com/pulumi/pulumi-java/pkg/codegen/java"
tfgen "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/convert"
yamlgen "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/codegen"
"github.com/pulumi/pulumi/pkg/v3/codegen/convert"
"github.com/pulumi/pulumi/pkg/v3/codegen/dotnet"
gogen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
hclsyntax "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
Expand All @@ -50,6 +51,7 @@ func newConvertCmd() *cobra.Command {
var from string
var language string
var generateOnly bool
var mappings []string

cmd := &cobra.Command{
Use: "convert",
Expand All @@ -64,7 +66,7 @@ func newConvertCmd() *cobra.Command {
return result.FromError(fmt.Errorf("could not resolve current working directory"))
}

return runConvert(cwd, from, language, outDir, generateOnly)
return runConvert(cwd, mappings, from, language, outDir, generateOnly)
}),
}

Expand All @@ -87,6 +89,10 @@ func newConvertCmd() *cobra.Command {
//nolint:lll
&generateOnly, "generate-only", false, "Generate the converted program(s) only; do not install dependencies")

cmd.PersistentFlags().StringSliceVar(
//nolint:lll
&mappings, "mappings", []string{}, "Any mapping files to use in the conversion")

return cmd
}

Expand Down Expand Up @@ -152,7 +158,10 @@ func pclEject(directory string, loader schema.ReferenceLoader) (*workspace.Proje
return &workspace.Project{Name: "pcl"}, program, nil
}

func runConvert(cwd string, from string, language string, outDir string, generateOnly bool) result.Result {
func runConvert(
cwd string, mappings []string, from string, language string,
outDir string, generateOnly bool) result.Result {

var projectGenerator projectGeneratorFunc
switch language {
case "csharp", "c#":
Expand Down Expand Up @@ -191,6 +200,11 @@ func runConvert(cwd string, from string, language string, outDir string, generat
}
defer contract.IgnoreClose(host)
loader := schema.NewPluginLoader(host)
// TODO: Mapper will be used by tfconvert (and others as we add them)
_, err = convert.NewPluginMapper(host, from, mappings)
if err != nil {
return result.FromError(fmt.Errorf("could not create provider mapper: %w", err))
}

var proj *workspace.Project
var program *pcl.Program
Expand Down
6 changes: 3 additions & 3 deletions pkg/cmd/pulumi/convert_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
// See: https://github.com/golang/vscode-go/wiki/debugging
//
// Your mileage may vary with other tooling.
func TestConvert(t *testing.T) {
func TestYamlConvert(t *testing.T) {
t.Parallel()

if info, err := os.Stat("convert_testdata/Pulumi.yaml"); err != nil && os.IsNotExist(err) {
Expand All @@ -32,7 +32,7 @@ func TestConvert(t *testing.T) {
t.Fatalf("Pulumi.yaml is a directory, not a file")
}

result := runConvert("convert_testdata", "yaml", "go", "convert_testdata/go", true)
result := runConvert("convert_testdata", []string{}, "yaml", "go", "convert_testdata/go", true)
require.Nil(t, result, "convert failed: %v", result)
}

Expand All @@ -44,7 +44,7 @@ func TestPclConvert(t *testing.T) {
tmp, err := os.MkdirTemp("", "pulumi-convert-test")
assert.NoError(t, err)

result := runConvert("pcl_convert_testdata", "pcl", "pcl", tmp, true)
result := runConvert("pcl_convert_testdata", []string{}, "pcl", "pcl", tmp, true)
assert.Nil(t, result)

// Check that we made one file
Expand Down
110 changes: 110 additions & 0 deletions pkg/codegen/convert/mapper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright 2016-2022, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package convert

import (
"fmt"
"os"
"path/filepath"
"strings"

"github.com/blang/semver"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
)

type Mapper interface {
GetMapping(provider string) ([]byte, error)
}

type pluginMapper struct {
entries map[string][]byte
}

func NewPluginMapper(host plugin.Host, key string, mappings []string) (Mapper, error) {
entries := map[string][]byte{}

// Enumerate _all_ our installed plugins to ask for any mappings they provider. This allows users to
// convert aws terraform code for example by just having 'pulumi-aws' plugin locally, without needing to
// specify it anywhere on the command line, and without tf2pulumi needing to know about every possible plugin.
plugins, err := workspace.GetPlugins()
if err != nil {
return nil, fmt.Errorf("could not get plugins: %w", err)
}
// We only care about the latest version of each plugin
latestVersions := make(map[string]semver.Version)
for _, plugin := range plugins {
if plugin.Kind != workspace.ResourcePlugin {
continue
}

if cur, has := latestVersions[plugin.Name]; has {
if plugin.Version.GT(cur) {
latestVersions[plugin.Name] = *plugin.Version
}
} else {
latestVersions[plugin.Name] = *plugin.Version
}
}
// Now go through each of those plugins and ask for any conversion data they have for the given key we're
// looking for.
//for pkg, version := range latestVersions {
// TODO: We have to do a dance here where first we publish a version of pulumi with these RPC structures
// then add methods to terraform-bridge to implement this method as if it did exist, and then actually add
// the RPC method and uncomment out the code below. This is all because we currently build these in a loop
// (pulumi include terraform-bridge, which includes pulumi).

//provider, err := host.Provider(tokens.Package(pkg), &version)
//if err != nil {
// return nil, fmt.Errorf("could not create provider '%s': %w", pkg, err)
//}

//data, mappedProvider, err := provider.GetMapping(key)
//if err != nil {
// return nil, fmt.Errorf("could not get mapping for provider '%s': %w", pkg, err)
//}
//entries[mappedProvider] = data
//}

// These take precedence over any plugin returned mappings so we do them last and just overwrite the
// entries
for _, path := range mappings {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("could not read mapping file '%s': %w", path, err)
}

// Mapping file names are assumed to be the provider key.
provider := filepath.Base(path)
// strip the extension
dotIndex := strings.LastIndex(provider, ".")
if dotIndex != -1 {
provider = provider[0:dotIndex]
}

entries[provider] = data
}
return &pluginMapper{
entries: entries,
}, nil
}

func (l *pluginMapper) GetMapping(provider string) ([]byte, error) {
entry, ok := l.entries[provider]
if ok {
return entry, nil
}
return nil, fmt.Errorf("could not find any conversion mapping for %s", provider)
}
4 changes: 4 additions & 0 deletions pkg/resource/deploy/builtins.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ func (p *builtinProvider) GetSchema(version int) ([]byte, error) {
return []byte("{}"), nil
}

func (p *builtinProvider) GetMapping(key string) ([]byte, string, error) {
return nil, "", nil
}

// CheckConfig validates the configuration for this resource provider.
func (p *builtinProvider) CheckConfig(urn resource.URN, olds,
news resource.PropertyMap, allowUnknowns bool) (resource.PropertyMap, []plugin.CheckFailure, error) {
Expand Down
6 changes: 6 additions & 0 deletions pkg/resource/deploy/providers/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,12 @@ func (r *Registry) GetSchema(version int) ([]byte, error) {
return nil, errors.New("the provider registry has no schema")
}

func (r *Registry) GetMapping(key string) ([]byte, string, error) {
contract.Fail()

return nil, "", errors.New("the provider registry has no mappings")
}

// CheckConfig validates the configuration for this resource provider.
func (r *Registry) CheckConfig(urn resource.URN, olds,
news resource.PropertyMap, allowUnknowns bool) (resource.PropertyMap, []plugin.CheckFailure, error) {
Expand Down
6 changes: 6 additions & 0 deletions pkg/resource/provider/component_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,9 @@ func (p *componentProvider) Attach(ctx context.Context,
p.host = host
return &pbempty.Empty{}, nil
}

// GetMapping fetches the conversion mapping (if any) for this resource provider.
func (p *componentProvider) GetMapping(ctx context.Context,
req *pulumirpc.GetMappingRequest) (*pulumirpc.GetMappingResponse, error) {
return &pulumirpc.GetMappingResponse{Provider: "", Data: nil}, nil
}
2 changes: 1 addition & 1 deletion proto/.checksum.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@
3421371250 793 proto/pulumi/errors.proto
3818289820 5711 proto/pulumi/language.proto
2700626499 1743 proto/pulumi/plugin.proto
1451439690 19667 proto/pulumi/provider.proto
3998073491 20713 proto/pulumi/provider.proto
1325776472 11014 proto/pulumi/resource.proto
22 changes: 22 additions & 0 deletions proto/pulumi/provider.proto
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ service ResourceProvider {

// Attach sends the engine address to an already running plugin.
rpc Attach(PluginAttach) returns (google.protobuf.Empty) {}

// GetMapping fetches the mapping for this resource provider, if any.
// rpc GetMapping(GetMappingRequest) returns (GetMappingResponse) {}
}

message GetSchemaRequest {
Expand Down Expand Up @@ -348,3 +351,22 @@ message ErrorResourceInitFailed {
repeated string reasons = 3; // error messages associated with initialization failure.
google.protobuf.Struct inputs = 4; // the current inputs to this resource (only applicable for Read)
}

// GetMappingRequest allows providers to return ecosystem specific information to allow the provider to be
// converted from a source markup to Pulumi. It's expected that provider bridges that target a given ecosystem
// (e.g. Terraform, Kubernetes) would also publish a conversion plugin to convert markup from that ecosystem
// to Pulumi, using the bridged providers.
message GetMappingRequest {
// the conversion key for the mapping being requested.
string key = 1;
}

// GetMappingResponse returns convert plugin specific data for this provider. This will normally be human
// readable JSON, but the engine doesn't mandate any form.
message GetMappingResponse {
// the provider key this is mapping for. For example the Pulumi provider "terraform-template" would return "template" for this.
string provider = 1;

// the conversion plugin specific data.
bytes data = 2;
}
3 changes: 3 additions & 0 deletions sdk/go/common/resource/plugin/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ type Provider interface {
// non-blocking; it is up to the host to decide how long to wait after SignalCancellation is
// called before (e.g.) hard-closing any gRPC connection.
SignalCancellation() error

// GetMapping returns the mapping (if any) for the provider.
// GetMapping(key string) ([]byte, string, error)
}

type GrpcProvider interface {
Expand Down
17 changes: 17 additions & 0 deletions sdk/go/common/resource/plugin/provider_plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -1733,3 +1733,20 @@ func decorateProviderSpans(span opentracing.Span, method string, req, resp inter
span.SetTag("pulumi-decorator", req.(*pulumirpc.InvokeRequest).Tok)
}
}

// GetMapping fetches the conversion mapping (if any) for this resource provider.
func (p *provider) GetMapping(key string) ([]byte, string, error) {
// TODO: We have to do a dance here where first we publish a version of pulumi with these RPC structures
// then add methods to terraform-bridge to implement this method as if it did exist, and then actually add
// the RPC method and uncomment out the code below. This is all because we currently build these in a loop
// (pulumi include terraform-bridge, which includes pulumi).
return nil, "", nil

//resp, err := p.clientRaw.GetMapping(p.requestContext(), &pulumirpc.GetMappingRequest{
// Key: key,
//})
//if err != nil {
// return nil, "", err
//}
//return resp.Data, resp.Provider, nil
}
15 changes: 15 additions & 0 deletions sdk/go/common/resource/plugin/provider_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -621,3 +621,18 @@ func (p *providerServer) Call(ctx context.Context, req *pulumirpc.CallRequest) (
Failures: rpcFailures,
}, nil
}

func (p *providerServer) GetMapping(ctx context.Context,
req *pulumirpc.GetMappingRequest) (*pulumirpc.GetMappingResponse, error) {
// TODO: We have to do a dance here where first we publish a version of pulumi with these RPC structures
// then add methods to terraform-bridge to implement this method as if it did exist, and then actually add
// the RPC method and uncomment out the code below. This is all because we currently build these in a loop
// (pulumi include terraform-bridge, which includes pulumi).
return &pulumirpc.GetMappingResponse{Data: nil, Provider: ""}, nil

//data, provider, err := p.provider.GetMapping(req.Key)
//if err != nil {
// return nil, err
//}
//return &pulumirpc.GetMappingResponse{Data: data, Provider: provider}, nil
}

0 comments on commit 14d8b59

Please sign in to comment.