From 330791e758b809cd3e0fadba6ad27fc1af5c45d3 Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Fri, 4 Nov 2022 13:10:01 +0000 Subject: [PATCH] Addd Eject function to use from "pulumi convert" (#630) --- pkg/tf2pulumi/convert/convert.go | 76 +++++---------- pkg/tf2pulumi/convert/eject.go | 155 +++++++++++++++++++++++++++++++ pkg/tf2pulumi/convert/tf11.go | 131 +++----------------------- pkg/tf2pulumi/convert/tf12.go | 4 +- pkg/tfgen/convert_test.go | 2 +- 5 files changed, 193 insertions(+), 175 deletions(-) create mode 100644 pkg/tf2pulumi/convert/eject.go diff --git a/pkg/tf2pulumi/convert/convert.go b/pkg/tf2pulumi/convert/convert.go index 4e4bd5f81..2b59c800a 100644 --- a/pkg/tf2pulumi/convert/convert.go +++ b/pkg/tf2pulumi/convert/convert.go @@ -15,7 +15,6 @@ package convert import ( - "bytes" "io" "log" "os" @@ -34,7 +33,6 @@ import ( hcl2python "github.com/pulumi/pulumi/pkg/v3/codegen/python" "github.com/pulumi/pulumi/pkg/v3/codegen/schema" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) const ( @@ -75,59 +73,40 @@ func Convert(opts Options) (map[string][]byte, Diagnostics, error) { opts.ProviderInfoSource = il.PluginProviderInfoSource } - // Attempt to load the config as TF11 first. If this succeeds, use TF11 semantics unless either the config - // or the options specify otherwise. - generatedFiles, useTF12, tf11Err := convertTF11(opts) - if !useTF12 { - if tf11Err != nil { - return nil, Diagnostics{}, tf11Err - } - return generatedFiles, Diagnostics{}, nil - } - - var tf12Files []*syntax.File - var diagnostics hcl.Diagnostics - - if tf11Err == nil { - // Parse the config. - parser := syntax.NewParser() - for filename, contents := range generatedFiles { - err := parser.ParseFile(bytes.NewReader(contents), filename) - contract.Assert(err == nil) - } - if parser.Diagnostics.HasErrors() { - return nil, Diagnostics{All: parser.Diagnostics, files: parser.Files}, nil - } - tf12Files, diagnostics = parser.Files, append(diagnostics, parser.Diagnostics...) - } else { - files, diags := parseTF12(opts) - if !diags.HasErrors() { - tf12Files, diagnostics = files, append(diagnostics, diags...) - } else if opts.TerraformVersion != "11" { - return nil, Diagnostics{All: diags, files: files}, nil - } else { - return nil, Diagnostics{}, tf11Err - } + ejectOpts := EjectOptions{ + AllowMissingProperties: opts.AllowMissingProperties, + AllowMissingProviders: opts.AllowMissingProviders, + AllowMissingVariables: opts.AllowMissingVariables, + AllowMissingComments: opts.AllowMissingComments, + AnnotateNodesWithLocations: opts.AnnotateNodesWithLocations, + FilterResourceNames: opts.FilterResourceNames, + ResourceNameProperty: opts.ResourceNameProperty, + Root: opts.Root, + PackageCache: opts.PackageCache, + PluginHost: opts.PluginHost, + Loader: opts.Loader, + ProviderInfoSource: opts.ProviderInfoSource, + Logger: opts.Logger, + SkipResourceTypechecking: opts.SkipResourceTypechecking, + TargetSDKVersion: opts.TargetSDKVersion, + TerraformVersion: opts.TerraformVersion, } - tf12Files, program, programDiags, err := convertTF12(tf12Files, opts) - if err != nil { - return nil, Diagnostics{}, err - } + tfFiles, program, diagnostics, err := internalEject(ejectOpts) - diagnostics = append(diagnostics, programDiags...) if diagnostics.HasErrors() { - return nil, Diagnostics{All: diagnostics, files: tf12Files}, nil + return nil, Diagnostics{All: diagnostics, files: tfFiles}, nil } var genDiags hcl.Diagnostics + var generatedFiles map[string][]byte switch opts.TargetLanguage { case LanguageTypescript: generatedFiles, genDiags, err = hcl2nodejs.GenerateProgram(program) diagnostics = append(diagnostics, genDiags...) case LanguagePulumi: generatedFiles = map[string][]byte{} - for _, f := range tf12Files { + for _, f := range tfFiles { generatedFiles[f.Name] = f.Bytes } case LanguagePython: @@ -147,13 +126,13 @@ func Convert(opts Options) (map[string][]byte, Diagnostics, error) { diagnostics = append(diagnostics, genDiags...) } if err != nil { - return nil, Diagnostics{All: diagnostics, files: tf12Files}, err + return nil, Diagnostics{All: diagnostics, files: tfFiles}, err } if diagnostics.HasErrors() { - return nil, Diagnostics{All: diagnostics, files: tf12Files}, nil + return nil, Diagnostics{All: diagnostics, files: tfFiles}, nil } - return generatedFiles, Diagnostics{All: diagnostics, files: tf12Files}, nil + return generatedFiles, Diagnostics{All: diagnostics, files: tfFiles}, nil } type Options struct { @@ -200,10 +179,3 @@ type Options struct { // TargetOptions captures any target-specific options. TargetOptions interface{} } - -// logf writes a formatted message to the configured logger, if any. -func (o Options) logf(format string, arguments ...interface{}) { - if o.Logger != nil { - o.Logger.Printf(format, arguments...) - } -} diff --git a/pkg/tf2pulumi/convert/eject.go b/pkg/tf2pulumi/convert/eject.go new file mode 100644 index 000000000..d47e0ed53 --- /dev/null +++ b/pkg/tf2pulumi/convert/eject.go @@ -0,0 +1,155 @@ +// Copyright 2016-2018, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package convert + +import ( + "bytes" + "fmt" + "log" + "os" + + "github.com/hashicorp/hcl/v2" + "github.com/spf13/afero" + + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/il" + "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax" + "github.com/pulumi/pulumi/pkg/v3/codegen/pcl" + "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" +) + +type EjectOptions struct { + // AllowMissingProperties, if true, allows code-gen to continue even if the input configuration does not include. + // values for required properties. + AllowMissingProperties bool + // AllowMissingProviders, if true, allows code-gen to continue even if resource providers are missing. + AllowMissingProviders bool + // AllowMissingVariables, if true, allows code-gen to continue even if the input configuration references missing + // variables. + AllowMissingVariables bool + // AllowMissingComments allows binding to succeed even if there are errors extracting comments from the source. + AllowMissingComments bool + // AnnotateNodesWithLocations is true if the generated source code should contain comments that annotate top-level + // nodes with their original source locations. + AnnotateNodesWithLocations bool + // FilterResourceNames, if true, removes the property indicated by ResourceNameProperty from all resources in the + // graph. + FilterResourceNames bool + // ResourceNameProperty sets the key of the resource name property that will be removed if FilterResourceNames is + // true. + ResourceNameProperty string + // Root, when set, overrides the default filesystem used to load the source Terraform module. + Root afero.Fs + // Optional package cache. + PackageCache *pcl.PackageCache + // Optional plugin host. + PluginHost plugin.Host + // Optional Loader. + Loader schema.Loader + // Optional source for provider schema information. + ProviderInfoSource il.ProviderInfoSource + // Optional logger for diagnostic information. + Logger *log.Logger + // SkipResourceTypechecking, if true, allows code-gen to continue even if resource inputs fail to typecheck. + SkipResourceTypechecking bool + // The target SDK version. + TargetSDKVersion string + // The version of Terraform targeteds by the input configuration. + TerraformVersion string +} + +// logf writes a formatted message to the configured logger, if any. +func (o EjectOptions) logf(format string, arguments ...interface{}) { + if o.Logger != nil { + o.Logger.Printf(format, arguments...) + } +} + +// Eject converts a Terraform module at the provided location into a Pulumi module. +func Eject(dir string, loader schema.ReferenceLoader) (*workspace.Project, *pcl.Program, error) { + if loader == nil { + panic("must provide a non-nil loader") + } + + opts := EjectOptions{ + Root: afero.NewBasePathFs(afero.NewOsFs(), dir), + Loader: loader, + } + + tfFiles, program, diags, err := internalEject(opts) + + d := Diagnostics{All: diags, files: tfFiles} + diagWriter := d.NewDiagnosticWriter(os.Stderr, 0, true) + if len(diags) != 0 { + err := diagWriter.WriteDiagnostics(diags) + if err != nil { + return nil, nil, err + } + } + if err != nil { + return nil, nil, fmt.Errorf("failed to load Terraform configuration, %v", err) + } + + project := &workspace.Project{ + Name: "tf2pulumi", + } + + return project, program, nil +} + +func internalEject(opts EjectOptions) ([]*syntax.File, *pcl.Program, hcl.Diagnostics, error) { + // Set default options where appropriate. + if opts.ProviderInfoSource == nil { + opts.ProviderInfoSource = il.PluginProviderInfoSource + } + + // Attempt to load the config as TF11 first. If this succeeds, use TF11 semantics unless either the config + // or the options specify otherwise. + generatedFiles, tf11Err := convertTF11(opts) + + var tf12Files []*syntax.File + var diagnostics hcl.Diagnostics + + if tf11Err == nil { + // Parse the config. + parser := syntax.NewParser() + for filename, contents := range generatedFiles { + err := parser.ParseFile(bytes.NewReader(contents), filename) + contract.Assert(err == nil) + } + tf12Files, diagnostics = parser.Files, append(diagnostics, parser.Diagnostics...) + if diagnostics.HasErrors() { + return tf12Files, nil, diagnostics, nil + } + } else { + tf12Files, diagnostics = parseTF12(opts) + if diagnostics.HasErrors() { + return tf12Files, nil, diagnostics, nil + } + } + + tf12Files, program, programDiags, err := convertTF12(tf12Files, opts) + if err != nil { + return nil, nil, nil, err + } + + diagnostics = append(diagnostics, programDiags...) + if diagnostics.HasErrors() { + return tf12Files, nil, diagnostics, nil + } + return tf12Files, program, diagnostics, nil +} diff --git a/pkg/tf2pulumi/convert/tf11.go b/pkg/tf2pulumi/convert/tf11.go index d47fe4031..2c191eaf8 100644 --- a/pkg/tf2pulumi/convert/tf11.go +++ b/pkg/tf2pulumi/convert/tf11.go @@ -8,17 +8,12 @@ import ( "sort" "strings" - "github.com/hashicorp/hcl/hcl/token" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hil/ast" - "github.com/pkg/errors" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/gen" - "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/gen/nodejs" - "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/gen/python" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/il" - "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/internal/config" tf11module "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/internal/config/module" ) @@ -27,113 +22,31 @@ import ( // Note that the output of the conversion process may not be valid input for Terraform itself: in particular, the block // structure of the original source code may not be preserved, so entities that were blocks in the input may be // attributes in the output. The TF12 -> PCL converter must be able to handle this sort of input. -func convertTF11(opts Options) (map[string][]byte, bool, error) { +func convertTF11(opts EjectOptions) (map[string][]byte, error) { moduleStorage := tf11module.NewStorage(filepath.Join(".terraform", "modules")) mod, err := tf11module.NewTreeFs("", opts.Root) if err != nil { - return nil, true, fmt.Errorf("failed to create tree: %w", err) + return nil, fmt.Errorf("failed to create tree: %w", err) } if err = mod.Load(moduleStorage); err != nil { - return nil, true, fmt.Errorf("failed to load module: %w", err) + return nil, fmt.Errorf("failed to load module: %w", err) } gs, err := buildGraphs(mod, opts) if err != nil { - return nil, true, fmt.Errorf("failed to build graphs: %w", err) - } - - if opts.TerraformVersion == "12" || opts.TargetLanguage != "typescript" { - // Generate TF12 code from the TF11 graph, then pass the result off to the TF12 pipeline. - g := &tf11generator{} - g.Emitter = gen.NewEmitter(nil, g) - files, err := g.genModules(gs) - return files, true, err - } - - // Filter resource name properties if requested. - if opts.FilterResourceNames { - filterAutoNames := opts.ResourceNameProperty == "" - for _, g := range gs { - for _, r := range g.Resources { - if r.Config.Mode == config.ManagedResourceMode { - il.FilterProperties(r, func(key string, _ il.BoundNode) bool { - if filterAutoNames { - sch := r.Schemas().PropertySchemas(key).Pulumi - return sch == nil || sch.Default == nil || !sch.Default.AutoNamed - } - return key != opts.ResourceNameProperty - }) - } - } - } - } - - // Annotate nodes with the location of their original definition if requested. - if opts.AnnotateNodesWithLocations { - for _, g := range gs { - addLocationAnnotations(g) - } - } - - var buf bytes.Buffer - - generator, filename, err := newGenerator(&buf, "auto", opts) - if err != nil { - return nil, false, errors.Wrapf(err, "creating generator") - } - - if err = gen.Generate(gs, generator); err != nil { - return nil, false, err - } - - files := map[string][]byte{ - filename: buf.Bytes(), - } - return files, false, nil -} - -func addLocationAnnotation(location token.Pos, comments **il.Comments) { - if !location.IsValid() { - return + return nil, fmt.Errorf("failed to build graphs: %w", err) } - c := *comments - if c == nil { - c = &il.Comments{} - *comments = c - } - - if len(c.Leading) != 0 { - c.Leading = append(c.Leading, "") - } - c.Leading = append(c.Leading, fmt.Sprintf(" Originally defined at %v:%v", location.Filename, location.Line)) -} - -// addLocationAnnotations adds comments that record the original source location of each top-level node in a module. -func addLocationAnnotations(m *il.Graph) { - for _, n := range m.Modules { - addLocationAnnotation(n.Location, &n.Comments) - } - for _, n := range m.Providers { - addLocationAnnotation(n.Location, &n.Comments) - } - for _, n := range m.Resources { - addLocationAnnotation(n.Location, &n.Comments) - } - for _, n := range m.Outputs { - addLocationAnnotation(n.Location, &n.Comments) - } - for _, n := range m.Locals { - addLocationAnnotation(n.Location, &n.Comments) - } - for _, n := range m.Variables { - addLocationAnnotation(n.Location, &n.Comments) - } + // Generate TF12 code from the TF11 graph, then pass the result off to the TF12 pipeline. + g := &tf11generator{} + g.Emitter = gen.NewEmitter(nil, g) + files, err := g.genModules(gs) + return files, err } -func buildGraphs(tree *tf11module.Tree, opts Options) ([]*il.Graph, error) { +func buildGraphs(tree *tf11module.Tree, opts EjectOptions) ([]*il.Graph, error) { // TODO: move this into the il package and unify modules based on path children := []*il.Graph{} @@ -160,29 +73,7 @@ func buildGraphs(tree *tf11module.Tree, opts Options) ([]*il.Graph, error) { return append(children, g), nil } -func newGenerator(w io.Writer, projectName string, opts Options) (gen.Generator, string, error) { - switch opts.TargetLanguage { - case LanguageTypescript: - nodeOpts, ok := opts.TargetOptions.(nodejs.Options) - if !ok && opts.TargetOptions != nil { - return nil, "", errors.Errorf("invalid target options of type %T", opts.TargetOptions) - } - g, err := nodejs.New(projectName, opts.TargetSDKVersion, nodeOpts.UsePromptDataSources, w) - if err != nil { - return nil, "", err - } - return g, "index.ts", nil - case LanguagePython: - return python.New(projectName, w), "__main__.py", nil - default: - validLanguages := make([]string, len(ValidLanguages)) - copy(validLanguages, ValidLanguages) - return nil, "", errors.Errorf("invalid language '%s', expected one of %s", - opts.TargetLanguage, strings.Join(validLanguages, ", ")) - } -} - -// tf11generator generates Typescript code that targets the Pulumi libraries from a Terraform configuration. +// tf11generator generates tf12 code from a tf11 configuration. type tf11generator struct { // The emitter to use when generating code. *gen.Emitter diff --git a/pkg/tf2pulumi/convert/tf12.go b/pkg/tf2pulumi/convert/tf12.go index 83c0b31a4..1d7f1759f 100644 --- a/pkg/tf2pulumi/convert/tf12.go +++ b/pkg/tf2pulumi/convert/tf12.go @@ -36,7 +36,7 @@ func parseFile(parser *syntax.Parser, fs afero.Fs, path string) error { } // parseTF12 parses a TF12 config. -func parseTF12(opts Options) ([]*syntax.File, hcl.Diagnostics) { +func parseTF12(opts EjectOptions) ([]*syntax.File, hcl.Diagnostics) { // Find the config files in the requested directory. configs, overrides, diags := configs.NewParser(opts.Root).ConfigDirFiles("/") if diags.HasErrors() { @@ -73,7 +73,7 @@ func parseTF12(opts Options) ([]*syntax.File, hcl.Diagnostics) { return parser.Files, parser.Diagnostics } -func convertTF12(files []*syntax.File, opts Options) ([]*syntax.File, *pcl.Program, hcl.Diagnostics, error) { +func convertTF12(files []*syntax.File, opts EjectOptions) ([]*syntax.File, *pcl.Program, hcl.Diagnostics, error) { var hcl2Options []model.BindOption var pulumiOptions []pcl.BindOption if opts.AllowMissingProperties { diff --git a/pkg/tfgen/convert_test.go b/pkg/tfgen/convert_test.go index 0eb6c6528..f9b9d12fb 100644 --- a/pkg/tfgen/convert_test.go +++ b/pkg/tfgen/convert_test.go @@ -74,7 +74,7 @@ func TestConvert(t *testing.T) { import * as pulumi from "@pulumi/pulumi"; const config = new pulumi.Config(); -const regionNumber = config.get("regionNumber") || { +const regionNumber = config.getObject("regionNumber") || { "us-east-1": 1, };`