From 40abc849b2a7c3e5565785082adc48718569b9c5 Mon Sep 17 00:00:00 2001 From: Robbie McKinstry Date: Wed, 7 Dec 2022 10:03:01 -0500 Subject: [PATCH] Reintroduce type splitting in NodeJS codegen. This commit reintroduces the changes that were reverted in https://github.com/pulumi/pulumi/pull/11534. This time, we hide the feature behind language-specific config in the schema, and default it to off. At this time, providers should not enable the feature until https://github.com/pulumi/pulumi/issues/11560 has been addressed. --- pkg/codegen/nodejs/gen.go | 409 +++++++++++++++++++++++++++++---- pkg/codegen/nodejs/importer.go | 4 + 2 files changed, 372 insertions(+), 41 deletions(-) diff --git a/pkg/codegen/nodejs/gen.go b/pkg/codegen/nodejs/gen.go index 40241014db09..1c80fc9c4d18 100644 --- a/pkg/codegen/nodejs/gen.go +++ b/pkg/codegen/nodejs/gen.go @@ -43,9 +43,12 @@ import ( ) // The minimum version of @pulumi/pulumi compatible with the generated SDK. -const MinimumValidSDKVersion string = "^3.42.0" -const MinimumTypescriptVersion string = "^4.3.5" -const MinimumNodeTypesVersion string = "^14" +const ( + MinimumValidSDKVersion string = "^3.42.0" + MinimumTypescriptVersion string = "^4.3.5" + MinimumNodeTypesVersion string = "^14" + SplitTypesDefault = false +) type typeDetails struct { outputType bool @@ -1545,45 +1548,166 @@ func (mod *modContext) genConfig(w io.Writer, variables []*schema.Property) erro return nil } -func (mod *modContext) getRelativePath() string { - rel, err := filepath.Rel(mod.mod, "") +// getRelativePath returns a path to the top level of the package +// relative to directory passed in. You must pass in the name +// of a directory. If you provide a file name, like "index.ts", it's assumed +// to be a directory named "index.ts". +// It's a thin wrapper around the standard library's implementation. +func getRelativePath(dirname string) string { + var rel, err = filepath.Rel(dirname, "") contract.Assert(err == nil) return path.Dir(filepath.ToSlash(rel)) } -func (mod *modContext) sdkImports(nested, utilities bool) []string { - imports := []string{"import * as pulumi from \"@pulumi/pulumi\";"} +// The parameter dirRoot is used as the relative path +func (mod *modContext) getRelativePath() string { + return getRelativePath(mod.mod) +} - relRoot := mod.getRelativePath() +// sdkImports generates the imports at the top of a source file. +// This function is only intended to be called from resource files and +// at the index. For files nested in the `/types` folder, call +// sdkImportsForTypes instead. +func (mod *modContext) sdkImports(nested, utilities bool) []string { + return mod.sdkImportsWithPath(nested, utilities, mod.mod) +} + +// sdkImportsWithPath generates the import functions at the top of each file. +// If nested is true, then the file is assumed not to be at the top-level i.e. +// it's located in a subfolder of the root, perhaps deeply nested. +// If utilities is true, then utility functions are imported. +// dirpath injects the directory name of the input file relative to the root of the package. +func (mod *modContext) sdkImportsWithPath(nested, utilities bool, dirpath string) []string { + // All files need to import the SDK. + var imports = []string{"import * as pulumi from \"@pulumi/pulumi\";"} + var relRoot = getRelativePath(dirpath) + // Add nested imports if enabled. if nested { - imports = append(imports, []string{ - fmt.Sprintf(`import * as inputs from "%s/types/input";`, relRoot), - fmt.Sprintf(`import * as outputs from "%s/types/output";`, relRoot), - }...) - - if mod.pkg.Language["nodejs"].(NodePackageInfo).ContainsEnums { - code := `import * as enums from "%s/types/enums";` - if lookupNodePackageInfo(mod.pkg).UseTypeOnlyReferences { - code = `import type * as enums from "%s/types/enums";` - } - imports = append(imports, fmt.Sprintf(code, relRoot)) - } + imports = append(imports, mod.genNestedImports(relRoot)...) } - + // Add utility imports if enabled. if utilities { - imports = append(imports, mod.utilitiesImport()) + imports = append(imports, utilitiesImport(relRoot)) } return imports } -func (mod *modContext) utilitiesImport() string { - relRoot := mod.getRelativePath() +func utilitiesImport(relRoot string) string { return fmt.Sprintf("import * as utilities from \"%s/utilities\";", relRoot) } -func (mod *modContext) genTypes() (string, string, error) { - externalImports, imports := codegen.NewStringSet(), map[string]codegen.StringSet{} +// relRoot is the path that ajoins this module or file with the top-level +// of the repo. For example, if this file was located in "./foo/index.ts", +// then relRoot would be "..", since that's the path to the top-level directory. +func (mod *modContext) genNestedImports(relRoot string) []string { + // Always import all input and output types. + var imports = []string{ + fmt.Sprintf(`import * as inputs from "%s/types/input";`, relRoot), + fmt.Sprintf(`import * as outputs from "%s/types/output";`, relRoot), + } + // Next, if there are enums, then we import them too. + if mod.pkg.Language["nodejs"].(NodePackageInfo).ContainsEnums { + code := `import * as enums from "%s/types/enums";` + if lookupNodePackageInfo(mod.pkg).UseTypeOnlyReferences { + code = `import type * as enums from "%s/types/enums";` + } + imports = append(imports, fmt.Sprintf(code, relRoot)) + } + return imports +} + +// the parameter defaultNs is expected to be the top-level namespace. +// If its nil, then we skip importing types files, since they will not exist. +func (mod *modContext) buildImports() (codegen.StringSet, map[string]codegen.StringSet) { + var externalImports = codegen.NewStringSet() + var imports = map[string]codegen.StringSet{} + for _, t := range mod.types { + if t.IsOverlay { + // This type is generated by the provider, so no further action is required. + continue + } + + mod.getImports(t, externalImports, imports) + } + return externalImports, imports +} + +func (mod *modContext) splitTypesEnabled() bool { + var info, ok = mod.pkg.Language["nodejs"].(NodePackageInfo) + // Use the default if no language-specific configuration was provided. + if !ok { + return SplitTypesDefault + } + // Use the default if no config was explicitly provided. + if info.SplitTypesFiles == nil { + return SplitTypesDefault + } + return *info.SplitTypesFiles +} + +func (mod *modContext) genTypes() ([]*ioFile, error) { + if mod.splitTypesEnabled() { + return mod.genTypesWithSplitting() + } + return mod.genTypesNoSplitting() +} + +func (mod *modContext) genTypesWithSplitting() ([]*ioFile, error) { + var ( + inputFiles, outputFiles []*ioFile + err error + // Build a file tree out of the types, then emit them. + namespaces = mod.getNamespaces() + // Fetch the collection of imports needed by these modules. + externalImports, imports = mod.buildImports() + buildCtx = func(input bool) *ioContext { + return &ioContext{ + mod: mod, + input: input, + imports: imports, + externalImports: externalImports, + } + } + + inputCtx = buildCtx(true) + outputCtx = buildCtx(false) + ) + // Convert the imports into a path relative to the root. + + var modifiedImports = map[string]codegen.StringSet{} + for name, value := range imports { + var modifiedName = path.Base(name) + // Special case: If we're importing the top-level of the module, leave as is. + if name == ".." { + modifiedImports[name] = value + continue + } + var relativePath = fmt.Sprintf("@/%s", modifiedName) + modifiedImports[relativePath] = value + } + inputCtx.imports = modifiedImports + outputCtx.imports = modifiedImports + + // If there are no namespaces, then we generate empty + // input and output files. + if namespaces[""] == nil { + return nil, fmt.Errorf("encountered a nil top-level namespace, and namespaces can't be nil even if it is empty") + } + // Iterate through the namespaces, generating one per node in the tree. + if inputFiles, err = namespaces[""].intoIOFiles(inputCtx, "./types"); err != nil { + return nil, err + } + if outputFiles, err = namespaces[""].intoIOFiles(outputCtx, "./types"); err != nil { + return nil, err + } + + return append(inputFiles, outputFiles...), nil +} + +func (mod *modContext) genTypesNoSplitting() ([]*ioFile, error) { + var modDir = strings.ToLower(mod.mod) + var externalImports, imports = codegen.NewStringSet(), map[string]codegen.StringSet{} var hasDefaultObjects bool for _, t := range mod.types { if t.IsOverlay { @@ -1602,20 +1726,27 @@ func (mod *modContext) genTypes() (string, string, error) { externalImports.Add(fmt.Sprintf("import * as utilities from \"%s/utilities\";", mod.getRelativePath())) } - inputs, outputs := &bytes.Buffer{}, &bytes.Buffer{} - mod.genHeader(inputs, mod.sdkImports(true, false), externalImports, imports) - mod.genHeader(outputs, mod.sdkImports(true, false), externalImports, imports) + var inputName = path.Join(modDir, "input.ts") + var outputName = path.Join(modDir, "output.ts") + var inputs, outputs = newIOFile(inputName), newIOFile(outputName) + var allFiles = []*ioFile{inputs, outputs} + mod.genHeader(inputs.writer(), mod.sdkImports(true, false), externalImports, imports) + mod.genHeader(outputs.writer(), mod.sdkImports(true, false), externalImports, imports) // Build a namespace tree out of the types, then emit them. + // We emit these namespaces as individual files if SplitTypes is enabled. + // Otherwise, they are written to /types/{input, output}.ts, and defined + // as nested TypeScript namespaces. namespaces := mod.getNamespaces() - if err := mod.genNamespace(inputs, namespaces[""], true, 0); err != nil { - return "", "", err + + if err := mod.genNamespace(inputs.writer(), namespaces[""], true, 0); err != nil { + return allFiles, err } - if err := mod.genNamespace(outputs, namespaces[""], false, 0); err != nil { - return "", "", err + if err := mod.genNamespace(outputs.writer(), namespaces[""], false, 0); err != nil { + return allFiles, err } - return inputs.String(), outputs.String(), nil + return allFiles, nil } type namespace struct { @@ -1625,6 +1756,19 @@ type namespace struct { children []*namespace } +// sortItems will sort each of the internal slices of this object. +func (ns *namespace) sortItems() { + sort.Slice(ns.types, func(i, j int) bool { + return objectTypeLessThan(ns.types[i], ns.types[j]) + }) + sort.Slice(ns.enums, func(i, j int) bool { + return tokenToName(ns.enums[i].Token) < tokenToName(ns.enums[j].Token) + }) + sort.Slice(ns.children, func(i, j int) bool { + return ns.children[i].name < ns.children[j].name + }) +} + func (mod *modContext) getNamespaces() map[string]*namespace { namespaces := map[string]*namespace{} var getNamespace func(string) *namespace @@ -1710,6 +1854,191 @@ func (mod *modContext) genNamespace(w io.Writer, ns *namespace, input bool, leve return nil } +// An ioFile represents a file containing Input/Output type definitions. +type ioFile struct { + // Each file has a name relative to the top-level directory. + filename string + // This writer stores the contents of the file as we build it incrementally. + buffer *bytes.Buffer +} + +// newIOFile constructs a new ioFile +func newIOFile(name string) *ioFile { + return &ioFile{ + filename: name, + buffer: bytes.NewBuffer(nil), + } +} + +func (f *ioFile) name() string { + return f.filename +} + +func (f *ioFile) writer() io.Writer { + return f.buffer +} + +func (f *ioFile) contents() []byte { + return f.buffer.Bytes() +} + +// ioContext defines a set of parameters used when generating input/output +// type definitions. These parameters are stable no matter which directory +// is getting generated. +type ioContext struct { + mod *modContext + input bool + imports map[string]codegen.StringSet + externalImports codegen.StringSet +} + +// filename returns the unique name of the input/output file +// given a directory root. +func (ctx *ioContext) filename(dirRoot string) string { + var fname = fmt.Sprintf("%s.ts", ctx.filetype()) + return path.Join(dirRoot, fname) +} + +func (ctx *ioContext) filetype() string { + if ctx.input { + return "input" + } + return "output" +} + +// intoIOFiles converts this namespace into one or more files. +// It recursively builds one file for each node in the tree. +// If ctx.input=true, then it builds input types. Otherwise, it +// builds output types. +// The parameters in ctx are stable regardless of the depth of recursion, +// but parent is expected to change with each recursive call. +func (ns *namespace) intoIOFiles(ctx *ioContext, parent string) ([]*ioFile, error) { + // We generate the input and output namespaces when there are enums, + // regardless of whether they are empty. + if ns == nil { + return nil, fmt.Errorf("Generating IO files for a nil namespace") + } + + // We want to organize the items in the source file by alphabetical order. + // Before we go any further, sort the items so downstream calls don't have to. + ns.sortItems() + + // Declare a new file to store the contents exposed at this directory level. + var dirRoot = path.Join(parent, ns.name) + var file, err = ns.genOwnedTypes(ctx, dirRoot) + if err != nil { + return nil, err + } + var files = []*ioFile{file} + + // We have successfully written all types at this level to + // input.ts/output.ts. + // Next, we want to recurse to the next directory level. + // We also need to write the index.file at this level (only once), + // and when we do, we need to re-export the items subdirectories. + children, err := ns.genNestedTypes(ctx, dirRoot) + if err != nil { + return files, err + } + files = append(files, children...) + // Lastly, we write the index file for this directory once. + // We don't want to generate the file twice, when this function is called + // with input=true and again when input=false, so we only generate it + // when input=true. + // As a special case, we skip the top-level directory /types/, since that + // is written elsewhere. + if parent != "./types" && ctx.input { + var indexFile = ns.genIndexFile(ctx, dirRoot) + files = append(files, indexFile) + } + return files, nil +} + +// genOwnedTypes generates the types for the file in the current context. +// The file is unique to the ioContext (either input/output) and the current +// directory (dirRoot). It skips over types that are neither input nor output types. +func (ns *namespace) genOwnedTypes(ctx *ioContext, dirRoot string) (*ioFile, error) { + // file is either an input file or an output file. + var file = newIOFile(ctx.filename(dirRoot)) + // We start every file with the header information. + ctx.mod.genHeader( + file.writer(), + ctx.mod.sdkImportsWithPath(true, true, dirRoot), + ctx.externalImports, + ctx.imports, + ) + // Next, we recursively export the nested types at each subdirectory. + for _, child := range ns.children { + // Defensive coding: child should never be null, but + // child.intoIOFiles will break if it is. + if child == nil { + continue + } + fmt.Fprintf(file.writer(), "export * as %[1]s from \"./%[1]s/%[2]s\";\n", child.name, ctx.filetype()) + } + + // Now, we write out the types declared at this directory + // level to the file. + for i, t := range ns.types { + var isInputType = ctx.input && ctx.mod.details(t).inputType + var isOutputType = !ctx.input && ctx.mod.details(t).outputType + // Only write input and output types. + if isInputType || isOutputType { + if err := ctx.mod.genType(file.writer(), t, ctx.input, 0); err != nil { + return file, err + } + if i != len(ns.types)-1 { + fmt.Fprintf(file.writer(), "\n") + } + } + } + return file, nil +} + +// genNestedTypes will recurse to child namespaces and generate those files. +// It will also generate the index file for this namespace, re-exporting +// identifiers in child namespaces as they are created. +func (ns *namespace) genNestedTypes(ctx *ioContext, dirRoot string) ([]*ioFile, error) { + var files []*ioFile + for _, child := range ns.children { + // Defensive coding: child should never be null, but + // child.intoIOFiles will break if it is. + if child == nil { + continue + } + // At this level, we export any nested definitions from + // the next level. + nestedFiles, err := child.intoIOFiles(ctx, dirRoot) + if err != nil { + return nil, err + } + // Collect these files to return. + files = append(files, nestedFiles...) + } + return files, nil +} + +// genIndexTypes generates an index.ts file for this directory. It must be called +// only once. It exports the files defined at this level in input.ts and output.ts, +// and it exposes the types in all submodules one level down. +func (ns *namespace) genIndexFile(ctx *ioContext, dirRoot string) *ioFile { + var indexPath = path.Join(dirRoot, "index.ts") + var file = newIOFile(indexPath) + ctx.mod.genHeader(file.writer(), nil, nil, nil) + // Export the types defined at the current level. + // Now, recursively export the items in each submodule. + for _, child := range ns.children { + // Defensive coding: child should never be null, but + // child.intoIOFiles will break if it is. + if child == nil { + continue + } + + fmt.Fprintf(file.writer(), "export * as %[1]s from \"./%[1]s/%[2]s\";\n", child.name, ctx.filetype()) + } + return file +} + func enumMemberName(typeName string, member *schema.Enum) (string, error) { if member.Name == "" { member.Name = fmt.Sprintf("%v", member.Value) @@ -1901,16 +2230,14 @@ func (mod *modContext) gen(fs codegen.Fs) error { fileName = path.Join("types", "enums", fileName) fs.Add(fileName, buffer.Bytes()) } - - // Nested types - // Importing enums always imports inputs and outputs, so if we have enums we generate inputs and outputs if len(mod.types) > 0 || (mod.pkg.Language["nodejs"].(NodePackageInfo).ContainsEnums && mod.mod == "types") { - input, output, err := mod.genTypes() + var files, err = mod.genTypes() if err != nil { return err } - fs.Add(path.Join(modDir, "input.ts"), []byte(input)) - fs.Add(path.Join(modDir, "output.ts"), []byte(output)) + for _, file := range files { + fs.Add(file.name(), file.contents()) + } } // Index @@ -1951,7 +2278,7 @@ func (mod *modContext) genIndex(exports []fileInfo) string { imports = mod.sdkImports(false /*nested*/, true /*utilities*/) } else if len(children) > 0 || len(mod.functions) > 0 { // Even if there are no resources, exports ref utilities. - imports = append(imports, mod.utilitiesImport()) + imports = append(imports, utilitiesImport(mod.getRelativePath())) } mod.genHeader(w, imports, nil, nil) diff --git a/pkg/codegen/nodejs/importer.go b/pkg/codegen/nodejs/importer.go index d3f158f85c69..a8514fb87016 100644 --- a/pkg/codegen/nodejs/importer.go +++ b/pkg/codegen/nodejs/importer.go @@ -79,6 +79,10 @@ type NodePackageInfo struct { // requires TypeScript 3.8 or higher to compile the generated // code. UseTypeOnlyReferences bool `json:"useTypeOnlyReferences,omitempty"` + + // Experimental flag that splits type definitions at /types + // into smaller units, recursively creating new files for modules. + SplitTypesFiles *bool `json:"split_types_files,omitempty"` } // NodeObjectInfo contains NodeJS-specific information for an object.