diff --git a/changelog/pending/20240317--engine--default-provider-versions-can-be-set-in-pulumi-yaml.yaml b/changelog/pending/20240317--engine--default-provider-versions-can-be-set-in-pulumi-yaml.yaml new file mode 100644 index 000000000000..4c122b2dc2de --- /dev/null +++ b/changelog/pending/20240317--engine--default-provider-versions-can-be-set-in-pulumi-yaml.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: engine + description: Default provider versions can be set in Pulumi.yaml diff --git a/pkg/engine/plugins.go b/pkg/engine/plugins.go index a33003453c86..4e7343e4d4f8 100644 --- a/pkg/engine/plugins.go +++ b/pkg/engine/plugins.go @@ -290,11 +290,13 @@ func installPlugin(ctx context.Context, plugin workspace.PluginSpec) error { // computeDefaultProviderPlugins computes, for every resource plugin, a mapping from packages to semver versions // reflecting the version of a provider that should be used as the "default" resource when registering resources. This -// function takes two sets of plugins: a set of plugins given to us from the language host and the full set of plugins. -// If the language host has sent us a non-empty set of plugins, we will use those exclusively to service default -// provider requests. Otherwise, we will use the full set of plugins, which is the existing behavior today. +// function takes three sets of plugins: a set of plugins given to us from the language host, the full set of plugins +// from the snapshot, and the set of plugins from the project file. If the language host has sent us a non-empty set of +// plugins, we will use those exclusively to service default provider requests. Otherwise, we will use the full set of +// plugins, which is the existing behavior today. Either way anything in the project file is included and takes +// precedence. // -// The justification for favoring language plugins over all else is that, ultimately, it is the language plugin that +// The justification for favoring language plugins over the snapshot is that, ultimately, it is the language plugin that // produces resource registrations and therefore it is the language plugin that should dictate exactly what plugins to // use to satisfy a resource registration. SDKs have the opportunity to specify what plugin (pluginDownloadURL and // version) they want to use in RegisterResource. If the plugin is left unspecified, we make a best guess effort to @@ -305,7 +307,10 @@ func installPlugin(ctx context.Context, plugin workspace.PluginSpec) error { // that the engine uses to determine which version of a particular provider to load. // // it is critical that this function be 100% deterministic. -func computeDefaultProviderPlugins(languagePlugins, allPlugins pluginSet) map[tokens.Package]workspace.PluginSpec { +func computeDefaultProviderPlugins( + languagePlugins, allPlugins pluginSet, + projectPlugins []workspace.ProjectPlugin, +) map[tokens.Package]workspace.PluginSpec { // Language hosts are not required to specify the full set of plugins they depend on. If the set of plugins received // from the language host does not include any resource providers, fall back to the full set of plugins. languageReportedProviderPlugins := false @@ -372,6 +377,34 @@ func computeDefaultProviderPlugins(languagePlugins, allPlugins pluginSet) map[to defaultProviderPlugins[tokens.Package(p.Name)] = p } + // Now take into account the project plugins. These take precedence over anything the plugin queries returned. + for _, p := range projectPlugins { + if p.Kind != workspace.ResourcePlugin { + // Default providers are only relevant for resource plugins. + continue + } + + spec := workspace.PluginSpec{ + Name: p.Name, + Kind: p.Kind, + Version: p.Version, + } + + pkg := tokens.Package(p.Name) + + if seenPlugin, has := defaultProviderPlugins[pkg]; has { + logging.V(preparePluginLog).Infof( + "computeDefaultProviderPlugins(): plugin %s selected for package %s (override, previous was %s)", + spec, pkg, seenPlugin.Version) + defaultProviderPlugins[pkg] = spec + continue + } + + logging.V(preparePluginLog).Infof( + "computeDefaultProviderPlugins(): plugin %s selected for package %s (first seen)", spec, pkg) + defaultProviderPlugins[pkg] = spec + } + if logging.V(preparePluginLog) { logging.V(preparePluginLog).Infoln("computeDefaultProviderPlugins(): summary of default plugins:") for pkg, info := range defaultProviderPlugins { diff --git a/pkg/engine/plugins_test.go b/pkg/engine/plugins_test.go index 01963fb4b8af..c799a6eb7f5d 100644 --- a/pkg/engine/plugins_test.go +++ b/pkg/engine/plugins_test.go @@ -46,7 +46,7 @@ func TestDefaultProvidersSingle(t *testing.T) { PluginDownloadURL: "com.server.url", }) - defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet()) + defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil) assert.NotNil(t, defaultProviders) aws, ok := defaultProviders[tokens.Package("aws")] @@ -78,7 +78,7 @@ func TestDefaultProvidersOverrideNoVersion(t *testing.T) { Kind: workspace.ResourcePlugin, }) - defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet()) + defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil) assert.NotNil(t, defaultProviders) aws, ok := defaultProviders[tokens.Package("aws")] assert.True(t, ok) @@ -107,7 +107,7 @@ func TestDefaultProvidersOverrideNewerVersion(t *testing.T) { Kind: workspace.ResourcePlugin, }) - defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet()) + defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil) assert.NotNil(t, defaultProviders) aws, ok := defaultProviders[tokens.Package("aws")] assert.True(t, ok) @@ -131,7 +131,7 @@ func TestDefaultProvidersSnapshotOverrides(t *testing.T) { Kind: workspace.ResourcePlugin, }) - defaultProviders := computeDefaultProviderPlugins(languagePlugins, snapshotPlugins) + defaultProviders := computeDefaultProviderPlugins(languagePlugins, snapshotPlugins, nil) assert.NotNil(t, defaultProviders) aws, ok := defaultProviders[tokens.Package("aws")] assert.True(t, ok) @@ -209,8 +209,77 @@ func TestDefaultProviderPluginsSorting(t *testing.T) { Kind: workspace.ResourcePlugin, } plugins := newPluginSet(p1, p2) - result := computeDefaultProviderPlugins(plugins, plugins) + result := computeDefaultProviderPlugins(plugins, plugins, nil) assert.Equal(t, map[tokens.Package]workspace.PluginSpec{ "foo": p2, }, result) } + +func TestDefaultVersionsFromProject(t *testing.T) { + t.Parallel() + v1 := semver.MustParse("0.0.1") + v2 := semver.MustParse("0.0.2") + + p1 := workspace.PluginSpec{ + Name: "foo", + Version: &v2, + Kind: workspace.ResourcePlugin, + } + p2 := workspace.PluginSpec{ + Name: "bar", + Version: &v2, + Kind: workspace.ResourcePlugin, + } + p3 := workspace.PluginSpec{ + Name: "frob", + Version: &v2, + Kind: workspace.LanguagePlugin, + } + plugins := newPluginSet(p1, p2, p3) + + path := "./bar" + projectPlugins := []workspace.ProjectPlugin{ + { + Name: "foo", + Kind: workspace.ResourcePlugin, + Version: &v1, + }, + { + Name: "bar", + Kind: workspace.ResourcePlugin, + Version: &v1, + Path: &path, + }, + { + Name: "xyz", + Kind: workspace.ResourcePlugin, + Version: &v1, + }, + { + // This plugin should be ignored because it's not a resource plugin. + Name: "frob", + Kind: workspace.LanguagePlugin, + Version: &v1, + }, + } + result := computeDefaultProviderPlugins(plugins, plugins, projectPlugins) + assert.Equal(t, map[tokens.Package]workspace.PluginSpec{ + // Project plugin versions take precedence over anything the plugin query returned. + "foo": { + Name: "foo", + Kind: workspace.ResourcePlugin, + Version: &v1, + }, + "bar": { + Name: "bar", + Kind: workspace.ResourcePlugin, + Version: &v1, + }, + // If the project defines a plugin that the query didn't return, it's still included in the default providers. + "xyz": { + Name: "xyz", + Kind: workspace.ResourcePlugin, + Version: &v1, + }, + }, result) +} diff --git a/pkg/engine/update.go b/pkg/engine/update.go index 50e679597b35..9fa32c1d7b57 100644 --- a/pkg/engine/update.go +++ b/pkg/engine/update.go @@ -255,6 +255,7 @@ func installPlugins(ctx context.Context, } allPlugins := languagePlugins.Union(snapshotPlugins) + projectPlugins := plugctx.Host.GetProjectPlugins() // If there are any plugins that are not available, we can attempt to install them here. // @@ -262,7 +263,7 @@ func installPlugins(ctx context.Context, // with an error message indicating exactly what plugins are missing. If `returnInstallErrors` is set, then return // the error. if err := ensurePluginsAreInstalled(ctx, plugctx.Diag, allPlugins.Deduplicate(), - plugctx.Host.GetProjectPlugins()); err != nil { + projectPlugins); err != nil { if returnInstallErrors { return nil, nil, err } @@ -270,7 +271,7 @@ func installPlugins(ctx context.Context, } // Collect the version information for default providers. - defaultProviderVersions := computeDefaultProviderPlugins(languagePlugins, allPlugins) + defaultProviderVersions := computeDefaultProviderPlugins(languagePlugins, allPlugins, projectPlugins) return allPlugins, defaultProviderVersions, nil } diff --git a/sdk/go/common/resource/plugin/host.go b/sdk/go/common/resource/plugin/host.go index ce1c59188598..926618e68a2b 100644 --- a/sdk/go/common/resource/plugin/host.go +++ b/sdk/go/common/resource/plugin/host.go @@ -175,29 +175,33 @@ func parsePluginOpts( } var v *semver.Version if providerOpts.Version != "" { - ver, err := semver.Parse(providerOpts.Version) + ver, err := semver.ParseTolerant(providerOpts.Version) if err != nil { return workspace.ProjectPlugin{}, err } v = &ver } - stat, err := os.Stat(providerOpts.Path) - if os.IsNotExist(err) { - return handleErr("no folder at path '%s'", providerOpts.Path) - } else if err != nil { - return handleErr("checking provider folder: %w", err) - } else if !stat.IsDir() { - return handleErr("provider folder '%s' is not a directory", providerOpts.Path) - } + var path *string + if providerOpts.Path != "" { + stat, err := os.Stat(providerOpts.Path) + if os.IsNotExist(err) { + return handleErr("no folder at path '%s'", path) + } else if err != nil { + return handleErr("checking provider folder: %w", err) + } else if !stat.IsDir() { + return handleErr("provider folder '%s' is not a directory", path) + } - // The path is relative to the project root. Make it absolute here so we don't need to track that everywhere its used. - path := providerOpts.Path - if !filepath.IsAbs(path) { - path, err = filepath.Abs(filepath.Join(root, path)) - if err != nil { - return handleErr("getting absolute path for plugin path %s: %w", providerOpts.Path, err) + // The path is relative to the project root. Make it absolute here so we don't need to track that everywhere its used. + p := providerOpts.Path + if !filepath.IsAbs(p) { + p, err = filepath.Abs(filepath.Join(root, p)) + if err != nil { + return handleErr("getting absolute path for plugin path %s: %w", path, err) + } } + path = &p } pluginInfo := workspace.ProjectPlugin{ diff --git a/sdk/go/common/workspace/plugins.go b/sdk/go/common/workspace/plugins.go index ce6b8a9d5978..097b574cfa80 100644 --- a/sdk/go/common/workspace/plugins.go +++ b/sdk/go/common/workspace/plugins.go @@ -706,7 +706,7 @@ type ProjectPlugin struct { Name string // the simple name of the plugin. Kind PluginKind // the kind of the plugin (language, resource, etc). Version *semver.Version // the plugin's semantic version, if present. - Path string // the path that a plugin is to be loaded from (this will always be a directory) + Path *string // the path that a plugin is to be loaded from (this will always be a directory) } // Spec Return a PluginSpec object for this project plugin. @@ -1761,13 +1761,16 @@ func getPluginInfoAndPath( continue } } + if plugin.Path == nil { + continue + } spec := plugin.Spec() info := &PluginInfo{ Name: spec.Name, Kind: spec.Kind, Version: spec.Version, - Path: plugin.Path, + Path: *plugin.Path, } path := getPluginPath(info) // computing plugin sizes can be very expensive (nested node_modules) diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 758e6058b099..61a757ff90dd 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -257,8 +257,7 @@ "type":"object", "additionalProperties":false, "required":[ - "name", - "path" + "name" ], "properties":{ "name":{ diff --git a/tests/smoke_test.go b/tests/smoke_test.go index 325af05352e8..7de1d03b12cc 100644 --- a/tests/smoke_test.go +++ b/tests/smoke_test.go @@ -10,7 +10,9 @@ import ( "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -423,3 +425,41 @@ func TestRelativePluginPath(t *testing.T) { e.RunCommand("pulumi", "install") e.RunCommand("pulumi", "preview") } + +// Small integration test for default provider version overrides. This manually registers a resource without a version +// to check the default provider version is used. +func TestDefaultProviderVersion(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer deleteIfNotFailed(e) + + e.ImportDirectory("testdata/default_plugin_node") + + // Make sure random is installed, this is the version from Pulumi.yaml not package.json + e.RunCommand("pulumi", "plugin", "install", "resource", "random", "4.15.0") + + e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "test") + e.RunCommand("pulumi", "install") + e.RunCommand("pulumi", "up", "--yes") + e.RunCommand("pulumi", "stack", "export", "--file", "export.json") + // Read the export.json and check the provider got the version 4.15 + exportBytes, err := os.ReadFile(filepath.Join(e.CWD, "export.json")) + require.NoError(t, err) + var export apitype.UntypedDeployment + err = json.Unmarshal(exportBytes, &export) + require.NoError(t, err) + + var deployment apitype.DeploymentV3 + err = json.Unmarshal(export.Deployment, &deployment) + require.NoError(t, err) + + resources := deployment.Resources + assert.Len(t, resources, 3) + resource := resources[1] + assert.Equal(t, tokens.Type("pulumi:providers:random"), resource.Type) + assert.Equal(t, map[string]interface{}{ + "version": "4.15.0", + }, resource.Inputs) +} diff --git a/tests/testdata/default_plugin_node/.gitignore b/tests/testdata/default_plugin_node/.gitignore new file mode 100644 index 000000000000..b529cb6b8cbc --- /dev/null +++ b/tests/testdata/default_plugin_node/.gitignore @@ -0,0 +1,4 @@ +/bin/ +/node_modules/ +package-lock.json +Pulumi.test.yaml diff --git a/tests/testdata/default_plugin_node/Pulumi.yaml b/tests/testdata/default_plugin_node/Pulumi.yaml new file mode 100644 index 000000000000..5026c2aa7fb8 --- /dev/null +++ b/tests/testdata/default_plugin_node/Pulumi.yaml @@ -0,0 +1,7 @@ +name: import_node +runtime: nodejs +description: A minimal NodeJS program to test relative provider plugin paths. +plugins: + providers: + - name: random + version: v4.15 # This is a version lower than what package.json specifies \ No newline at end of file diff --git a/tests/testdata/default_plugin_node/index.ts b/tests/testdata/default_plugin_node/index.ts new file mode 100644 index 000000000000..7c19d6765a4f --- /dev/null +++ b/tests/testdata/default_plugin_node/index.ts @@ -0,0 +1,16 @@ +// Copyright 2016-2024, Pulumi Corporation. All rights reserved. + +import * as pulumi from "@pulumi/pulumi"; + +export interface RandomStringArgs { + length: pulumi.Input; +} + +// Register a random resource manually so there's no version set +export class RandomString extends pulumi.CustomResource { + constructor(name: string, args: RandomStringArgs, opts?: pulumi.ResourceOptions) { + super("random:index/randomString:RandomString", name, args, opts); + } + } + +new RandomString("random", { length: 10 }); diff --git a/tests/testdata/default_plugin_node/package.json b/tests/testdata/default_plugin_node/package.json new file mode 100644 index 000000000000..e62b1e5344a6 --- /dev/null +++ b/tests/testdata/default_plugin_node/package.json @@ -0,0 +1,10 @@ +{ + "name": "relative_plugin_node", + "main": "index.ts", + "devDependencies": { + "@types/node": "^16" + }, + "dependencies": { + "@pulumi/random": "4.16" + } +} diff --git a/tests/testdata/default_plugin_node/tsconfig.json b/tests/testdata/default_plugin_node/tsconfig.json new file mode 100644 index 000000000000..ab65afa6135b --- /dev/null +++ b/tests/testdata/default_plugin_node/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "outDir": "bin", + "target": "es2016", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true + }, + "files": [ + "index.ts" + ] +}