Allow Pulumi.yaml to set default plugin version

changelog/pending/20240317--engine--default-provider-versions-can-be-set-in-pulumi-yaml.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: engine
+  description: Default provider versions can be set in Pulumi.yaml

pkg/engine/plugins.go

@@ -290,11 +290,13 @@ func installPlugin(ctx context.Context, plugin workspace.PluginSpec) error {
 
 // computeDefaultProviderPlugins computes, for every resource plugin, a mapping from packages to semver versions
 // reflecting the version of a provider that should be used as the "default" resource when registering resources. This
-// function takes two sets of plugins: a set of plugins given to us from the language host and the full set of plugins.
-// If the language host has sent us a non-empty set of plugins, we will use those exclusively to service default
-// provider requests. Otherwise, we will use the full set of plugins, which is the existing behavior today.
+// function takes three sets of plugins: a set of plugins given to us from the language host, the full set of plugins
+// from the snapshot, and the set of plugins from the project file. If the language host has sent us a non-empty set of
+// plugins, we will use those exclusively to service default provider requests. Otherwise, we will use the full set of
+// plugins, which is the existing behavior today. Either way anything in the project file is included and takes
+// precedence.
 //
-// The justification for favoring language plugins over all else is that, ultimately, it is the language plugin that
+// The justification for favoring language plugins over the snapshot is that, ultimately, it is the language plugin that
 // produces resource registrations and therefore it is the language plugin that should dictate exactly what plugins to
 // use to satisfy a resource registration. SDKs have the opportunity to specify what plugin (pluginDownloadURL and
 // version) they want to use in RegisterResource. If the plugin is left unspecified, we make a best guess effort to
@@ -305,7 +307,10 @@ func installPlugin(ctx context.Context, plugin workspace.PluginSpec) error {
 // that the engine uses to determine which version of a particular provider to load.
 //
 // it is critical that this function be 100% deterministic.
-func computeDefaultProviderPlugins(languagePlugins, allPlugins pluginSet) map[tokens.Package]workspace.PluginSpec {
+func computeDefaultProviderPlugins(
+	languagePlugins, allPlugins pluginSet,
+	projectPlugins []workspace.ProjectPlugin,
+) map[tokens.Package]workspace.PluginSpec {
 	// Language hosts are not required to specify the full set of plugins they depend on. If the set of plugins received
 	// from the language host does not include any resource providers, fall back to the full set of plugins.
 	languageReportedProviderPlugins := false
@@ -372,6 +377,34 @@ func computeDefaultProviderPlugins(languagePlugins, allPlugins pluginSet) map[to
 		defaultProviderPlugins[tokens.Package(p.Name)] = p
 	}
 
+	// Now take into account the project plugins. These take precedence over anything the plugin queries returned.
+	for _, p := range projectPlugins {
+		if p.Kind != workspace.ResourcePlugin {
+			// Default providers are only relevant for resource plugins.
+			continue
+		}
+
+		spec := workspace.PluginSpec{
+			Name:    p.Name,
+			Kind:    p.Kind,
+			Version: p.Version,
+		}
+
+		pkg := tokens.Package(p.Name)
+
+		if seenPlugin, has := defaultProviderPlugins[pkg]; has {
+			logging.V(preparePluginLog).Infof(
+				"computeDefaultProviderPlugins(): plugin %s selected for package %s (override, previous was %s)",
+				spec, pkg, seenPlugin.Version)
+			defaultProviderPlugins[pkg] = spec
+			continue
+		}
+
+		logging.V(preparePluginLog).Infof(
+			"computeDefaultProviderPlugins(): plugin %s selected for package %s (first seen)", spec, pkg)
+		defaultProviderPlugins[pkg] = spec
+	}
+
 	if logging.V(preparePluginLog) {
 		logging.V(preparePluginLog).Infoln("computeDefaultProviderPlugins(): summary of default plugins:")
 		for pkg, info := range defaultProviderPlugins {

pkg/engine/plugins_test.go

@@ -46,7 +46,7 @@ func TestDefaultProvidersSingle(t *testing.T) {
 		PluginDownloadURL: "com.server.url",
 	})
 
-	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
+	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil)
 	assert.NotNil(t, defaultProviders)
 
 	aws, ok := defaultProviders[tokens.Package("aws")]
@@ -78,7 +78,7 @@ func TestDefaultProvidersOverrideNoVersion(t *testing.T) {
 		Kind:    workspace.ResourcePlugin,
 	})
 
-	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
+	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil)
 	assert.NotNil(t, defaultProviders)
 	aws, ok := defaultProviders[tokens.Package("aws")]
 	assert.True(t, ok)
@@ -107,7 +107,7 @@ func TestDefaultProvidersOverrideNewerVersion(t *testing.T) {
 		Kind:    workspace.ResourcePlugin,
 	})
 
-	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet())
+	defaultProviders := computeDefaultProviderPlugins(languagePlugins, newPluginSet(), nil)
 	assert.NotNil(t, defaultProviders)
 	aws, ok := defaultProviders[tokens.Package("aws")]
 	assert.True(t, ok)
@@ -131,7 +131,7 @@ func TestDefaultProvidersSnapshotOverrides(t *testing.T) {
 		Kind:    workspace.ResourcePlugin,
 	})
 
-	defaultProviders := computeDefaultProviderPlugins(languagePlugins, snapshotPlugins)
+	defaultProviders := computeDefaultProviderPlugins(languagePlugins, snapshotPlugins, nil)
 	assert.NotNil(t, defaultProviders)
 	aws, ok := defaultProviders[tokens.Package("aws")]
 	assert.True(t, ok)
@@ -209,8 +209,77 @@ func TestDefaultProviderPluginsSorting(t *testing.T) {
 		Kind:    workspace.ResourcePlugin,
 	}
 	plugins := newPluginSet(p1, p2)
-	result := computeDefaultProviderPlugins(plugins, plugins)
+	result := computeDefaultProviderPlugins(plugins, plugins, nil)
 	assert.Equal(t, map[tokens.Package]workspace.PluginSpec{
 		"foo": p2,
 	}, result)
 }
+
+func TestDefaultVersionsFromProject(t *testing.T) {
+	t.Parallel()
+	v1 := semver.MustParse("0.0.1")
+	v2 := semver.MustParse("0.0.2")
+
+	p1 := workspace.PluginSpec{
+		Name:    "foo",
+		Version: &v2,
+		Kind:    workspace.ResourcePlugin,
+	}
+	p2 := workspace.PluginSpec{
+		Name:    "bar",
+		Version: &v2,
+		Kind:    workspace.ResourcePlugin,
+	}
+	p3 := workspace.PluginSpec{
+		Name:    "frob",
+		Version: &v2,
+		Kind:    workspace.LanguagePlugin,
+	}
+	plugins := newPluginSet(p1, p2, p3)
+
+	path := "./bar"
+	projectPlugins := []workspace.ProjectPlugin{
+		{
+			Name:    "foo",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+		},
+		{
+			Name:    "bar",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+			Path:    &path,
+		},
+		{
+			Name:    "xyz",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+		},
+		{
+			// This plugin should be ignored because it's not a resource plugin.
+			Name:    "frob",
+			Kind:    workspace.LanguagePlugin,
+			Version: &v1,
+		},
+	}
+	result := computeDefaultProviderPlugins(plugins, plugins, projectPlugins)
+	assert.Equal(t, map[tokens.Package]workspace.PluginSpec{
+		// Project plugin versions take precedence over anything the plugin query returned.
+		"foo": {
+			Name:    "foo",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+		},
+		"bar": {
+			Name:    "bar",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+		},
+		// If the project defines a plugin that the query didn't return, it's still included in the default providers.
+		"xyz": {
+			Name:    "xyz",
+			Kind:    workspace.ResourcePlugin,
+			Version: &v1,
+		},
+	}, result)
+}

pkg/engine/update.go

@@ -255,22 +255,23 @@ func installPlugins(ctx context.Context,
 	}
 
 	allPlugins := languagePlugins.Union(snapshotPlugins)
+	projectPlugins := plugctx.Host.GetProjectPlugins()
 
 	// If there are any plugins that are not available, we can attempt to install them here.
 	//
 	// Note that this is purely a best-effort thing. If we can't install missing plugins, just proceed; we'll fail later
 	// with an error message indicating exactly what plugins are missing. If `returnInstallErrors` is set, then return
 	// the error.
 	if err := ensurePluginsAreInstalled(ctx, plugctx.Diag, allPlugins.Deduplicate(),
-		plugctx.Host.GetProjectPlugins()); err != nil {
+		projectPlugins); err != nil {
 		if returnInstallErrors {
 			return nil, nil, err
 		}
 		logging.V(7).Infof("newUpdateSource(): failed to install missing plugins: %v", err)
 	}
 
 	// Collect the version information for default providers.
-	defaultProviderVersions := computeDefaultProviderPlugins(languagePlugins, allPlugins)
+	defaultProviderVersions := computeDefaultProviderPlugins(languagePlugins, allPlugins, projectPlugins)
 
 	return allPlugins, defaultProviderVersions, nil
 }

sdk/go/common/resource/plugin/host.go

@@ -175,29 +175,33 @@ func parsePluginOpts(
 	}
 	var v *semver.Version
 	if providerOpts.Version != "" {
-		ver, err := semver.Parse(providerOpts.Version)
+		ver, err := semver.ParseTolerant(providerOpts.Version)
 		if err != nil {
 			return workspace.ProjectPlugin{}, err
 		}
 		v = &ver
 	}
 
-	stat, err := os.Stat(providerOpts.Path)
-	if os.IsNotExist(err) {
-		return handleErr("no folder at path '%s'", providerOpts.Path)
-	} else if err != nil {
-		return handleErr("checking provider folder: %w", err)
-	} else if !stat.IsDir() {
-		return handleErr("provider folder '%s' is not a directory", providerOpts.Path)
-	}
+	var path *string
+	if providerOpts.Path != "" {
+		stat, err := os.Stat(providerOpts.Path)
+		if os.IsNotExist(err) {
+			return handleErr("no folder at path '%s'", path)
+		} else if err != nil {
+			return handleErr("checking provider folder: %w", err)
+		} else if !stat.IsDir() {
+			return handleErr("provider folder '%s' is not a directory", path)
+		}
 
-	// The path is relative to the project root. Make it absolute here so we don't need to track that everywhere its used.
-	path := providerOpts.Path
-	if !filepath.IsAbs(path) {
-		path, err = filepath.Abs(filepath.Join(root, path))
-		if err != nil {
-			return handleErr("getting absolute path for plugin path %s: %w", providerOpts.Path, err)
+		// The path is relative to the project root. Make it absolute here so we don't need to track that everywhere its used.
+		p := providerOpts.Path
+		if !filepath.IsAbs(p) {
+			p, err = filepath.Abs(filepath.Join(root, p))
+			if err != nil {
+				return handleErr("getting absolute path for plugin path %s: %w", path, err)
+			}
 		}
+		path = &p
 	}
 
 	pluginInfo := workspace.ProjectPlugin{

sdk/go/common/workspace/plugins.go

@@ -706,7 +706,7 @@ type ProjectPlugin struct {
 	Name    string          // the simple name of the plugin.
 	Kind    PluginKind      // the kind of the plugin (language, resource, etc).
 	Version *semver.Version // the plugin's semantic version, if present.
-	Path    string          // the path that a plugin is to be loaded from (this will always be a directory)
+	Path    *string         // the path that a plugin is to be loaded from (this will always be a directory)
 }
 
 // Spec Return a PluginSpec object for this project plugin.
@@ -1761,13 +1761,16 @@ func getPluginInfoAndPath(
 				continue
 			}
 		}
+		if plugin.Path == nil {
+			continue
+		}
 
 		spec := plugin.Spec()
 		info := &PluginInfo{
 			Name:    spec.Name,
 			Kind:    spec.Kind,
 			Version: spec.Version,
-			Path:    plugin.Path,
+			Path:    *plugin.Path,
 		}
 		path := getPluginPath(info)
 		// computing plugin sizes can be very expensive (nested node_modules)

sdk/go/common/workspace/project.json

@@ -257,8 +257,7 @@
             "type":"object",
             "additionalProperties":false,
             "required":[
-                "name",
-                "path"
+                "name"
             ],
             "properties":{
                 "name":{

tests/smoke_test.go

@@ -10,7 +10,9 @@ import (
 
 	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 
+	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 	ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -423,3 +425,41 @@ func TestRelativePluginPath(t *testing.T) {
 	e.RunCommand("pulumi", "install")
 	e.RunCommand("pulumi", "preview")
 }
+
+// Small integration test for default provider version overrides. This manually registers a resource without a version
+// to check the default provider version is used.
+func TestDefaultProviderVersion(t *testing.T) {
+	t.Parallel()
+
+	e := ptesting.NewEnvironment(t)
+	defer deleteIfNotFailed(e)
+
+	e.ImportDirectory("testdata/default_plugin_node")
+
+	// Make sure random is installed, this is the version from Pulumi.yaml not package.json
+	e.RunCommand("pulumi", "plugin", "install", "resource", "random", "4.15.0")
+
+	e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL())
+	e.RunCommand("pulumi", "stack", "init", "test")
+	e.RunCommand("pulumi", "install")
+	e.RunCommand("pulumi", "up", "--yes")
+	e.RunCommand("pulumi", "stack", "export", "--file", "export.json")
+	// Read the export.json and check the provider got the version 4.15
+	exportBytes, err := os.ReadFile(filepath.Join(e.CWD, "export.json"))
+	require.NoError(t, err)
+	var export apitype.UntypedDeployment
+	err = json.Unmarshal(exportBytes, &export)
+	require.NoError(t, err)
+
+	var deployment apitype.DeploymentV3
+	err = json.Unmarshal(export.Deployment, &deployment)
+	require.NoError(t, err)
+
+	resources := deployment.Resources
+	assert.Len(t, resources, 3)
+	resource := resources[1]
+	assert.Equal(t, tokens.Type("pulumi:providers:random"), resource.Type)
+	assert.Equal(t, map[string]interface{}{
+		"version": "4.15.0",
+	}, resource.Inputs)
+}

tests/testdata/default_plugin_node/.gitignore

@@ -0,0 +1,4 @@
+/bin/
+/node_modules/
+package-lock.json
+Pulumi.test.yaml

tests/testdata/default_plugin_node/Pulumi.yaml

@@ -0,0 +1,7 @@
+name: import_node
+runtime: nodejs
+description: A minimal NodeJS program to test relative provider plugin paths.
+plugins:
+  providers:
+    - name: random
+      version: v4.15 # This is a version lower than what package.json specifies

tests/testdata/default_plugin_node/index.ts

@@ -0,0 +1,16 @@
+// Copyright 2016-2024, Pulumi Corporation.  All rights reserved.
+
+import * as pulumi from "@pulumi/pulumi";
+
+export interface RandomStringArgs {
+    length: pulumi.Input<number>;
+}
+
+// Register a random resource manually so there's no version set
+export class RandomString extends pulumi.CustomResource {
+    constructor(name: string, args: RandomStringArgs, opts?: pulumi.ResourceOptions) {
+      super("random:index/randomString:RandomString", name, args, opts);
+    }
+  }
+
+new RandomString("random", { length: 10 });

tests/testdata/default_plugin_node/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "relative_plugin_node",
+  "main": "index.ts",
+  "devDependencies": {
+    "@types/node": "^16"
+  },
+  "dependencies": {
+    "@pulumi/random": "4.16"
+  }
+}