Merge #11184 #11226

11184: Decouple the generation of plans from the automatic use of plans r=Frassle a=Frassle

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

This threads a new option "Experimental" through to the engine which is used to decided if plans should be used or not. This also generates plans for any preview with --save-plan, but also now generates plans during the preview phase of up.


11226: Handle None being passed to register_resource_outputs r=Frassle a=Frassle

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Fixes #11227.

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [ ] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Fraser Waters <fraser@pulumi.com>

changelog/pending/20221102--sdk-python--handle-none-being-passed-to-register_resource_outputs.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: sdk/python
+  description: Handle None being passed to register_resource_outputs.

pkg/backend/apply.go

@@ -220,12 +220,14 @@ func PreviewThenPromptThenExecute(ctx context.Context, kind apitype.UpdateKind,
 			return changes, res
 		}
 
-		// If we had an original plan use it, else use the newly generated plan (might be nil if we've turned
+		// If we had an original plan use it, else if experimental use the newly generated plan (might be nil if we've turned
 		// plan generation off)
 		if originalPlan != nil {
 			op.Opts.Engine.Plan = originalPlan
-		} else {
+		} else if op.Opts.Engine.Experimental {
 			op.Opts.Engine.Plan = plan
+		} else {
+			op.Opts.Engine.Plan = nil
 		}
 	}
 
@@ -235,6 +237,9 @@ func PreviewThenPromptThenExecute(ctx context.Context, kind apitype.UpdateKind,
 		DryRun:   false,
 		ShowLink: true,
 	}
+	// No need to generate a plan at this stage, there's no way for the system or user to extract the plan
+	// after here.
+	op.Opts.Engine.GeneratePlan = false
 	_, changes, res := apply(ctx, kind, stack, op, opts, nil /*events*/)
 	return changes, res
 }

pkg/backend/backend.go

@@ -258,8 +258,6 @@ type UpdateOptions struct {
 	AutoApprove bool
 	// SkipPreview, when true, causes the preview step to be skipped.
 	SkipPreview bool
-	// GeneratePlan when true cause plans to be generated.
-	GeneratePlan bool
 }
 
 // QueryOptions configures a query to operate against a backend and the engine.

pkg/cmd/pulumi/destroy.go

@@ -246,6 +246,7 @@ func newDestroyCmd() *cobra.Command {
 				DisableProviderPreview:    disableProviderPreview(),
 				DisableResourceReferences: disableResourceReferences(),
 				DisableOutputValues:       disableOutputValues(),
+				Experimental:              hasExperimentalCommands(),
 			}
 
 			_, res := s.Destroy(ctx, backend.UpdateOperation{

pkg/cmd/pulumi/import.go

@@ -513,6 +513,7 @@ func newImportCmd() *cobra.Command {
 				Parallel:      parallel,
 				Debug:         debug,
 				UseLegacyDiff: useLegacyDiff(),
+				Experimental:  hasExperimentalCommands(),
 			}
 
 			_, res := s.Import(ctx, backend.UpdateOperation{

pkg/cmd/pulumi/preview.go

@@ -231,6 +231,7 @@ func newPreviewCmd() *cobra.Command {
 					// If we're trying to save a plan then we _need_ to generate it. We also turn this on in
 					// experimental mode to just get more testing of it.
 					GeneratePlan: hasExperimentalCommands() || planFilePath != "",
+					Experimental: hasExperimentalCommands(),
 				},
 				Display: displayOpts,
 			}

pkg/cmd/pulumi/query.go

@@ -66,7 +66,9 @@ func newQueryCmd() *cobra.Command {
 				return result.FromError(err)
 			}
 
-			opts.Engine = engine.UpdateOptions{}
+			opts.Engine = engine.UpdateOptions{
+				Experimental: hasExperimentalCommands(),
+			}
 
 			res := b.Query(ctx, backend.QueryOperation{
 				Proj:   proj,

pkg/cmd/pulumi/refresh.go

@@ -257,6 +257,7 @@ func newRefreshCmd() *cobra.Command {
 				DisableResourceReferences: disableResourceReferences(),
 				DisableOutputValues:       disableOutputValues(),
 				RefreshTargets:            deploy.NewUrnTargets(targetUrns),
+				Experimental:              hasExperimentalCommands(),
 			}
 
 			changes, res := s.Refresh(ctx, backend.UpdateOperation{

pkg/cmd/pulumi/up.go

@@ -157,9 +157,10 @@ func newUpCmd() *cobra.Command {
 			DisableOutputValues:       disableOutputValues(),
 			UpdateTargets:             deploy.NewUrnTargets(targetURNs),
 			TargetDependents:          targetDependents,
-			// If we're in experimental mode then we trigger a plan to be generated during the preview phase
-			// which will be constrained to during the update phase.
-			GeneratePlan: hasExperimentalCommands(),
+			// Trigger a plan to be generated during the preview phase which can be constrained to during the
+			// update phase.
+			GeneratePlan: true,
+			Experimental: hasExperimentalCommands(),
 		}
 
 		if planFilePath != "" {
@@ -362,6 +363,7 @@ func newUpCmd() *cobra.Command {
 			// If we're in experimental mode then we trigger a plan to be generated during the preview phase
 			// which will be constrained to during the update phase.
 			GeneratePlan: hasExperimentalCommands(),
+			Experimental: hasExperimentalCommands(),
 		}
 
 		// TODO for the URL case:

pkg/cmd/pulumi/watch.go

@@ -137,6 +137,7 @@ func newWatchCmd() *cobra.Command {
 				DisableProviderPreview:    disableProviderPreview(),
 				DisableResourceReferences: disableResourceReferences(),
 				DisableOutputValues:       disableOutputValues(),
+				Experimental:              hasExperimentalCommands(),
 			}
 
 			res := s.Watch(ctx, backend.UpdateOperation{

pkg/engine/lifecycletest/update_plan_test.go

@@ -62,7 +62,7 @@ func TestPlannedUpdate(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -174,7 +174,7 @@ func TestUnplannedCreate(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -241,7 +241,7 @@ func TestUnplannedDelete(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -313,7 +313,7 @@ func TestExpectedDelete(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -382,7 +382,7 @@ func TestExpectedCreate(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -444,7 +444,7 @@ func TestPropertySetChange(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -495,7 +495,7 @@ func TestExpectedUnneededCreate(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -560,7 +560,7 @@ func TestExpectedUnneededDelete(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -637,7 +637,7 @@ func TestResoucesWithSames(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -728,7 +728,7 @@ func TestPlannedPreviews(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -813,7 +813,7 @@ func TestPlannedUpdateChangedStack(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -896,7 +896,7 @@ func TestPlannedOutputChanges(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -963,7 +963,7 @@ func TestPlannedInputOutputDifferences(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1047,7 +1047,7 @@ func TestAliasWithPlans(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1112,7 +1112,7 @@ func TestComputedCanBeDropped(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1264,7 +1264,7 @@ func TestPlannedUpdateWithNondeterministicCheck(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1338,7 +1338,7 @@ func TestPlannedUpdateWithCheckFailure(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1398,7 +1398,7 @@ func TestPluginsAreDownloaded(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()
@@ -1475,7 +1475,7 @@ func TestProviderDeterministicPreview(t *testing.T) {
 	host := deploytest.NewPluginHost(nil, nil, program, loaders...)
 
 	p := &TestPlan{
-		Options: UpdateOptions{Host: host, GeneratePlan: true},
+		Options: UpdateOptions{Host: host, GeneratePlan: true, Experimental: true},
 	}
 
 	project := p.GetProject()

pkg/engine/update.go

@@ -149,8 +149,11 @@ type UpdateOptions struct {
 	// The plan to use for the update, if any.
 	Plan *deploy.Plan
 
-	// true if plans should be generated.
+	// GeneratePlan when true cause plans to be generated, we skip this if we know their not needed (e.g. during up)
 	GeneratePlan bool
+
+	// Experimental is true if the engine is in experimental mode (i.e. PULUMI_EXPERIMENTAL was set)
+	Experimental bool
 }
 
 // HasChanges returns true if there are any non-same changes in the resulting summary.

sdk/python/lib/pulumi/runtime/resource.py

@@ -682,7 +682,9 @@ def register_resource_outputs(
 ):
     async def do_register_resource_outputs():
         urn = await res.urn.future()
-        serialized_props = await rpc.serialize_properties(outputs, {})
+        # serialize_properties expects a collection (empty is fine) but not None, but this is called pretty
+        # much directly by users who could pass None in (although the type hints say they shouldn't).
+        serialized_props = await rpc.serialize_properties(outputs or {}, {})
         log.debug(
             f"register resource outputs prepared: urn={urn}, props={serialized_props}"
         )