Merge #11475 #11494

11475: Handle replacements of resources marked for deletion r=Frassle a=Frassle

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

This required getting state into a bit of unusual failure mode, but clearly people are hitting this (see #11391 and #11464).

If we replace a resource with create before delete semantics, and then fail to delete it we end up with two copies of the resource in state, but one copy will have the "delete" flag set.

If we then manage to trigger another resource to replace with delete before replace semantics that has the first resource as a dependent we would hit an assertion about Delete and PendingReplace both being set the same, if you just removed that asset then state management got confused and left the first resource in the state file even after it had really been deleted.

So now we track if we've seen a pendingDelete for a resource via the state pointer not the URN (so we can handle the multiple state objects) and don't trigger a DeleteReplacement, but a plain Delete if the resource is already flagged to Delete.

Fixes #11391

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


11494: Jazzyfresh/codegen python nonstring secrets r=jazzyfresh a=jazzyfresh

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

Fixes incorrectly generated python code when secret is a non-string type.

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Fixes #11278 

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [ ] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Fraser Waters <fraser@pulumi.com>
Co-authored-by: Jasmine Dahilig <jasmine@pulumi.com>

changelog/pending/20221129--engine--fix-an-assert-for-resources-being-replaced-but-also-pending-deletion.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: engine
+  description: Fix an assert for resources being replaced but also pending deletion.

changelog/pending/20221206--pkg--fixes-codegen-python-nonstring-secrets.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: pkg
+  description: Fixes codegen/python generation of non-string secrets in provider properties

pkg/codegen/python/gen.go

@@ -1301,11 +1301,17 @@ func (mod *modContext) genResource(res *schema.Resource) (string, error) {
 		// If this resource is a provider then, regardless of the schema of the underlying provider
 		// type, we must project all properties as strings. For all properties that are not strings,
 		// we'll marshal them to JSON and use the JSON string as a string input.
+		handledSecret := false
 		if res.IsProvider && !isStringType(prop.Type) {
-			arg = fmt.Sprintf("pulumi.Output.from_input(%s).apply(pulumi.runtime.to_json) if %s is not None else None", arg, arg)
+			if prop.Secret {
+				arg = fmt.Sprintf("pulumi.Output.secret(%s).apply(pulumi.runtime.to_json) if %s is not None else None", arg, arg)
+				handledSecret = true
+			} else {
+				arg = fmt.Sprintf("pulumi.Output.from_input(%s).apply(pulumi.runtime.to_json) if %s is not None else None", arg, arg)
+			}
 		}
 		name := PyName(prop.Name)
-		if prop.Secret {
+		if prop.Secret && !handledSecret {
 			fmt.Fprintf(w, "            __props__.__dict__[%[1]q] = None if %[2]s is None else pulumi.Output.secret(%[2]s)\n", name, arg)
 		} else {
 			fmt.Fprintf(w, "            __props__.__dict__[%q] = %s\n", name, arg)

pkg/codegen/testing/test/sdk_driver.go

@@ -151,9 +151,10 @@ var PulumiPulumiSDKTests = []*SDKTest{
 		Description: "Simple schema encoded using YAML",
 	},
 	{
-		Directory:        "provider-config-schema",
-		Description:      "Simple provider config schema",
-		SkipCompileCheck: codegen.NewStringSet(dotnet),
+		Directory:   "provider-config-schema",
+		Description: "Simple provider config schema",
+		// For golang skip check, see https://github.com/pulumi/pulumi/issues/11567
+		SkipCompileCheck: codegen.NewStringSet(dotnet, golang),
 	},
 	{
 		Directory:   "replace-on-change",

pkg/codegen/testing/test/testdata/provider-config-schema/docs/_index.md

@@ -1,6 +1,6 @@
 ---
 title: "configstation"
-title_tag: "configstation.configstation"
+title_tag: "configstation Package"
 meta_desc: ""
 layout: api
 no_edit_this_page: true
@@ -11,6 +11,11 @@ no_edit_this_page: true
 
 
 
+<h2 id="modules">Modules</h2>
+<ul class="api">
+    <li><a href="config/" title="config"><span class="api-symbol api-symbol--module"></span>config</a></li>
+</ul>
+
 <h2 id="resources">Resources</h2>
 <ul class="api">
     <li><a href="provider" title="Provider"><span class="api-symbol api-symbol--resource"></span>Provider</a></li>

pkg/codegen/testing/test/testdata/provider-config-schema/docs/codegen-manifest.json

@@ -1,6 +1,7 @@
 {
   "emittedFiles": [
     "_index.md",
+    "config/_index.md",
     "funcwithalloptionalinputs/_index.md",
     "provider/_index.md"
   ]

pkg/codegen/testing/test/testdata/provider-config-schema/docs/config/_index.md

@@ -0,0 +1,23 @@
+---
+title: "config"
+title_tag: "configstation.config"
+meta_desc: "Explore the resources and functions of the configstation.config module."
+layout: api
+no_edit_this_page: true
+---
+
+<!-- WARNING: this file was generated by test. -->
+<!-- Do not edit by hand unless you're certain you know what you are doing! -->
+
+Explore the resources and functions of the configstation.config module.
+
+<h2 id="package-details">Package Details</h2>
+<dl class="package-details">
+	<dt>Repository</dt>
+	<dd><a href=""></a></dd>
+	<dt>License</dt>
+	<dd></dd>
+	<dt>Version</dt>
+	<dd>0.0.1</dd>
+</dl>
+

pkg/codegen/testing/test/testdata/provider-config-schema/docs/provider/_index.md

@@ -32,7 +32,8 @@ no_edit_this_page: true
 <div class="highlight"><pre class="chroma"><code class="language-python" data-lang="python"><span class=nd>@overload</span>
 <span class="k">def </span><span class="nx">Provider</span><span class="p">(</span><span class="nx">resource_name</span><span class="p">:</span> <span class="nx">str</span><span class="p">,</span>
              <span class="nx">opts</span><span class="p">:</span> <span class="nx"><a href="/docs/reference/pkg/python/pulumi/#pulumi.ResourceOptions">Optional[ResourceOptions]</a></span> = None<span class="p">,</span>
-             <span class="nx">favorite_color</span><span class="p">:</span> <span class="nx">Optional[Union[str, Color]]</span> = None<span class="p">)</span>
+             <span class="nx">favorite_color</span><span class="p">:</span> <span class="nx">Optional[Union[str, Color]]</span> = None<span class="p">,</span>
+             <span class="nx">secret_sandwiches</span><span class="p">:</span> <span class="nx">Optional[Sequence[_config.SandwichArgs]]</span> = None<span class="p">)</span>
 <span class=nd>@overload</span>
 <span class="k">def </span><span class="nx">Provider</span><span class="p">(</span><span class="nx">resource_name</span><span class="p">:</span> <span class="nx">str</span><span class="p">,</span>
              <span class="nx">args</span><span class="p">:</span> <span class="nx"><a href="#inputs">Optional[ProviderArgs]</a></span> = None<span class="p">,</span>
@@ -228,6 +229,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">string | <a href="#color">Configstation.<wbr>Pulumi.<wbr>Configstation.<wbr>Color</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secretsandwiches_csharp">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secretsandwiches_csharp" style="color: inherit; text-decoration: inherit;">Secret<wbr>Sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">List&lt;Configstation.<wbr>Pulumi.<wbr>Configstation.<wbr>Config.<wbr>Inputs.<wbr>Sandwich<wbr>Args&gt;</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -243,6 +253,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">string | <a href="#color">Color</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secretsandwiches_go">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secretsandwiches_go" style="color: inherit; text-decoration: inherit;">Secret<wbr>Sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">Sandwich<wbr>Args</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -258,6 +277,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">String | <a href="#color">Color</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secretsandwiches_java">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secretsandwiches_java" style="color: inherit; text-decoration: inherit;">secret<wbr>Sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">List&lt;Sandwich<wbr>Args&gt;</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -273,6 +301,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">string | <a href="#color">Color</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secretsandwiches_nodejs">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secretsandwiches_nodejs" style="color: inherit; text-decoration: inherit;">secret<wbr>Sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">config<wbr>Sandwich<wbr>Args[]</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -288,6 +325,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">str | <a href="#color">Color</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secret_sandwiches_python">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secret_sandwiches_python" style="color: inherit; text-decoration: inherit;">secret_<wbr>sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">Sandwich<wbr>Args]</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -303,6 +349,15 @@ The Provider resource accepts the following [input](/docs/intro/concepts/inputs-
         <span class="property-type">String | <a href="#color">&#34;blue&#34; | &#34;red&#34;</a></span>
     </dt>
     <dd><p>this is a relaxed string enum which can also be set via env var It can also be sourced from the following environment variable: <code>FAVE_COLOR</code></p>
+</dd><dt class="property-optional"
+            title="Optional">
+        <span id="secretsandwiches_yaml">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#secretsandwiches_yaml" style="color: inherit; text-decoration: inherit;">secret<wbr>Sandwiches</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type"><a href="#sandwich">List&lt;Property Map&gt;</a></span>
+    </dt>
+    <dd><p>Super duper secret sandwiches.</p>
 </dd></dl>
 </pulumi-choosable>
 </div>
@@ -464,6 +519,140 @@ All [input](#inputs) properties are implicitly available as output properties. A
 </pulumi-choosable>
 </div>
 
+<h4 id="sandwich">Sandwich</h4>
+
+<div>
+<pulumi-choosable type="language" values="csharp">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_csharp">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_csharp" style="color: inherit; text-decoration: inherit;">Bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">string</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_csharp">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_csharp" style="color: inherit; text-decoration: inherit;">Veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">List&lt;string&gt;</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
+<div>
+<pulumi-choosable type="language" values="go">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_go">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_go" style="color: inherit; text-decoration: inherit;">Bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">string</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_go">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_go" style="color: inherit; text-decoration: inherit;">Veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">[]string</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
+<div>
+<pulumi-choosable type="language" values="java">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_java">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_java" style="color: inherit; text-decoration: inherit;">bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">String</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_java">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_java" style="color: inherit; text-decoration: inherit;">veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">List&lt;String&gt;</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
+<div>
+<pulumi-choosable type="language" values="javascript,typescript">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_nodejs">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_nodejs" style="color: inherit; text-decoration: inherit;">bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">string</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_nodejs">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_nodejs" style="color: inherit; text-decoration: inherit;">veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">string[]</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
+<div>
+<pulumi-choosable type="language" values="python">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_python">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_python" style="color: inherit; text-decoration: inherit;">bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">str</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_python">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_python" style="color: inherit; text-decoration: inherit;">veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">Sequence[str]</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
+<div>
+<pulumi-choosable type="language" values="yaml">
+<dl class="resources-properties"><dt class="property-optional"
+            title="Optional">
+        <span id="bread_yaml">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#bread_yaml" style="color: inherit; text-decoration: inherit;">bread</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">String</span>
+    </dt>
+    <dd></dd><dt class="property-optional"
+            title="Optional">
+        <span id="veggies_yaml">
+<a data-swiftype-name="resource-property" data-swiftype-type="text" href="#veggies_yaml" style="color: inherit; text-decoration: inherit;">veggies</a>
+</span>
+        <span class="property-indicator"></span>
+        <span class="property-type">List&lt;String&gt;</span>
+    </dt>
+    <dd></dd></dl>
+</pulumi-choosable>
+</div>
+
 
 <h2 id="package-details">Package Details</h2>
 <dl class="package-details">

pkg/codegen/testing/test/testdata/provider-config-schema/dotnet/Configstation/Provider.cs

@@ -46,6 +46,22 @@ public sealed class ProviderArgs : global::Pulumi.ResourceArgs
         [Input("favoriteColor", json: true)]
         public InputUnion<string, Configstation.Pulumi.Configstation.Color>? FavoriteColor { get; set; }
 
+        [Input("secretSandwiches", json: true)]
+        private InputList<Configstation.Pulumi.Configstation.Config.Inputs.SandwichArgs>? _secretSandwiches;
+
+        /// <summary>
+        /// Super duper secret sandwiches.
+        /// </summary>
+        public InputList<Configstation.Pulumi.Configstation.Config.Inputs.SandwichArgs> SecretSandwiches
+        {
+            get => _secretSandwiches ?? (_secretSandwiches = new InputList<Configstation.Pulumi.Configstation.Config.Inputs.SandwichArgs>());
+            set
+            {
+                var emptySecret = Output.CreateSecret(ImmutableArray.Create<Configstation.Pulumi.Configstation.Config.Inputs.SandwichArgs>());
+                _secretSandwiches = Output.All(value, emptySecret).Apply(v => v[0]);
+            }
+        }
+
         public ProviderArgs()
         {
             FavoriteColor = Utilities.GetEnv("FAVE_COLOR");