[program-gen] Emit Output-returning JSON serialization methods withou…

…t rewriting applies (#15371)

### Description

A while ago we started implementing [specialized JSON serialization
methods](#12519) for Pulumi
programs which can accept nested outputs without having to rewrite and
combine applies.
 - `Output.SerializeJson` in .NET
 - `pulumi.jsonStringify` in nodejs
 - `pulumi.Output.json_dumps` in Python

This PR extends program-gen for TypeScript, C# and Python to start
emitting these JSON serialization functions (when necessary). The PR
special-cases the `toJSON` PCL function when rewriting applies so that
nested outputs aren't rewritted.

Example PCL program and generated results:

> Also check out the downstream codegen tests to see improved generated
examples

```
resource vpc "aws:ec2:Vpc" {
	cidrBlock = "10.100.0.0/16"
	instanceTenancy = "default"
}

resource policy "aws:iam/policy:Policy" {
	description = "test"
	policy = toJSON({
		"Version" = "2012-10-17"
		"Interpolated" = "arn:${vpc.arn}:value"
		"Value" = vpc.id
	})
}
```


### Generated TypeScript Before
```typescript
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const vpc = new aws.ec2.Vpc("vpc", {
    cidrBlock: "10.100.0.0/16",
    instanceTenancy: "default",
});
const policy = new aws.iam.Policy("policy", {
    description: "test",
    policy: pulumi.all([vpc.arn, vpc.id]).apply(([arn, id]) => JSON.stringify({
        Version: "2012-10-17",
        Interpolated: `arn:${arn}:value`,
        Value: id,
    })),
});
```

### Generated TypeScript After
```typescript
import * as pulumi from "@pulumi/pulumi";
import * as aws from "@pulumi/aws";

const vpc = new aws.ec2.Vpc("vpc", {
    cidrBlock: "10.100.0.0/16",
    instanceTenancy: "default",
});
const policy = new aws.iam.Policy("policy", {
    description: "test",
    policy: pulumi.jsonStringify({
        Version: "2012-10-17",
        Interpolated: pulumi.interpolate`arn:${vpc.arn}:value`,
        Value: vpc.id,
    }),
});
```
### Generated Python Before
```python
import pulumi
import json
import pulumi_aws as aws

vpc = aws.ec2.Vpc("vpc",
    cidr_block="10.100.0.0/16",
    instance_tenancy="default")
policy = aws.iam.Policy("policy",
    description="test",
    policy=pulumi.Output.all(vpc.arn, vpc.id).apply(lambda arn, id: json.dumps({
        "Version": "2012-10-17",
        "Interpolated": f"arn:{arn}:value",
        "Value": id,
    })))
```

### Generated Python After
```python
import pulumi
import json
import pulumi_aws as aws

vpc = aws.ec2.Vpc("vpc",
    cidr_block="10.100.0.0/16",
    instance_tenancy="default")
policy = aws.iam.Policy("policy",
    description="test",
    policy=pulumi.Output.json_dumps({
        "Version": "2012-10-17",
        "Interpolated": vpc.arn.apply(lambda arn: f"arn:{arn}:value"),
        "Value": vpc.id,
    }))
```

### Generated C# Before
```csharp
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var vpc = new Aws.Ec2.Vpc("vpc", new()
    {
        CidrBlock = "10.100.0.0/16",
        InstanceTenancy = "default",
    });

    var policy = new Aws.Iam.Policy("policy", new()
    {
        Description = "test",
        PolicyDocument = Output.Tuple(vpc.Arn, vpc.Id).Apply(values =>
        {
            var arn = values.Item1;
            var id = values.Item2;
            return JsonSerializer.Serialize(new Dictionary<string, object?>
            {
                ["Version"] = "2012-10-17",
                ["Interpolated"] = $"arn:{arn}:value",
                ["Value"] = id,
            });
        }),
    });

});
```

### Generated C# After
```csharp
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using Pulumi;
using Aws = Pulumi.Aws;

return await Deployment.RunAsync(() => 
{
    var vpc = new Aws.Ec2.Vpc("vpc", new()
    {
        CidrBlock = "10.100.0.0/16",
        InstanceTenancy = "default",
    });

    var policy = new Aws.Iam.Policy("policy", new()
    {
        Description = "test",
        PolicyDocument = Output.JsonSerialize(Output.Create(new Dictionary<string, object?>
        {
            ["Version"] = "2012-10-17",
            ["Interpolated"] = vpc.Arn.Apply(arn => $"arn:{arn}:value"),
            ["Value"] = vpc.Id,
        })),
    });

});
```

## Checklist

- [ ] I have run `make tidy` to update any new dependencies
- [x] I have run `make lint` to verify my code passes the lint check
  - [x] I have formatted my code using `gofumpt`

<!--- Please provide details if the checkbox below is to be left
unchecked. -->
- [x] I have added tests that prove my fix is effective or that my
feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the
`changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the
Pulumi Cloud,
then the service should honor older versions of the CLI where this
change would not exist.
You must then bump the API version in
/pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi
Cloud API version
<!-- @pulumi employees: If yes, you must submit corresponding changes in
the service repo. -->

changelog/pending/20240204--programgen-dotnet-nodejs-python--emit-output-returning-json-serialization-methods-without-rewriting-applies-for-top-level-function-expression.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: programgen/dotnet,nodejs,python
+  description: Emit Output-returning JSON serialization methods without rewriting applies for top-level function expression

pkg/codegen/dotnet/gen_program_expressions.go

@@ -39,7 +39,8 @@ func (g *generator) rewriteExpression(expr model.Expression, typ model.Type, rew
 	expr = pcl.RewritePropertyReferences(expr)
 	var diags hcl.Diagnostics
 	if rewriteApplies {
-		expr, diags = pcl.RewriteApplies(expr, nameInfo(0), !g.asyncInit)
+		skipToJSONWhenRewritingApplies := true
+		expr, diags = pcl.RewriteAppliesWithSkipToJSON(expr, nameInfo(0), !g.asyncInit, skipToJSONWhenRewritingApplies)
 	}
 
 	expr, convertDiags := pcl.RewriteConversions(expr, typ)
@@ -618,9 +619,15 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
 	case "fromBase64":
 		g.Fgenf(w, "System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(%v))", expr.Args[0])
 	case "toJSON":
-		g.Fgen(w, "JsonSerializer.Serialize(")
-		g.genDictionaryOrTuple(w, expr.Args[0])
-		g.Fgen(w, ")")
+		if model.ContainsOutputs(expr.Args[0].Type()) {
+			g.Fgen(w, "Output.JsonSerialize(Output.Create(")
+			g.genDictionaryOrTuple(w, expr.Args[0])
+			g.Fgen(w, "))")
+		} else {
+			g.Fgen(w, "JsonSerializer.Serialize(")
+			g.genDictionaryOrTuple(w, expr.Args[0])
+			g.Fgen(w, ")")
+		}
 	case "sha1":
 		// Assuming the existence of the following helper method located earlier in the preamble
 		g.Fgenf(w, "ComputeSHA1(%v)", expr.Args[0])
@@ -640,7 +647,12 @@ func (g *generator) genDictionaryOrTuple(w io.Writer, expr model.Expression) {
 	case *model.ObjectConsExpression:
 		g.genDictionary(w, expr, "object?")
 	case *model.TupleConsExpression:
-		g.Fgen(w, "new[]\n")
+		if g.isListOfDifferentTypes(expr) {
+			g.Fgen(w, "new object?[]\n")
+		} else {
+			g.Fgen(w, "new[]\n")
+		}
+
 		g.Fgenf(w, "%[1]s{\n", g.Indent)
 		g.Indented(func() {
 			for _, v := range expr.Expressions {
@@ -668,6 +680,35 @@ func (g *generator) genDictionary(w io.Writer, expr *model.ObjectConsExpression,
 	g.Fgenf(w, "%s}", g.Indent)
 }
 
+func (g *generator) isListOfDifferentTypes(expr *model.TupleConsExpression) bool {
+	var prevType model.Type
+	for _, v := range expr.Expressions {
+		if prevType == nil {
+			prevType = v.Type()
+			continue
+		}
+
+		_, isObjectType := prevType.(*model.ObjectType)
+		_, isMap := prevType.(*model.MapType)
+
+		if isObjectType || isMap {
+			// don't actually compare object types or maps because these are always
+			// mapped to Dictionary<string, object?> in C# so they will be the same type
+			// even if their contents are different
+			continue
+		}
+
+		conversionFrom := prevType.ConversionFrom(v.Type())
+		conversionTo := v.Type().ConversionFrom(prevType)
+
+		if conversionTo != model.SafeConversion || conversionFrom != model.SafeConversion {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (g *generator) GenIndexExpression(w io.Writer, expr *model.IndexExpression) {
 	g.Fgenf(w, "%.20v[%.v]", expr.Collection, expr.Key)
 }
@@ -1045,7 +1086,7 @@ func removeDuplicates(inputs []string) []string {
 	return distinctInputs
 }
 
-func (g *generator) isListOfDifferentTypes(expr *model.TupleConsExpression) bool {
+func (g *generator) isListOfDifferentObjectTypes(expr *model.TupleConsExpression) bool {
 	switch expr.Type().(type) {
 	case *model.TupleType:
 		tupleType := expr.Type().(*model.TupleType)
@@ -1070,7 +1111,7 @@ func (g *generator) GenTupleConsExpression(w io.Writer, expr *model.TupleConsExp
 	case 0:
 		g.Fgenf(w, "%s {}", g.listInitializer)
 	default:
-		if !g.isListOfDifferentTypes(expr) {
+		if !g.isListOfDifferentObjectTypes(expr) {
 			// only generate a list initializer when we don't have a list of union types
 			// because list of a union is mapped to InputList<object>
 			// which means new[] will not work because type-inference won't

pkg/codegen/dotnet/test.go

@@ -31,7 +31,7 @@ func Check(t *testing.T, path string, dependencies codegen.StringSet, pulumiSDKP
 		require.NoError(t, err)
 	}
 	err = integration.RunCommand(t, "create dotnet project",
-		[]string{ex, "new", "console"}, dir, &integration.ProgramTestOptions{})
+		[]string{ex, "new", "console", "-f", "net6.0"}, dir, &integration.ProgramTestOptions{})
 	require.NoError(t, err, "Failed to create C# project")
 
 	// Remove Program.cs again generated from "dotnet new console"
@@ -46,6 +46,7 @@ func Check(t *testing.T, path string, dependencies codegen.StringSet, pulumiSDKP
 		for _, pkg := range pkgs {
 			pkg.install(t, ex, dir)
 		}
+		dep{"Pulumi", test.PulumiDotnetSDKVersion}.install(t, ex, dir)
 	} else {
 		// We would like this regardless of other dependencies, but dotnet
 		// packages do not play well with package references.
@@ -55,7 +56,7 @@ func Check(t *testing.T, path string, dependencies codegen.StringSet, pulumiSDKP
 				dir, &integration.ProgramTestOptions{})
 			require.NoError(t, err, "Failed to dotnet sdk package reference")
 		} else {
-			dep{"Pulumi", ""}.install(t, ex, dir)
+			dep{"Pulumi", test.PulumiDotnetSDKVersion}.install(t, ex, dir)
 		}
 	}
 

pkg/codegen/nodejs/gen_program_expressions.go

@@ -31,7 +31,8 @@ func (g *generator) lowerExpression(expr model.Expression, typ model.Type) model
 		expr = g.awaitInvokes(expr)
 	}
 	expr = pcl.RewritePropertyReferences(expr)
-	expr, diags := pcl.RewriteApplies(expr, nameInfo(0), !g.asyncMain)
+	skipToJSONWhenRewritingApplies := true
+	expr, diags := pcl.RewriteAppliesWithSkipToJSON(expr, nameInfo(0), !g.asyncMain, skipToJSONWhenRewritingApplies)
 	if typ != nil {
 		var convertDiags hcl.Diagnostics
 		expr, convertDiags = pcl.RewriteConversions(expr, typ)
@@ -476,7 +477,11 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
 	case "fromBase64":
 		g.Fgenf(w, "Buffer.from(%v, \"base64\").toString(\"utf8\")", expr.Args[0])
 	case "toJSON":
-		g.Fgenf(w, "JSON.stringify(%v)", expr.Args[0])
+		if model.ContainsOutputs(expr.Args[0].Type()) {
+			g.Fgenf(w, "pulumi.jsonStringify(%v)", expr.Args[0])
+		} else {
+			g.Fgenf(w, "JSON.stringify(%v)", expr.Args[0])
+		}
 	case "sha1":
 		g.Fgenf(w, "crypto.createHash('sha1').update(%v).digest('hex')", expr.Args[0])
 	case "stack":

pkg/codegen/pcl/rewrite_apply.go

@@ -34,6 +34,7 @@ type NameInfo interface {
 type applyRewriter struct {
 	nameInfo      NameInfo
 	applyPromises bool
+	skipToJSON    bool
 
 	activeContext applyRewriteContext
 	exprStack     []model.Expression
@@ -154,6 +155,11 @@ func (r *applyRewriter) inspectsEventualValues(x model.Expression) bool {
 	case *model.ForExpression:
 		return r.hasEventualElements(x.Collection)
 	case *model.FunctionCallExpression:
+		// special case toJSON function because we map it to pulumi.jsonStringify which accepts anything
+		// such that it doesn't have to rewrite its subexpressions to apply but can be used directly
+		if r.skipToJSON && x.Name == "toJSON" {
+			return false
+		}
 		_, isEventual := r.isEventualType(x.Signature.ReturnType)
 		if isEventual {
 			return true
@@ -193,6 +199,12 @@ func (r *applyRewriter) observesEventualValues(x model.Expression) bool {
 		_, collectionIsEventual := r.isEventualType(x.Collection.Type())
 		return collectionIsEventual
 	case *model.FunctionCallExpression:
+		// special case toJSON function because we map it to pulumi.jsonStringify which accepts anything
+		// such that it doesn't have to rewrite its subexpressions to apply but can be used directly
+		if r.skipToJSON && x.Name == "toJSON" {
+			return false
+		}
+
 		for i, arg := range x.Args {
 			if !r.isPromptArg(x.Signature.Parameters[i].Type, arg) {
 				return true
@@ -648,9 +660,19 @@ func (ctx *inspectContext) PostVisit(expr model.Expression) (model.Expression, h
 // This form is amenable to code generation for targets that require that outputs are resolved before their values are
 // accessible (e.g. Pulumi's JS/TS libraries).
 func RewriteApplies(expr model.Expression, nameInfo NameInfo, applyPromises bool) (model.Expression, hcl.Diagnostics) {
+	return RewriteAppliesWithSkipToJSON(expr, nameInfo, applyPromises, false)
+}
+
+func RewriteAppliesWithSkipToJSON(
+	expr model.Expression,
+	nameInfo NameInfo,
+	applyPromises bool,
+	skipToJSON bool,
+) (model.Expression, hcl.Diagnostics) {
 	applyRewriter := &applyRewriter{
 		nameInfo:      nameInfo,
 		applyPromises: applyPromises,
+		skipToJSON:    skipToJSON,
 	}
 	applyRewriter.activeContext = &inspectContext{
 		applyRewriter: applyRewriter,

pkg/codegen/pcl/rewrite_apply_test.go

@@ -183,4 +183,35 @@ func TestApplyRewriter(t *testing.T) {
 			assert.Equal(t, c.output, fmt.Sprintf("%v", expr))
 		})
 	}
+
+	t.Run("skip rewriting applies with toJSON", func(t *testing.T) {
+		input := `toJSON({
+	Version = "2012-10-17"
+	Statement = [{
+		Effect = "Allow"
+		Principal = "*"
+		Action = [ "s3:GetObject" ]
+		Resource = [ "arn:aws:s3:::${resource.id}/*" ]
+	}]
+})`
+		expectedOutput := `toJSON({
+	Version = "2012-10-17"
+	Statement = [{
+		Effect = "Allow"
+		Principal = "*"
+		Action = [ "s3:GetObject" ]
+		Resource = [
+                __apply(resource.id,eval(id,  "arn:aws:s3:::${id}/*")) ]
+	}]
+})`
+
+		expr, diags := model.BindExpressionText(input, scope, hcl.Pos{})
+		assert.Len(t, diags, 0)
+
+		expr, diags = RewriteAppliesWithSkipToJSON(expr, nameInfo(0), false, true /* skiToJson */)
+		assert.Len(t, diags, 0)
+
+		output := fmt.Sprintf("%v", expr)
+		assert.Equal(t, expectedOutput, output)
+	})
 }

pkg/codegen/python/gen_program_expressions.go

@@ -28,7 +28,8 @@ func (g *generator) lowerExpression(expr model.Expression, typ model.Type) (mode
 	// TODO(pdg): diagnostics
 
 	expr = pcl.RewritePropertyReferences(expr)
-	expr, diags := pcl.RewriteApplies(expr, nameInfo(0), false)
+	skipToJSONWhenRewritingApplies := true
+	expr, diags := pcl.RewriteAppliesWithSkipToJSON(expr, nameInfo(0), false, skipToJSONWhenRewritingApplies)
 	expr, lowerProxyDiags := g.lowerProxyApplies(expr)
 	expr, convertDiags := pcl.RewriteConversions(expr, typ)
 	expr, quotes, quoteDiags := g.rewriteQuotes(expr)
@@ -404,7 +405,11 @@ func (g *generator) GenFunctionCallExpression(w io.Writer, expr *model.FunctionC
 	case "fromBase64":
 		g.Fgenf(w, "base64.b64decode(%.16v.encode()).decode()", expr.Args[0])
 	case "toJSON":
-		g.Fgenf(w, "json.dumps(%.v)", expr.Args[0])
+		if model.ContainsOutputs(expr.Args[0].Type()) {
+			g.Fgenf(w, "pulumi.Output.json_dumps(%.v)", expr.Args[0])
+		} else {
+			g.Fgenf(w, "json.dumps(%.v)", expr.Args[0])
+		}
 	case "sha1":
 		g.Fgenf(w, "hashlib.sha1(%v.encode()).hexdigest()", expr.Args[0])
 	case "project":

pkg/codegen/testing/test/helpers.go

@@ -374,3 +374,6 @@ const (
 	RandomSchema      SchemaVersion = "4.11.2"
 	EksSchema         SchemaVersion = "0.37.1"
 )
+
+// PulumiDotnetSDKVersion is the version of the Pulumi .NET SDK to use in program-gen tests
+const PulumiDotnetSDKVersion = "3.59.0"

pkg/codegen/testing/test/testdata/aws-eks-pp/dotnet/aws-eks.cs

@@ -228,62 +228,56 @@
     return new Dictionary<string, object?>
     {
         ["clusterName"] = eksCluster.Name,
-        ["kubeconfig"] = Output.Tuple(eksCluster.Endpoint, eksCluster.CertificateAuthority, eksCluster.Name).Apply(values =>
+        ["kubeconfig"] = Output.JsonSerialize(Output.Create(new Dictionary<string, object?>
         {
-            var endpoint = values.Item1;
-            var certificateAuthority = values.Item2;
-            var name = values.Item3;
-            return JsonSerializer.Serialize(new Dictionary<string, object?>
+            ["apiVersion"] = "v1",
+            ["clusters"] = new[]
             {
-                ["apiVersion"] = "v1",
-                ["clusters"] = new[]
+                new Dictionary<string, object?>
                 {
-                    new Dictionary<string, object?>
+                    ["cluster"] = new Dictionary<string, object?>
                     {
-                        ["cluster"] = new Dictionary<string, object?>
-                        {
-                            ["server"] = endpoint,
-                            ["certificate-authority-data"] = certificateAuthority.Data,
-                        },
-                        ["name"] = "kubernetes",
+                        ["server"] = eksCluster.Endpoint,
+                        ["certificate-authority-data"] = eksCluster.CertificateAuthority.Apply(certificateAuthority => certificateAuthority.Data),
                     },
+                    ["name"] = "kubernetes",
                 },
-                ["contexts"] = new[]
+            },
+            ["contexts"] = new[]
+            {
+                new Dictionary<string, object?>
                 {
-                    new Dictionary<string, object?>
+                    ["contest"] = new Dictionary<string, object?>
                     {
-                        ["contest"] = new Dictionary<string, object?>
-                        {
-                            ["cluster"] = "kubernetes",
-                            ["user"] = "aws",
-                        },
+                        ["cluster"] = "kubernetes",
+                        ["user"] = "aws",
                     },
                 },
-                ["current-context"] = "aws",
-                ["kind"] = "Config",
-                ["users"] = new[]
+            },
+            ["current-context"] = "aws",
+            ["kind"] = "Config",
+            ["users"] = new[]
+            {
+                new Dictionary<string, object?>
                 {
-                    new Dictionary<string, object?>
+                    ["name"] = "aws",
+                    ["user"] = new Dictionary<string, object?>
                     {
-                        ["name"] = "aws",
-                        ["user"] = new Dictionary<string, object?>
+                        ["exec"] = new Dictionary<string, object?>
                         {
-                            ["exec"] = new Dictionary<string, object?>
-                            {
-                                ["apiVersion"] = "client.authentication.k8s.io/v1alpha1",
-                                ["command"] = "aws-iam-authenticator",
-                            },
-                            ["args"] = new[]
-                            {
-                                "token",
-                                "-i",
-                                name,
-                            },
+                            ["apiVersion"] = "client.authentication.k8s.io/v1alpha1",
+                            ["command"] = "aws-iam-authenticator",
+                        },
+                        ["args"] = new object?[]
+                        {
+                            "token",
+                            "-i",
+                            eksCluster.Name,
                         },
                     },
                 },
-            });
-        }),
+            },
+        })),
     };
 });
 

pkg/codegen/testing/test/testdata/aws-eks-pp/nodejs/aws-eks.ts

@@ -146,12 +146,12 @@ export = async () => {
     });
     return {
         clusterName: eksCluster.name,
-        kubeconfig: pulumi.all([eksCluster.endpoint, eksCluster.certificateAuthority, eksCluster.name]).apply(([endpoint, certificateAuthority, name]) => JSON.stringify({
+        kubeconfig: pulumi.jsonStringify({
             apiVersion: "v1",
             clusters: [{
                 cluster: {
-                    server: endpoint,
-                    "certificate-authority-data": certificateAuthority.data,
+                    server: eksCluster.endpoint,
+                    "certificate-authority-data": eksCluster.certificateAuthority.apply(certificateAuthority => certificateAuthority.data),
                 },
                 name: "kubernetes",
             }],
@@ -173,10 +173,10 @@ export = async () => {
                     args: [
                         "token",
                         "-i",
-                        name,
+                        eksCluster.name,
                     ],
                 },
             }],
-        })),
+        }),
     };
 }