Merge pulumi/pulumi#11556

11556: Add Output.JsonSerialize to dotnet sdk r=Frassle a=Frassle

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Plan is to add functions like this to _all_ the SDKs. JsonSerialization is _very_ language specific, dotnet for example uses System.Text.Json, go would use JsonMarshal, etc. So it's worth having it built into SDKs and then exposed as a PCL intrinsic (with the caveat that the cross-language result will be _valid_ JSON, but with no commmitment to formatting for example).

This is just the first part of this work, to add it to the dotnet SDK (simply because I know that best).


## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Fraser Waters <fraser@pulumi.com>

sdk/Pulumi.Tests/Core/OutputTests.cs

@@ -1,5 +1,6 @@
 // Copyright 2016-2019, Pulumi Corporation
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
 using System.Threading.Tasks;
@@ -9,12 +10,30 @@
 
 namespace Pulumi.Tests.Core
 {
+    // Simple struct used for JSON tests
+    public struct TestStructure {
+        public int X { get; set;}
+
+        private int y;
+
+        public string Z => (y+1).ToString();
+
+        public TestStructure(int x, int y) {
+            X = x;
+            this.y = y;
+        }
+    }
+
     public class OutputTests : PulumiTest
     {
         private static Output<T> CreateOutput<T>(T value, bool isKnown, bool isSecret = false)
             => new Output<T>(Task.FromResult(OutputData.Create(
                 ImmutableHashSet<Resource>.Empty, value, isKnown, isSecret)));
 
+        private static Output<T> CreateOutput<T>(IEnumerable<Resource> resources, T value, bool isKnown, bool isSecret = false)
+            => new Output<T>(Task.FromResult(OutputData.Create(
+                ImmutableHashSet.CreateRange(resources), value, isKnown, isSecret)));
+
         public class PreviewTests
         {
             [Fact]
@@ -618,6 +637,122 @@ public Task CreateSecretSetsSecret()
                     Assert.True(data.IsSecret);
                     Assert.Equal(0, data.Value);
                 });
+
+            [Fact]
+            public Task JsonSerializeBasic()
+                => RunInNormal(async () =>
+                {
+                    var o1 = CreateOutput(new int[]{ 0, 1} , true);
+                    var o2 = Output.JsonSerialize(o1);
+                    var data = await o2.DataTask.ConfigureAwait(false);
+                    Assert.True(data.IsKnown);
+                    Assert.False(data.IsSecret);
+                    Assert.Equal("[0,1]", data.Value);
+                });
+
+            [Fact]
+            public Task JsonSerializeNested()
+                => RunInNormal(async () =>
+                {
+                    var o1 = CreateOutput(new Output<int>[] {
+                        CreateOutput(0, true),
+                        CreateOutput(1, true),
+                    }, true);
+                    var o2 = Output.JsonSerialize(o1);
+                    var data = await o2.DataTask.ConfigureAwait(false);
+                    Assert.True(data.IsKnown);
+                    Assert.False(data.IsSecret);
+                    Assert.Equal("[0,1]", data.Value);
+                });
+
+            [Fact]
+            public Task JsonSerializeNestedUnknown()
+                => RunInNormal(async () =>
+                {
+                    var o1 = CreateOutput(new Output<int>[] {
+                        CreateOutput<int>(default, false),
+                        CreateOutput(1, true),
+                    }, true);
+                    var o2 = Output.JsonSerialize(o1);
+                    var data = await o2.DataTask.ConfigureAwait(false);
+                    Assert.False(data.IsKnown);
+                    Assert.False(data.IsSecret);
+                });
+
+            [Fact]
+            public Task JsonSerializeNestedSecret()
+                => RunInNormal(async () =>
+                {
+                    var o1 = CreateOutput(new Output<int>[] {
+                        CreateOutput(0, true, true),
+                        CreateOutput(1, true),
+                    }, true);
+                    var o2 = Output.JsonSerialize(o1);
+                    var data = await o2.DataTask.ConfigureAwait(false);
+                    Assert.True(data.IsKnown);
+                    Assert.True(data.IsSecret);
+                    Assert.Equal("[0,1]", data.Value);
+                });
+
+            [Fact]
+            public Task JsonSerializeWithOptions()
+                => RunInNormal(async () =>
+                {
+                    var v = new System.Collections.Generic.Dictionary<string, TestStructure>();
+                    v.Add("a", new TestStructure(1, 2));
+                    v.Add("b", new TestStructure(int.MinValue, int.MaxValue));
+                    var o1 = CreateOutput(v, true);
+                    var options = new System.Text.Json.JsonSerializerOptions();
+                    options.WriteIndented = true;
+                    var o2 = Output.JsonSerialize(o1, options);
+                    var data = await o2.DataTask.ConfigureAwait(false);
+                    Assert.True(data.IsKnown);
+                    Assert.False(data.IsSecret);
+                    var expected = @"{
+  ""a"": {
+    ""X"": 1,
+    ""Z"": ""3""
+  },
+  ""b"": {
+    ""X"": -2147483648,
+    ""Z"": ""-2147483648""
+  }
+}";
+                    Assert.Equal(expected, data.Value);
+                });
+
+            [Fact]
+            public async Task JsonSerializeNestedDependencies() {
+                // We need a custom mock setup for this because new CustomResource will call into the
+                // deployment to try and register.
+                var runner = new Moq.Mock<IRunner>(Moq.MockBehavior.Strict);
+                runner.Setup(r => r.RegisterTask(Moq.It.IsAny<string>(), Moq.It.IsAny<Task>()));
+
+                var logger = new Moq.Mock<IEngineLogger>(Moq.MockBehavior.Strict);
+                logger.Setup(l => l.DebugAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<Resource>(), Moq.It.IsAny<int?>(), Moq.It.IsAny<bool?>())).Returns(Task.CompletedTask);
+
+                var mock = new Moq.Mock<IDeploymentInternal>(Moq.MockBehavior.Strict);
+                mock.Setup(d => d.IsDryRun).Returns(false);
+                mock.Setup(d => d.Stack).Returns(() => null!);
+                mock.Setup(d => d.Runner).Returns(runner.Object);
+                mock.Setup(d => d.Logger).Returns(logger.Object);
+                mock.Setup(d => d.ReadOrRegisterResource(Moq.It.IsAny<Resource>(), Moq.It.IsAny<bool>(), Moq.It.IsAny<System.Func<string, Resource>>(), Moq.It.IsAny<ResourceArgs>(), Moq.It.IsAny<ResourceOptions>()));
+
+                Deployment.Instance = new DeploymentInstance(mock.Object);
+
+                var resource = new CustomResource("type", "name", null);
+
+                var o1 = CreateOutput(new Output<int>[] {
+                    CreateOutput(new Resource[] { resource}, 0, true, true),
+                    CreateOutput(1, true),
+                }, true);
+                var o2 = Output.JsonSerialize(o1);
+                var data = await o2.DataTask.ConfigureAwait(false);
+                Assert.True(data.IsKnown);
+                Assert.True(data.IsSecret);
+                Assert.Contains(resource, data.Resources);
+                Assert.Equal("[0,1]", data.Value);
+            }
         }
     }
 }

sdk/Pulumi/Core/Output.cs

@@ -5,11 +5,88 @@
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using Pulumi.Serialization;
 
 namespace Pulumi
 {
+    /// <summary>
+    /// Internal class used for Output.JsonSerialize.
+    /// </summary>
+    sealed class OutputJsonConverter : System.Text.Json.Serialization.JsonConverterFactory
+    {
+        private sealed class OutputJsonConverterInner<T> : System.Text.Json.Serialization.JsonConverter<Output<T>>
+        {
+            readonly OutputJsonConverter Parent;
+            readonly JsonConverter<T> Converter;
+
+            public OutputJsonConverterInner(OutputJsonConverter parent, JsonSerializerOptions options) {
+                Parent = parent;
+                Converter = (JsonConverter<T>)options.GetConverter(typeof(T));
+            }
+
+            public override Output<T> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+            {
+                throw new NotImplementedException("JsonSerialize only supports writing to JSON");
+            }
+
+            public override void Write(Utf8JsonWriter writer, Output<T> value, JsonSerializerOptions options)
+            {
+                // Sadly we have to block here as converters aren't async
+                var result = value.DataTask.Result;
+                // Add the seen dependencies to the resources set
+                Parent.Resources.AddRange(result.Resources);
+                if (!result.IsKnown)
+                {
+                    // If the result isn't known we can just write a null and flag the parent to reject this whole serialization
+                    writer.WriteNullValue();
+                    Parent.SeenUnknown = true;
+                }
+                else
+                {
+                    // The result is known we can just serialize the inner value, but flag the parent if we've seen a secret
+                    Converter.Write(writer, result.Value, options);
+                    Parent.SeenSecret |= result.IsSecret;
+                }
+            }
+        }
+
+        public bool SeenUnknown {get; private set;}
+        public bool SeenSecret {get; private set;}
+        public ImmutableHashSet<Resource> SeenResources => Resources.ToImmutableHashSet();
+        private readonly HashSet<Resource> Resources;
+
+        public OutputJsonConverter()
+        {
+            Resources = new HashSet<Resource>();
+        }
+
+        public override bool CanConvert(Type typeToConvert)
+        {
+            if (typeToConvert.IsGenericType)
+            {
+                var genericType = typeToConvert.GetGenericTypeDefinition();
+                return genericType == typeof(Output<>);
+            }
+            return false;
+        }
+
+        public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options)
+        {
+            Type elementType = typeToConvert.GetGenericArguments()[0];
+            JsonConverter converter = (JsonConverter)Activator.CreateInstance(
+                typeof(OutputJsonConverterInner<>).MakeGenericType(
+                    new Type[] { elementType }),
+                    System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public,
+                    binder: null,
+                    args: new object[] { this, options },
+                    culture: null)!;
+            return converter;
+        }
+    }
+
     /// <summary>
     /// Useful static utility methods for both creating and working with <see cref="Output{T}"/>s.
     /// </summary>
@@ -106,6 +183,70 @@ public static Output<string> Format(FormattableString formattableString)
 
         internal static Output<ImmutableArray<T>> Concat<T>(Output<ImmutableArray<T>> values1, Output<ImmutableArray<T>> values2)
             => Tuple(values1, values2).Apply(tuple => tuple.Item1.AddRange(tuple.Item2));
+
+        /// <summary>
+        /// Uses <see cref="System.Text.Json.JsonSerializer.SerializeAsync{T}"/> to serialize the given <see
+        /// cref="Output{T}"/> value into a JSON string.
+        /// </summary>
+        public static Output<string> JsonSerialize<T>(Output<T> value, System.Text.Json.JsonSerializerOptions? options = null)
+        {
+            if (value == null) {
+                throw new ArgumentNullException("value");
+            }
+
+            async Task<OutputData<string>> GetData()
+            {
+                var result = await value.DataTask;
+
+                if (!result.IsKnown) {
+                    return new OutputData<string>(result.Resources, "", false, result.IsSecret);
+                }
+
+                var utf8 = new System.IO.MemoryStream();
+                // This needs to handle nested potentially secret and unknown Output values, we do this by
+                // hooking options to handle any seen Output<T> values.
+
+                // TODO: This can be simplified in net6.0 to just new System.Text.Json.JsonSerializerOptions(options);
+                var internalOptions = new System.Text.Json.JsonSerializerOptions();
+                internalOptions.AllowTrailingCommas = options?.AllowTrailingCommas ?? internalOptions.AllowTrailingCommas;
+                if (options != null)
+                {
+                    foreach(var converter in options.Converters)
+                    {
+                        internalOptions.Converters.Add(converter);
+                    }
+                }
+                internalOptions.DefaultBufferSize = options?.DefaultBufferSize ?? internalOptions.DefaultBufferSize;
+                internalOptions.DictionaryKeyPolicy = options?.DictionaryKeyPolicy ?? internalOptions.DictionaryKeyPolicy;
+                internalOptions.Encoder = options?.Encoder ?? internalOptions.Encoder;
+                internalOptions.IgnoreNullValues = options?.IgnoreNullValues ?? internalOptions.IgnoreNullValues;
+                internalOptions.IgnoreReadOnlyProperties = options?.IgnoreReadOnlyProperties ?? internalOptions.IgnoreReadOnlyProperties;
+                internalOptions.MaxDepth = options?.MaxDepth ?? internalOptions.MaxDepth;
+                internalOptions.PropertyNameCaseInsensitive = options?.PropertyNameCaseInsensitive ?? internalOptions.PropertyNameCaseInsensitive;
+                internalOptions.PropertyNamingPolicy = options?.PropertyNamingPolicy ?? internalOptions.PropertyNamingPolicy;
+                internalOptions.ReadCommentHandling = options?.ReadCommentHandling ?? internalOptions.ReadCommentHandling;
+                internalOptions.WriteIndented = options?.WriteIndented ?? internalOptions.WriteIndented;
+
+                // Add the magic converter to allow us to do nested outputs
+                var outputConverter = new OutputJsonConverter();
+                internalOptions.Converters.Add(outputConverter);
+
+                await System.Text.Json.JsonSerializer.SerializeAsync<T>(utf8, result.Value, internalOptions);
+
+                // Check if the result is valid or not, that is if we saw any nulls we can just throw away the json string made and return unknown
+                if (outputConverter.SeenUnknown) {
+                    return new OutputData<string>(result.Resources.Union(outputConverter.SeenResources), "", false, result.IsSecret | outputConverter.SeenSecret);
+                }
+
+                // GetBuffer returns the entire byte array backing the MemoryStream, wrapping a span of the
+                // correct length around that rather than just calling ToArray() saves an array copy.
+                var json = System.Text.Encoding.UTF8.GetString(new ReadOnlySpan<byte>(utf8.GetBuffer(), 0, (int)utf8.Length));
+
+                return new OutputData<string>(result.Resources.Union(outputConverter.SeenResources), json, true, result.IsSecret | outputConverter.SeenSecret);
+            }
+
+            return new Output<string>(GetData());
+        }
     }
 
     /// <summary>
@@ -128,7 +269,7 @@ internal interface IOutput
     /// <see cref="Output{T}"/>s are a key part of how Pulumi tracks dependencies between <see
     /// cref="Resource"/>s. Because the values of outputs are not available until resources are
     /// created, these are represented using the special <see cref="Output{T}"/>s type, which
-    /// internally represents two things: an eventually available value of the output and 
+    /// internally represents two things: an eventually available value of the output and
     /// the dependency on the source(s) of the output value.
     /// In fact, <see cref="Output{T}"/>s is quite similar to <see cref="Task{TResult}"/>.
     /// Additionally, they carry along dependency information.