From 45ea5afdb18dbd2863415be7f98f1fc57f1e57cf Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Tue, 6 Dec 2022 14:40:37 +0000 Subject: [PATCH] Add Output.JsonSerialize to dotnet sdk Plan is to add functions like this to _all_ the SDKs. JsonSerialization is _very_ language specific, dotnet for example uses System.Text.Json, go would use JsonMarshal, etc. So it's worth having it built into SDKs and then exposed as a PCL intrinsic (with the caveat that the cross-language result will be _valid_ JSON, but with no commmitment to formatting for example). This is just the first part of this work, to add it to the dotnet SDK (simply because I know that best). --- ...-jsonserialize-using-system-text-json.yaml | 4 + sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs | 97 ++++++++++++ sdk/dotnet/Pulumi/Core/Output.cs | 138 +++++++++++++++++- 3 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 changelog/pending/20221206--sdk-dotnet--add-output-jsonserialize-using-system-text-json.yaml diff --git a/changelog/pending/20221206--sdk-dotnet--add-output-jsonserialize-using-system-text-json.yaml b/changelog/pending/20221206--sdk-dotnet--add-output-jsonserialize-using-system-text-json.yaml new file mode 100644 index 000000000000..c4047b5713fb --- /dev/null +++ b/changelog/pending/20221206--sdk-dotnet--add-output-jsonserialize-using-system-text-json.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: sdk/dotnet + description: Add Output.JsonSerialize using System.Text.Json. diff --git a/sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs b/sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs index cba9f7cefe72..fcfaa063b8a8 100644 --- a/sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs +++ b/sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs @@ -9,6 +9,20 @@ namespace Pulumi.Tests.Core { + // Simple struct used for JSON tests + public struct TestStructure { + public int X { get; set;} + + private int y; + + public string Z => (y+1).ToString(); + + public TestStructure(int x, int y) { + X = x; + this.y = y; + } + } + public class OutputTests : PulumiTest { private static Output CreateOutput(T value, bool isKnown, bool isSecret = false) @@ -618,6 +632,89 @@ public Task CreateSecretSetsSecret() Assert.True(data.IsSecret); Assert.Equal(0, data.Value); }); + + [Fact] + public Task JsonSerializeBasic() + => RunInNormal(async () => + { + var o1 = CreateOutput(new int[]{ 0, 1} , true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeNested() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(0, true), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeNestedUnknown() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(default, false), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.False(data.IsKnown); + Assert.False(data.IsSecret); + }); + + [Fact] + public Task JsonSerializeNestedSecret() + => RunInNormal(async () => + { + var o1 = CreateOutput(new Output[] { + CreateOutput(0, true, true), + CreateOutput(1, true), + }, true); + var o2 = Output.JsonSerialize(o1); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.True(data.IsSecret); + Assert.Equal("[0,1]", data.Value); + }); + + [Fact] + public Task JsonSerializeWithOptions() + => RunInNormal(async () => + { + var v = new System.Collections.Generic.Dictionary(); + v.Add("a", new TestStructure(1, 2)); + v.Add("b", new TestStructure(int.MinValue, int.MaxValue)); + var o1 = CreateOutput(v, true); + var options = new System.Text.Json.JsonSerializerOptions(); + options.WriteIndented = true; + var o2 = Output.JsonSerialize(o1, options); + var data = await o2.DataTask.ConfigureAwait(false); + Assert.True(data.IsKnown); + Assert.False(data.IsSecret); + var expected = @"{ + ""a"": { + ""X"": 1, + ""Z"": ""3"" + }, + ""b"": { + ""X"": -2147483648, + ""Z"": ""-2147483648"" + } +}"; + Assert.Equal(expected, data.Value); + }); } } } diff --git a/sdk/dotnet/Pulumi/Core/Output.cs b/sdk/dotnet/Pulumi/Core/Output.cs index 4523a7faa60f..aa5e6ed91490 100644 --- a/sdk/dotnet/Pulumi/Core/Output.cs +++ b/sdk/dotnet/Pulumi/Core/Output.cs @@ -5,11 +5,86 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Pulumi.Serialization; namespace Pulumi { + /// + /// Internal class used for Output.JsonSerialize. + /// + sealed class OutputJsonConverter : System.Text.Json.Serialization.JsonConverterFactory + { + private sealed class OutputJsonConverterInner : System.Text.Json.Serialization.JsonConverter> + { + readonly OutputJsonConverter Parent; + readonly JsonConverter Converter; + + public OutputJsonConverterInner(OutputJsonConverter parent, JsonSerializerOptions options) { + Parent = parent; + Converter = (JsonConverter)options.GetConverter(typeof(T)); + } + + public override Output Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException("JsonSerialize only supports writing to JSON"); + } + + public override void Write(Utf8JsonWriter writer, Output value, JsonSerializerOptions options) + { + // Sadly we have to block here as converters aren't async + var result = value.DataTask.Result; + if (!result.IsKnown) + { + // If the result isn't known we can just write a null and flag the parent to reject this whole serialization + writer.WriteNullValue(); + Parent.SeenUnknown = true; + } + else + { + // The result is known we can just serialize the inner value, but flag the parent if we've seen a secret + Converter.Write(writer, result.Value, options); + Parent.SeenSecret |= result.IsSecret; + } + } + } + + public bool SeenUnknown {get; private set;} + public bool SeenSecret {get; private set;} + public ImmutableHashSet SeenResources => Resources.ToImmutableHashSet(); + private readonly HashSet Resources; + + public OutputJsonConverter() + { + Resources = new HashSet(); + } + + public override bool CanConvert(Type typeToConvert) + { + if (typeToConvert.IsGenericType) + { + var genericType = typeToConvert.GetGenericTypeDefinition(); + return genericType == typeof(Output<>); + } + return false; + } + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + Type elementType = typeToConvert.GetGenericArguments()[0]; + JsonConverter converter = (JsonConverter)Activator.CreateInstance( + typeof(OutputJsonConverterInner<>).MakeGenericType( + new Type[] { elementType }), + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public, + binder: null, + args: new object[] { this, options }, + culture: null)!; + return converter; + } + } + /// /// Useful static utility methods for both creating and working with s. /// @@ -106,6 +181,67 @@ public static Output Format(FormattableString formattableString) internal static Output> Concat(Output> values1, Output> values2) => Tuple(values1, values2).Apply(tuple => tuple.Item1.AddRange(tuple.Item2)); + + /// + /// Uses to serialize the given value into a JSON string. + /// + public static Output JsonSerialize(Output value, System.Text.Json.JsonSerializerOptions? options = null) + { + if (value == null) { + throw new ArgumentNullException("value"); + } + + async Task> GetData() + { + var result = await value.DataTask; + + if (!result.IsKnown) { + return new OutputData(result.Resources, "", false, result.IsSecret); + } + + var utf8 = new System.IO.MemoryStream(); + // This needs to handle nested potentially secret and unknown Output values, we do this by + // hooking options to handle any seen Output values. + + var internalOptions = new System.Text.Json.JsonSerializerOptions(); + internalOptions.AllowTrailingCommas = options?.AllowTrailingCommas ?? internalOptions.AllowTrailingCommas; + if (options != null) + { + foreach(var converter in options.Converters) + { + internalOptions.Converters.Add(converter); + } + } + internalOptions.DefaultBufferSize = options?.DefaultBufferSize ?? internalOptions.DefaultBufferSize; + internalOptions.DictionaryKeyPolicy = options?.DictionaryKeyPolicy ?? internalOptions.DictionaryKeyPolicy; + internalOptions.Encoder = options?.Encoder ?? internalOptions.Encoder; + internalOptions.IgnoreNullValues = options?.IgnoreNullValues ?? internalOptions.IgnoreNullValues; + internalOptions.IgnoreReadOnlyProperties = options?.IgnoreReadOnlyProperties ?? internalOptions.IgnoreReadOnlyProperties; + internalOptions.MaxDepth = options?.MaxDepth ?? internalOptions.MaxDepth; + internalOptions.PropertyNameCaseInsensitive = options?.PropertyNameCaseInsensitive ?? internalOptions.PropertyNameCaseInsensitive; + internalOptions.PropertyNamingPolicy = options?.PropertyNamingPolicy ?? internalOptions.PropertyNamingPolicy; + internalOptions.ReadCommentHandling = options?.ReadCommentHandling ?? internalOptions.ReadCommentHandling; + internalOptions.WriteIndented = options?.WriteIndented ?? internalOptions.WriteIndented; + + // Add the magic converter to allow us to do nested outputs + var outputConverter = new OutputJsonConverter(); + internalOptions.Converters.Add(outputConverter); + + await System.Text.Json.JsonSerializer.SerializeAsync(utf8, result.Value, internalOptions); + + // Check if the result is valid or not, that is if we saw any nulls we can just throw away the json string made and return unknown + if (outputConverter.SeenUnknown) { + return new OutputData(result.Resources.Union(outputConverter.SeenResources), "", false, result.IsSecret | outputConverter.SeenSecret); + } + + var json = System.Text.Encoding.UTF8.GetString(new ReadOnlySpan(utf8.GetBuffer(), 0, (int)utf8.Length)); + + return new OutputData(result.Resources.Union(outputConverter.SeenResources), json, true, result.IsSecret | outputConverter.SeenSecret); + } + + return new Output(GetData()); + } } /// @@ -128,7 +264,7 @@ internal interface IOutput /// s are a key part of how Pulumi tracks dependencies between s. Because the values of outputs are not available until resources are /// created, these are represented using the special s type, which - /// internally represents two things: an eventually available value of the output and + /// internally represents two things: an eventually available value of the output and /// the dependency on the source(s) of the output value. /// In fact, s is quite similar to . /// Additionally, they carry along dependency information.