Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Output.JsonSerialize to dotnet sdk #11556

Merged
merged 1 commit into from Dec 9, 2022
Merged

Add Output.JsonSerialize to dotnet sdk #11556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions changelog/pending/20221206--sdk-dotnet--add-output-jsonserialize-using-system-text-json.yaml
View file
Open in desktop
@@ -0,0 +1,4 @@
changes:
- type: feat
scope: sdk/dotnet
description: Add Output.JsonSerialize using System.Text.Json.
135 changes: 135 additions & 0 deletions sdk/dotnet/Pulumi.Tests/Core/OutputTests.cs
View file
Open in desktop
@@ -1,5 +1,6 @@
// Copyright 2016-2019, Pulumi Corporation

using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
Expand All @@ -9,12 +10,30 @@

namespace Pulumi.Tests.Core
{
// Simple struct used for JSON tests
public struct TestStructure {
public int X { get; set;}

private int y;

public string Z => (y+1).ToString();

public TestStructure(int x, int y) {
X = x;
this.y = y;
}
}

public class OutputTests : PulumiTest
{
private static Output<T> CreateOutput<T>(T value, bool isKnown, bool isSecret = false)
=> new Output<T>(Task.FromResult(OutputData.Create(
ImmutableHashSet<Resource>.Empty, value, isKnown, isSecret)));

private static Output<T> CreateOutput<T>(IEnumerable<Resource> resources, T value, bool isKnown, bool isSecret = false)
=> new Output<T>(Task.FromResult(OutputData.Create(
ImmutableHashSet.CreateRange(resources), value, isKnown, isSecret)));

public class PreviewTests
{
[Fact]
Expand Down Expand Up @@ -618,6 +637,122 @@ public Task CreateSecretSetsSecret()
Assert.True(data.IsSecret);
Assert.Equal(0, data.Value);
});

[Fact]
public Task JsonSerializeBasic()
=> RunInNormal(async () =>
{
var o1 = CreateOutput(new int[]{ 0, 1} , true);
var o2 = Output.JsonSerialize(o1);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.True(data.IsKnown);
Assert.False(data.IsSecret);
Assert.Equal("[0,1]", data.Value);
});

[Fact]
public Task JsonSerializeNested()
=> RunInNormal(async () =>
{
var o1 = CreateOutput(new Output<int>[] {
CreateOutput(0, true),
CreateOutput(1, true),
}, true);
var o2 = Output.JsonSerialize(o1);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.True(data.IsKnown);
Assert.False(data.IsSecret);
Assert.Equal("[0,1]", data.Value);
});

[Fact]
public Task JsonSerializeNestedUnknown()
=> RunInNormal(async () =>
{
var o1 = CreateOutput(new Output<int>[] {
CreateOutput<int>(default, false),
CreateOutput(1, true),
}, true);
var o2 = Output.JsonSerialize(o1);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.False(data.IsKnown);
Assert.False(data.IsSecret);
});

[Fact]
public Task JsonSerializeNestedSecret()
=> RunInNormal(async () =>
{
var o1 = CreateOutput(new Output<int>[] {
CreateOutput(0, true, true),
CreateOutput(1, true),
}, true);
var o2 = Output.JsonSerialize(o1);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.True(data.IsKnown);
Assert.True(data.IsSecret);
Assert.Equal("[0,1]", data.Value);
});

[Fact]
public Task JsonSerializeWithOptions()
=> RunInNormal(async () =>
{
var v = new System.Collections.Generic.Dictionary<string, TestStructure>();
v.Add("a", new TestStructure(1, 2));
v.Add("b", new TestStructure(int.MinValue, int.MaxValue));
var o1 = CreateOutput(v, true);
var options = new System.Text.Json.JsonSerializerOptions();
options.WriteIndented = true;
var o2 = Output.JsonSerialize(o1, options);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.True(data.IsKnown);
Assert.False(data.IsSecret);
var expected = @"{
""a"": {
""X"": 1,
""Z"": ""3""
},
""b"": {
""X"": -2147483648,
""Z"": ""-2147483648""
}
}";
Assert.Equal(expected, data.Value);
});

[Fact]
public async Task JsonSerializeNestedDependencies() {
// We need a custom mock setup for this because new CustomResource will call into the
// deployment to try and register.
var runner = new Moq.Mock<IRunner>(Moq.MockBehavior.Strict);
runner.Setup(r => r.RegisterTask(Moq.It.IsAny<string>(), Moq.It.IsAny<Task>()));

var logger = new Moq.Mock<IEngineLogger>(Moq.MockBehavior.Strict);
logger.Setup(l => l.DebugAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<Resource>(), Moq.It.IsAny<int?>(), Moq.It.IsAny<bool?>())).Returns(Task.CompletedTask);

var mock = new Moq.Mock<IDeploymentInternal>(Moq.MockBehavior.Strict);
mock.Setup(d => d.IsDryRun).Returns(false);
mock.Setup(d => d.Stack).Returns(() => null!);
mock.Setup(d => d.Runner).Returns(runner.Object);
mock.Setup(d => d.Logger).Returns(logger.Object);
mock.Setup(d => d.ReadOrRegisterResource(Moq.It.IsAny<Resource>(), Moq.It.IsAny<bool>(), Moq.It.IsAny<System.Func<string, Resource>>(), Moq.It.IsAny<ResourceArgs>(), Moq.It.IsAny<ResourceOptions>()));

Deployment.Instance = new DeploymentInstance(mock.Object);

var resource = new CustomResource("type", "name", null);

var o1 = CreateOutput(new Output<int>[] {
CreateOutput(new Resource[] { resource}, 0, true, true),
CreateOutput(1, true),
}, true);
var o2 = Output.JsonSerialize(o1);
var data = await o2.DataTask.ConfigureAwait(false);
Assert.True(data.IsKnown);
Assert.True(data.IsSecret);
Assert.Contains(resource, data.Resources);
Assert.Equal("[0,1]", data.Value);
}
}
}
}
143 changes: 142 additions & 1 deletion sdk/dotnet/Pulumi/Core/Output.cs
View file
Open in desktop
Expand Up @@ -5,11 +5,88 @@
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Pulumi.Serialization;

namespace Pulumi
{
/// <summary>
/// Internal class used for Output.JsonSerialize.
/// </summary>
sealed class OutputJsonConverter : System.Text.Json.Serialization.JsonConverterFactory
{
private sealed class OutputJsonConverterInner<T> : System.Text.Json.Serialization.JsonConverter<Output<T>>
{
readonly OutputJsonConverter Parent;
readonly JsonConverter<T> Converter;

public OutputJsonConverterInner(OutputJsonConverter parent, JsonSerializerOptions options) {
Parent = parent;
Converter = (JsonConverter<T>)options.GetConverter(typeof(T));
}

public override Output<T> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
throw new NotImplementedException("JsonSerialize only supports writing to JSON");
}

public override void Write(Utf8JsonWriter writer, Output<T> value, JsonSerializerOptions options)
{
// Sadly we have to block here as converters aren't async
var result = value.DataTask.Result;
// Add the seen dependencies to the resources set
Parent.Resources.AddRange(result.Resources);
if (!result.IsKnown)
{
// If the result isn't known we can just write a null and flag the parent to reject this whole serialization
writer.WriteNullValue();
Parent.SeenUnknown = true;
}
else
{
// The result is known we can just serialize the inner value, but flag the parent if we've seen a secret
Converter.Write(writer, result.Value, options);
Parent.SeenSecret |= result.IsSecret;
}
}
}

public bool SeenUnknown {get; private set;}
public bool SeenSecret {get; private set;}
public ImmutableHashSet<Resource> SeenResources => Resources.ToImmutableHashSet();
private readonly HashSet<Resource> Resources;

public OutputJsonConverter()
{
Resources = new HashSet<Resource>();
}

public override bool CanConvert(Type typeToConvert)
{
if (typeToConvert.IsGenericType)
{
var genericType = typeToConvert.GetGenericTypeDefinition();
return genericType == typeof(Output<>);
}
return false;
}

public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options)
{
Type elementType = typeToConvert.GetGenericArguments()[0];
JsonConverter converter = (JsonConverter)Activator.CreateInstance(
typeof(OutputJsonConverterInner<>).MakeGenericType(
new Type[] { elementType }),
System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public,
binder: null,
args: new object[] { this, options },
culture: null)!;
return converter;
}
}

/// <summary>
/// Useful static utility methods for both creating and working with <see cref="Output{T}"/>s.
/// </summary>
Expand Down Expand Up @@ -106,6 +183,70 @@ public static Output<string> Format(FormattableString formattableString)

internal static Output<ImmutableArray<T>> Concat<T>(Output<ImmutableArray<T>> values1, Output<ImmutableArray<T>> values2)
=> Tuple(values1, values2).Apply(tuple => tuple.Item1.AddRange(tuple.Item2));

/// <summary>
/// Uses <see cref="System.Text.Json.JsonSerializer.SerializeAsync{T}"/> to serialize the given <see
/// cref="Output{T}"/> value into a JSON string.
/// </summary>
public static Output<string> JsonSerialize<T>(Output<T> value, System.Text.Json.JsonSerializerOptions? options = null)
{
if (value == null) {
throw new ArgumentNullException("value");
}

async Task<OutputData<string>> GetData()
{
var result = await value.DataTask;

if (!result.IsKnown) {
return new OutputData<string>(result.Resources, "", false, result.IsSecret);
}

var utf8 = new System.IO.MemoryStream();
// This needs to handle nested potentially secret and unknown Output values, we do this by
// hooking options to handle any seen Output<T> values.

// TODO: This can be simplified in net6.0 to just new System.Text.Json.JsonSerializerOptions(options);
var internalOptions = new System.Text.Json.JsonSerializerOptions();
internalOptions.AllowTrailingCommas = options?.AllowTrailingCommas ?? internalOptions.AllowTrailingCommas;
if (options != null)
{
foreach(var converter in options.Converters)
{
internalOptions.Converters.Add(converter);
}
}
internalOptions.DefaultBufferSize = options?.DefaultBufferSize ?? internalOptions.DefaultBufferSize;
internalOptions.DictionaryKeyPolicy = options?.DictionaryKeyPolicy ?? internalOptions.DictionaryKeyPolicy;
internalOptions.Encoder = options?.Encoder ?? internalOptions.Encoder;
internalOptions.IgnoreNullValues = options?.IgnoreNullValues ?? internalOptions.IgnoreNullValues;
internalOptions.IgnoreReadOnlyProperties = options?.IgnoreReadOnlyProperties ?? internalOptions.IgnoreReadOnlyProperties;
internalOptions.MaxDepth = options?.MaxDepth ?? internalOptions.MaxDepth;
internalOptions.PropertyNameCaseInsensitive = options?.PropertyNameCaseInsensitive ?? internalOptions.PropertyNameCaseInsensitive;
internalOptions.PropertyNamingPolicy = options?.PropertyNamingPolicy ?? internalOptions.PropertyNamingPolicy;
internalOptions.ReadCommentHandling = options?.ReadCommentHandling ?? internalOptions.ReadCommentHandling;
internalOptions.WriteIndented = options?.WriteIndented ?? internalOptions.WriteIndented;

// Add the magic converter to allow us to do nested outputs
var outputConverter = new OutputJsonConverter();
internalOptions.Converters.Add(outputConverter);

await System.Text.Json.JsonSerializer.SerializeAsync<T>(utf8, result.Value, internalOptions);

// Check if the result is valid or not, that is if we saw any nulls we can just throw away the json string made and return unknown
if (outputConverter.SeenUnknown) {
return new OutputData<string>(result.Resources.Union(outputConverter.SeenResources), "", false, result.IsSecret | outputConverter.SeenSecret);
}

// GetBuffer returns the entire byte array backing the MemoryStream, wrapping a span of the
// correct length around that rather than just calling ToArray() saves an array copy.
var json = System.Text.Encoding.UTF8.GetString(new ReadOnlySpan<byte>(utf8.GetBuffer(), 0, (int)utf8.Length));

return new OutputData<string>(result.Resources.Union(outputConverter.SeenResources), json, true, result.IsSecret | outputConverter.SeenSecret);
}

return new Output<string>(GetData());
}
}

/// <summary>
Expand All @@ -128,7 +269,7 @@ internal interface IOutput
/// <see cref="Output{T}"/>s are a key part of how Pulumi tracks dependencies between <see
/// cref="Resource"/>s. Because the values of outputs are not available until resources are
/// created, these are represented using the special <see cref="Output{T}"/>s type, which
/// internally represents two things: an eventually available value of the output and
/// internally represents two things: an eventually available value of the output and
/// the dependency on the source(s) of the output value.
/// In fact, <see cref="Output{T}"/>s is quite similar to <see cref="Task{TResult}"/>.
/// Additionally, they carry along dependency information.
Expand Down