Merge #11194

11194: [auto/dotnet] Support for remote operations r=Frassle a=justinvp

This change adds preview support for remote operations in .NET's Automation API.

**Note:** ~~I've tried _many_ incantations of `dotnet format dotnet.sln analyzers --diagnostics=RS0016`, but have not been able to get the code fix applied to update `sdk/dotnet/Pulumi.Automation/PublicAPI.Shipped.txt`. Perhaps a .NET 6 bug (I'm running on an M1 Mac)? dotnet/format#1416 might be relevant. I'll try again a bit later, but I might have to manually update the file, unless someone else's tools are working and can pull down the branch and run it.~~

Update: I didn't see any failures in the PR related to not having updated `sdk/dotnet/Pulumi.Automation/PublicAPI.Shipped.txt`, so maybe we're no longer checking this or something is broken? In any case, I still wasn't able to get `dotnet format` to automatically apply the updates, so I manually edited `PublicAPI.Shipped.txt`. I _think_ I got everything, but it's possible I missed something. Let's see what CI says.

Here's an example of using it:

```c#
using System;
using System.Linq;
using Pulumi.Automation;

bool destroy = args.Any() && args[0] == "destroy";

const string org = "justinvp";
const string projectName = "aws-ts-s3-folder";
var stackName = $"{org}/{projectName}/devdotnet";

var stackArgs = new RemoteGitProgramArgs(stackName, "https://github.com/pulumi/examples.git")
{
    Branch = "refs/heads/master",
    ProjectPath = projectName,
    EnvironmentVariables =
   {
      { "AWS_REGION", new EnvironmentVariableValue("us-west-2") },
      { "AWS_ACCESS_KEY_ID", RequireFromEnvironment("AWS_ACCESS_KEY_ID") },
      { "AWS_SECRET_ACCESS_KEY", RequireFromEnvironment("AWS_SECRET_ACCESS_KEY", isSecret: true) },
      { "AWS_SESSION_TOKEN", RequireFromEnvironment("AWS_SESSION_TOKEN", isSecret: true) },
   },
};
var stack = await RemoteWorkspace.CreateOrSelectStackAsync(stackArgs);

if (destroy)
{
    await stack.DestroyAsync(new RemoteDestroyOptions { OnStandardOutput = Console.WriteLine });
}
else
{
    Console.WriteLine("updating stack...");
    var result = await stack.UpAsync(new RemoteUpOptions { OnStandardOutput = Console.WriteLine });

    if (result.Summary.ResourceChanges != null)
    {
        Console.WriteLine("update summary:");
        foreach (var change in result.Summary.ResourceChanges)
            Console.WriteLine($"    {change.Key}: {change.Value}");
    }

    Console.WriteLine($"url: {result.Outputs["websiteUrl"].Value}");
}

static EnvironmentVariableValue RequireFromEnvironment(string variable, bool isSecret = false)
{
    var value = Environment.GetEnvironmentVariable(variable)
       ?? throw new InvalidOperationException($"Required environment variable {variable} not set.");
    return new EnvironmentVariableValue(value, isSecret);
}
```

I will add sanity tests subsequently.

Co-authored-by: Justin Van Patten <jvp@justinvp.com>

changelog/pending/20221031--auto-dotnet--support-for-remote-operations.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: auto/dotnet
+  description: Support for remote operations

sdk/dotnet/Pulumi.Automation.Tests/RemoteWorkspaceTests.cs

@@ -0,0 +1,24 @@
+// Copyright 2016-2022, Pulumi Corporation
+
+using Xunit;
+
+namespace Pulumi.Automation.Tests
+{
+    public class RemoteWorkspaceTests
+    {
+        [Theory]
+        [InlineData("owner/project/stack", true)]
+        [InlineData("", false)]
+        [InlineData("name", false)]
+        [InlineData("owner/name", false)]
+        [InlineData("/", false)]
+        [InlineData("//", false)]
+        [InlineData("///", false)]
+        [InlineData("owner/project/stack/wat", false)]
+        public void IsFullyQualifiedStackName(string input, bool expected)
+        {
+            var actual = RemoteWorkspace.IsFullyQualifiedStackName(input);
+            Assert.Equal(expected, actual);
+        }
+    }
+}

sdk/dotnet/Pulumi.Automation/EnvironmentVariableValue.cs

@@ -0,0 +1,19 @@
+// Copyright 2016-2022, Pulumi Corporation
+
+namespace Pulumi.Automation
+{
+    public class EnvironmentVariableValue
+    {
+        public string Value { get; set; }
+
+        public bool IsSecret { get; set; }
+
+        public EnvironmentVariableValue(
+            string value,
+            bool isSecret = false)
+        {
+            Value = value;
+            IsSecret = isSecret;
+        }
+    }
+}

sdk/dotnet/Pulumi.Automation/LocalWorkspace.cs

@@ -1,4 +1,4 @@
-// Copyright 2016-2021, Pulumi Corporation
+// Copyright 2016-2022, Pulumi Corporation
 
 using System;
 using System.Collections.Generic;
@@ -33,10 +33,15 @@ namespace Pulumi.Automation
     /// </summary>
     public sealed class LocalWorkspace : Workspace
     {
+        private static readonly SemVersion _minimumVersion = new SemVersion(3, 1, 0);
+
         private readonly LocalSerializer _serializer = new LocalSerializer();
         private readonly bool _ownsWorkingDir;
-        private readonly Task _readyTask;
-        private static readonly SemVersion _minimumVersion = new SemVersion(3, 1, 0);
+        private readonly RemoteGitProgramArgs? _remoteGitProgramArgs;
+        private readonly IDictionary<string, EnvironmentVariableValue>? _remoteEnvironmentVariables;
+        private readonly IList<string>? _remotePreRunCommands;
+
+        internal Task ReadyTask { get; }
 
         /// <inheritdoc/>
         public override string WorkDir { get; }
@@ -60,6 +65,11 @@ public sealed class LocalWorkspace : Workspace
         /// <inheritdoc/>
         public override IDictionary<string, string?>? EnvironmentVariables { get; set; }
 
+        /// <summary>
+        /// Whether this workspace is a remote workspace.
+        /// </summary>
+        internal bool Remote { get; }
+
         /// <summary>
         /// Creates a workspace using the specified options. Used for maximal control and
         /// customization of the underlying environment before any stacks are created or selected.
@@ -74,7 +84,7 @@ public sealed class LocalWorkspace : Workspace
                 new LocalPulumiCmd(),
                 options,
                 cancellationToken);
-            await ws._readyTask.ConfigureAwait(false);
+            await ws.ReadyTask.ConfigureAwait(false);
             return ws;
         }
 
@@ -278,7 +288,7 @@ public static Task<WorkspaceStack> CreateOrSelectStackAsync(LocalProgramArgs arg
                 new LocalPulumiCmd(),
                 args,
                 cancellationToken);
-            await ws._readyTask.ConfigureAwait(false);
+            await ws.ReadyTask.ConfigureAwait(false);
 
             return await initFunc(args.StackName, ws, cancellationToken).ConfigureAwait(false);
         }
@@ -292,7 +302,7 @@ public static Task<WorkspaceStack> CreateOrSelectStackAsync(LocalProgramArgs arg
                 new LocalPulumiCmd(),
                 args,
                 cancellationToken);
-            await ws._readyTask.ConfigureAwait(false);
+            await ws.ReadyTask.ConfigureAwait(false);
 
             return await initFunc(args.StackName, ws, cancellationToken).ConfigureAwait(false);
         }
@@ -315,9 +325,20 @@ public static Task<WorkspaceStack> CreateOrSelectStackAsync(LocalProgramArgs arg
                 this.Program = options.Program;
                 this.Logger = options.Logger;
                 this.SecretsProvider = options.SecretsProvider;
+                this.Remote = options.Remote;
+                this._remoteGitProgramArgs = options.RemoteGitProgramArgs;
 
                 if (options.EnvironmentVariables != null)
                     this.EnvironmentVariables = new Dictionary<string, string?>(options.EnvironmentVariables);
+
+                if (options.RemoteEnvironmentVariables != null)
+                    this._remoteEnvironmentVariables =
+                        new Dictionary<string, EnvironmentVariableValue>(options.RemoteEnvironmentVariables);
+
+                if (options.RemotePreRunCommands != null)
+                {
+                    this._remotePreRunCommands = new List<string>(options.RemotePreRunCommands);
+                }
             }
 
             if (string.IsNullOrWhiteSpace(dir))
@@ -346,7 +367,7 @@ public static Task<WorkspaceStack> CreateOrSelectStackAsync(LocalProgramArgs arg
                     readyTasks.Add(this.SaveStackSettingsAsync(pair.Key, pair.Value, cancellationToken));
             }
 
-            this._readyTask = Task.WhenAll(readyTasks);
+            ReadyTask = Task.WhenAll(readyTasks);
         }
 
         private async Task InitializeProjectSettingsAsync(ProjectSettings projectSettings,
@@ -380,6 +401,18 @@ private async Task PopulatePulumiVersionAsync(CancellationToken cancellationToke
             var hasSkipEnvVar = this.EnvironmentVariables?.ContainsKey(SkipVersionCheckVar) ?? false;
             var optOut = hasSkipEnvVar || Environment.GetEnvironmentVariable(SkipVersionCheckVar) != null;
             this._pulumiVersion = ParseAndValidatePulumiVersion(_minimumVersion, versionString, optOut);
+
+            // If remote was specified, ensure the CLI supports it.
+            if (!optOut && Remote)
+            {
+                // See if `--remote` is present in `pulumi preview --help`'s output.
+                var args = new[] { "preview", "--help" };
+                var previewResult = await RunCommandAsync(args, cancellationToken).ConfigureAwait(false);
+                if (!previewResult.StandardOutput.Contains("--remote"))
+                {
+                    throw new InvalidOperationException("The Pulumi CLI does not support remote operations. Please update the Pulumi CLI.");
+                }
+            }
         }
 
         internal static SemVersion? ParseAndValidatePulumiVersion(SemVersion minVersion, string currentVersion, bool optOut)
@@ -582,12 +615,30 @@ public override Task CreateStackAsync(string stackName, CancellationToken cancel
             if (!string.IsNullOrWhiteSpace(this.SecretsProvider))
                 args.AddRange(new[] { "--secrets-provider", this.SecretsProvider });
 
+            if (Remote)
+                args.Add("--no-select");
+
             return this.RunCommandAsync(args, cancellationToken);
         }
 
         /// <inheritdoc/>
         public override Task SelectStackAsync(string stackName, CancellationToken cancellationToken)
-            => this.RunCommandAsync(new[] { "stack", "select", stackName }, cancellationToken);
+        {
+            // If this is a remote workspace, we don't want to actually select the stack (which would modify
+            // global state); but we will ensure the stack exists by calling `pulumi stack`.
+            var args = new List<string>
+            {
+                "stack",
+            };
+            if (!Remote)
+            {
+                args.Add("select");
+            }
+            args.Add("--stack");
+            args.Add(stackName);
+
+            return RunCommandAsync(args, cancellationToken);
+        }
 
         /// <inheritdoc/>
         public override Task RemoveStackAsync(string stackName, CancellationToken cancellationToken = default)
@@ -730,5 +781,89 @@ public override void Dispose()
                 }
             }
         }
+
+        internal IReadOnlyList<string> GetRemoteArgs()
+        {
+            if (!Remote)
+            {
+                return Array.Empty<string>();
+            }
+
+            var args = new List<string>
+            {
+                "--remote"
+            };
+
+            if (_remoteGitProgramArgs != null)
+            {
+                if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Url))
+                {
+                    args.Add(_remoteGitProgramArgs.Url);
+                }
+                if (!string.IsNullOrEmpty(_remoteGitProgramArgs.ProjectPath))
+                {
+                    args.Add("--remote-git-repo-dir");
+                    args.Add(_remoteGitProgramArgs.ProjectPath);
+                }
+                if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Branch))
+                {
+                    args.Add("--remote-git-branch");
+                    args.Add(_remoteGitProgramArgs.Branch);
+                }
+                if (!string.IsNullOrEmpty(_remoteGitProgramArgs.CommitHash))
+                {
+                    args.Add("--remote-git-commit");
+                    args.Add(_remoteGitProgramArgs.CommitHash);
+                }
+                if (_remoteGitProgramArgs.Auth != null)
+                {
+                    if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Auth.PersonalAccessToken))
+                    {
+                        args.Add("--remote-git-auth-access-token");
+                        args.Add(_remoteGitProgramArgs.Auth.PersonalAccessToken);
+                    }
+                    if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Auth.SshPrivateKey))
+                    {
+                        args.Add("--remote-git-auth-ssh-private-key");
+                        args.Add(_remoteGitProgramArgs.Auth.SshPrivateKey);
+                    }
+                    if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Auth.SshPrivateKeyPath))
+                    {
+                        args.Add("--remote-git-auth-ssh-private-key-path");
+                        args.Add(_remoteGitProgramArgs.Auth.SshPrivateKeyPath);
+                    }
+                    if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Auth.Password))
+                    {
+                        args.Add("--remote-git-auth-password");
+                        args.Add(_remoteGitProgramArgs.Auth.Password);
+                    }
+                    if (!string.IsNullOrEmpty(_remoteGitProgramArgs.Auth.Username))
+                    {
+                        args.Add("--remote-git-username");
+                        args.Add(_remoteGitProgramArgs.Auth.Username);
+                    }
+                }
+            }
+
+            if (_remoteEnvironmentVariables != null)
+            {
+                foreach (var (name, value) in _remoteEnvironmentVariables)
+                {
+                    args.Add(value.IsSecret ? "--remote-env-secret" : "--remote-env");
+                    args.Add($"{name}={value.Value}");
+                }
+            }
+
+            if (_remotePreRunCommands != null)
+            {
+                foreach (var command in _remotePreRunCommands)
+                {
+                    args.Add("--remote-pre-run-command");
+                    args.Add(command);
+                }
+            }
+
+            return args;
+        }
     }
 }

sdk/dotnet/Pulumi.Automation/LocalWorkspaceOptions.cs

@@ -1,4 +1,4 @@
-// Copyright 2016-2021, Pulumi Corporation
+// Copyright 2016-2022, Pulumi Corporation
 
 using System.Collections.Generic;
 using Microsoft.Extensions.Logging;
@@ -64,5 +64,25 @@ public class LocalWorkspaceOptions
         /// <see cref="LocalWorkspace.SaveStackSettingsAsync(string, Automation.StackSettings, System.Threading.CancellationToken)"/>.
         /// </summary>
         public IDictionary<string, StackSettings>? StackSettings { get; set; }
+
+        /// <summary>
+        /// Whether the workspace is a remote workspace.
+        /// </summary>
+        internal bool Remote { get; set; }
+
+        /// <summary>
+        /// Args for remote workspace with Git source.
+        /// </summary>
+        internal RemoteGitProgramArgs? RemoteGitProgramArgs { get; set; }
+
+        /// <summary>
+        /// Environment values scoped to the remote workspace. These will be passed to remote operations.
+        /// </summary>
+        internal IDictionary<string, EnvironmentVariableValue>? RemoteEnvironmentVariables { get; set; }
+
+        /// <summary>
+        /// An optional list of arbitrary commands to run before a remote Pulumi operation is invoked.
+        /// </summary>
+        internal IList<string>? RemotePreRunCommands { get; set; }
     }
 }