allow retries for encrypt/decrypt API calls (#15600)

By default our HTTP client retries all GET requests. However these post
requests are also idempotent, as we just expect to encrypt/decrypt a
single value, without changing anything on the server side. Retry them,
so network errors won't abort the pulumi program.

There's also some log decryption events that might be retryable, but I'm
not sure about them, so I left them alone for now.

Fixes #15236

changelog/pending/20240305--backend-service--make-decrypt-encrypt-network-calls-retryable-to-help-work-around-network-hiccups.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: backend/service
+  description: Make decrypt/encrypt network calls retryable to help work around network hiccups

pkg/backend/httpstate/client/client.go

@@ -472,7 +472,10 @@ func isStackHasResourcesError(err error) bool {
 func (pc *Client) EncryptValue(ctx context.Context, stack StackIdentifier, plaintext []byte) ([]byte, error) {
 	req := apitype.EncryptValueRequest{Plaintext: plaintext}
 	var resp apitype.EncryptValueResponse
-	if err := pc.restCall(ctx, "POST", getStackPath(stack, "encrypt"), nil, &req, &resp); err != nil {
+	if err := pc.restCallWithOptions(
+		ctx, "POST", getStackPath(stack, "encrypt"), nil, &req, &resp,
+		httpCallOptions{RetryPolicy: retryAllMethods},
+	); err != nil {
 		return nil, err
 	}
 	return resp.Ciphertext, nil
@@ -482,7 +485,10 @@ func (pc *Client) EncryptValue(ctx context.Context, stack StackIdentifier, plain
 func (pc *Client) DecryptValue(ctx context.Context, stack StackIdentifier, ciphertext []byte) ([]byte, error) {
 	req := apitype.DecryptValueRequest{Ciphertext: ciphertext}
 	var resp apitype.DecryptValueResponse
-	if err := pc.restCall(ctx, "POST", getStackPath(stack, "decrypt"), nil, &req, &resp); err != nil {
+	if err := pc.restCallWithOptions(
+		ctx, "POST", getStackPath(stack, "decrypt"), nil, &req, &resp,
+		httpCallOptions{RetryPolicy: retryAllMethods},
+	); err != nil {
 		return nil, err
 	}
 	return resp.Plaintext, nil
@@ -513,7 +519,7 @@ func (pc *Client) BulkDecryptValue(ctx context.Context, stack StackIdentifier,
 	req := apitype.BulkDecryptValueRequest{Ciphertexts: ciphertexts}
 	var resp apitype.BulkDecryptValueResponse
 	if err := pc.restCallWithOptions(ctx, "POST", getStackPath(stack, "batch-decrypt"), nil, &req, &resp,
-		httpCallOptions{GzipCompress: true}); err != nil {
+		httpCallOptions{GzipCompress: true, RetryPolicy: retryAllMethods}); err != nil {
 		return nil, err
 	}