Allow Custom CLI commands to be any (complex) shell commands (#260)

* Allow Custom CLI commands to be any (complex) shell commands

... using mvdan.cc/sh/v3's shell parser and interpreter.

Also execute "valueCommand commands" the same way when defining
environment variables for custom CLI commands.

* Use full package reference

Co-authored-by: nitrocode <7775707+nitrocode@users.noreply.github.com>

* Actually, it's really 'mvdan.cc/sh/v3'

* Prevent potential parsing errors and quote words in custom commands

* In verbose mode, display shell command earlier

This could help debugging failing commands ;-)

* Factoring shell script execution

Co-authored-by: nitrocode <7775707+nitrocode@users.noreply.github.com>
Co-authored-by: Andriy Knysh <aknysh@users.noreply.github.com>

atmos.yaml

@@ -170,20 +170,20 @@ commands:
         # Steps support using Go templates and can access all configuration settings (e.g. {{ .ComponentConfig.xxx.yyy.zzz }})
         # Steps also have access to the ENV vars defined in the 'env' section of the 'command'
         steps:
-          - 'echo Atmos component from argument: {{ .Arguments.component }}'
-          - 'echo ATMOS_COMPONENT: $ATMOS_COMPONENT'
-          - 'echo Atmos stack: {{ .Flags.stack }}'
-          - 'echo Terraform component: {{ .ComponentConfig.component }}'
-          - 'echo Backend S3 bucket: {{ .ComponentConfig.backend.bucket }}'
-          - 'echo Terraform workspace: {{ .ComponentConfig.workspace }}'
-          - 'echo Namespace: {{ .ComponentConfig.vars.namespace }}'
-          - 'echo Tenant: {{ .ComponentConfig.vars.tenant }}'
-          - 'echo Environment: {{ .ComponentConfig.vars.environment }}'
-          - 'echo Stage: {{ .ComponentConfig.vars.stage }}'
-          - 'echo settings.spacelift.workspace_enabled: {{ .ComponentConfig.settings.spacelift.workspace_enabled }}'
-          - 'echo Dependencies: {{ .ComponentConfig.deps }}'
-          - 'echo settings.config.is_prod: {{ .ComponentConfig.settings.config.is_prod }}'
-          - 'echo ATMOS_IS_PROD: $ATMOS_IS_PROD'
+          - 'echo Atmos component from argument: "{{ .Arguments.component }}"'
+          - 'echo ATMOS_COMPONENT: "$ATMOS_COMPONENT"'
+          - 'echo Atmos stack: "{{ .Flags.stack }}"'
+          - 'echo Terraform component: "{{ .ComponentConfig.component }}"'
+          - 'echo Backend S3 bucket: "{{ .ComponentConfig.backend.bucket }}"'
+          - 'echo Terraform workspace: "{{ .ComponentConfig.workspace }}"'
+          - 'echo Namespace: "{{ .ComponentConfig.vars.namespace }}"'
+          - 'echo Tenant: "{{ .ComponentConfig.vars.tenant }}"'
+          - 'echo Environment: "{{ .ComponentConfig.vars.environment }}"'
+          - 'echo Stage: "{{ .ComponentConfig.vars.stage }}"'
+          - 'echo settings.spacelift.workspace_enabled: "{{ .ComponentConfig.settings.spacelift.workspace_enabled }}"'
+          - 'echo Dependencies: "{{ .ComponentConfig.deps }}"'
+          - 'echo settings.config.is_prod: "{{ .ComponentConfig.settings.config.is_prod }}"'
+          - 'echo ATMOS_IS_PROD: "$ATMOS_IS_PROD"'
 
 # Integrations
 integrations:

cmd/cmd_utils.go

@@ -7,7 +7,6 @@ import (
 	e "github.com/cloudposse/atmos/internal/exec"
 	"github.com/spf13/cobra"
 	"os"
-	"strings"
 	"text/template"
 
 	cfg "github.com/cloudposse/atmos/pkg/config"
@@ -192,8 +191,8 @@ func executeCustomCommand(cmd *cobra.Command, args []string, parentCommand *cobr
 
 			// If the command to get the value for the ENV var is provided, execute it
 			if valCommand != "" {
-				valCommandArgs := strings.Fields(valCommand)
-				res, err := e.ExecuteShellCommandAndReturnOutput(valCommandArgs[0], valCommandArgs[1:], ".", nil, false, commandConfig.Verbose)
+				valCommandName := fmt.Sprintf("env-var-%s-valcommand", key)
+				res, err := e.ExecuteShellAndReturnOutput(valCommand, valCommandName, ".", nil, false, commandConfig.Verbose)
 				if err != nil {
 					u.PrintErrorToStdErrorAndExit(err)
 				}
@@ -222,15 +221,14 @@ func executeCustomCommand(cmd *cobra.Command, args []string, parentCommand *cobr
 
 		// Process Go templates in the command's steps.
 		// Steps support Go templates and have access to {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables
-		commandTmpl, err := processTmpl(fmt.Sprintf("step-%d", i), step, data)
+		commandToRun, err := processTmpl(fmt.Sprintf("step-%d", i), step, data)
 		if err != nil {
 			u.PrintErrorToStdErrorAndExit(err)
 		}
-		commandToRun := os.ExpandEnv(commandTmpl)
 
 		// Execute the command step
-		stepArgs := strings.Fields(commandToRun)
-		err = e.ExecuteShellCommand(stepArgs[0], stepArgs[1:], ".", envVarsList, false, commandConfig.Verbose)
+		commandName := fmt.Sprintf("%s-step-%d", commandConfig.Name, i)
+		err = e.ExecuteShell(commandToRun, commandName, ".", envVarsList, false, commandConfig.Verbose)
 		if err != nil {
 			u.PrintErrorToStdErrorAndExit(err)
 		}

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/zclconf/go-cty v1.12.1
 	gopkg.in/yaml.v2 v2.4.0
+	mvdan.cc/sh/v3 v3.5.1
 )
 
 require (
@@ -111,6 +112,7 @@ require (
 	golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
 	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.102.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -745,6 +745,7 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1039,6 +1040,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+mvdan.cc/sh/v3 v3.5.1 h1:hmP3UOw4f+EYexsJjFxvU38+kn+V/s2CclXHanIBkmQ=
+mvdan.cc/sh/v3 v3.5.1/go.mod h1:1JcoyAKm1lZw/2bZje/iYKWicU/KMd0rsyJeKHnsK4E=
 oras.land/oras-go v1.2.1 h1:/VcGS8FUy3eEXLl/1vC4QypLHwrfSmgW7ygsoklqKK8=
 oras.land/oras-go v1.2.1/go.mod h1:3N11Z5E3c4ZzOjroCl1RtAdB4yNAYl7A27j2SVf913A=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

internal/exec/shell_utils.go

@@ -1,13 +1,19 @@
 package exec
 
 import (
+	"bytes"
+	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"runtime"
 	"strings"
 
 	u "github.com/cloudposse/atmos/pkg/utils"
+	"mvdan.cc/sh/v3/expand"
+	"mvdan.cc/sh/v3/interp"
+	"mvdan.cc/sh/v3/syntax"
 )
 
 // ExecuteShellCommand prints and executes the provided command with args and flags
@@ -31,6 +37,62 @@ func ExecuteShellCommand(command string, args []string, dir string, env []string
 	return cmd.Run()
 }
 
+// ExecuteShell runs a shell script
+func ExecuteShell(command string, name string, dir string, env []string, dryRun bool, verbose bool) error {
+	if verbose {
+		u.PrintInfo("\nExecuting command:")
+		fmt.Println(command)
+	}
+
+	if dryRun {
+		return nil
+	}
+
+	return shellRunner(command, name, dir, env, os.Stdout)
+}
+
+// ExecuteShellAndReturnOutput runs a shell script and capture its standard output
+func ExecuteShellAndReturnOutput(command string, name string, dir string, env []string, dryRun bool, verbose bool) (string, error) {
+	var b bytes.Buffer
+
+	if verbose {
+		u.PrintInfo("\nExecuting command:")
+		fmt.Println(command)
+	}
+
+	if dryRun {
+		return "", nil
+	}
+
+	err := shellRunner(command, name, dir, env, &b)
+	if err != nil {
+		return "", err
+	}
+
+	return b.String(), nil
+}
+
+// shellRunner uses mvdan.cc/sh/v3's parser and interpreter to run a shell script and divert its stdout
+func shellRunner(command string, name string, dir string, env []string, out io.Writer) error {
+	parser, err := syntax.NewParser().Parse(strings.NewReader(command), name)
+	if err != nil {
+		return err
+	}
+
+	environ := append(os.Environ(), env...)
+	listEnviron := expand.ListEnviron(environ...)
+	runner, err := interp.New(
+		interp.Dir(dir),
+		interp.Env(listEnviron),
+		interp.StdIO(os.Stdin, out, os.Stderr),
+	)
+	if err != nil {
+		return err
+	}
+
+	return runner.Run(context.TODO(), parser)
+}
+
 // ExecuteShellCommandAndReturnOutput prints and executes the provided command with args and flags and returns the command output
 func ExecuteShellCommandAndReturnOutput(command string, args []string, dir string, env []string, dryRun bool, verbose bool) (string, error) {
 	cmd := exec.Command(command, args...)