Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow Custom CLI commands to be any (complex) shell commands #260

Merged
merged 8 commits into from Nov 30, 2022
Merged

Allow Custom CLI commands to be any (complex) shell commands #260

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f53b981
Allow Custom CLI commands to be any (complex) shell commands
stoned Nov 20, 2022
f3f948d
Use full package reference
stoned Nov 20, 2022
b8f8c08
Merge branch 'master' into sh-as-custom-cmds
aknysh Nov 20, 2022
76752bf
Actually, it's really 'mvdan.cc/sh/v3'
stoned Nov 20, 2022
de34f16
Prevent potential parsing errors and quote words in custom commands
stoned Nov 20, 2022
73387a5
In verbose mode, display shell command earlier
stoned Nov 20, 2022
263e404
Merge branch 'master' into sh-as-custom-cmds
aknysh Nov 28, 2022
8eb75df
Factoring shell script execution
stoned Nov 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 5 additions & 7 deletions cmd/cmd_utils.go
View file
Open in desktop
Expand Up @@ -7,7 +7,6 @@ import (
e "github.com/cloudposse/atmos/internal/exec"
"github.com/spf13/cobra"
"os"
"strings"
"text/template"

cfg "github.com/cloudposse/atmos/pkg/config"
Expand Down Expand Up @@ -178,8 +177,8 @@ func executeCustomCommand(cmd *cobra.Command, args []string, parentCommand *cobr

// If the command to get the value for the ENV var is provided, execute it
if valCommand != "" {
valCommandArgs := strings.Fields(valCommand)
res, err := e.ExecuteShellCommandAndReturnOutput(valCommandArgs[0], valCommandArgs[1:], ".", nil, false, commandConfig.Verbose)
valCommandName := fmt.Sprintf("env-var-%s-valcommand", key)
res, err := e.ExecuteShellAndReturnOutput(valCommand, valCommandName, ".", nil, false, commandConfig.Verbose)
if err != nil {
u.PrintErrorToStdErrorAndExit(err)
}
Expand Down Expand Up @@ -208,15 +207,14 @@ func executeCustomCommand(cmd *cobra.Command, args []string, parentCommand *cobr

// Process Go templates in the command's steps.
// Steps support Go templates and have access to {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables
commandTmpl, err := processTmpl(fmt.Sprintf("step-%d", i), step, data)
commandToRun, err := processTmpl(fmt.Sprintf("step-%d", i), step, data)
if err != nil {
u.PrintErrorToStdErrorAndExit(err)
}
commandToRun := os.ExpandEnv(commandTmpl)

// Execute the command step
stepArgs := strings.Fields(commandToRun)
err = e.ExecuteShellCommand(stepArgs[0], stepArgs[1:], ".", envVarsList, false, commandConfig.Verbose)
commandName := fmt.Sprintf("%s-step-%d", commandConfig.Name, i)
err = e.ExecuteShell(commandToRun, commandName, ".", envVarsList, false, commandConfig.Verbose)
if err != nil {
u.PrintErrorToStdErrorAndExit(err)
}
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@ require (
github.com/stretchr/testify v1.8.1
github.com/zclconf/go-cty v1.12.0
gopkg.in/yaml.v2 v2.4.0
mvdan.cc/sh/v3 v3.5.1
)

require (
Expand Down Expand Up @@ -111,6 +112,7 @@ require (
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.102.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Open in desktop
Expand Up @@ -745,6 +745,7 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down Expand Up @@ -1039,6 +1040,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
mvdan.cc/sh/v3 v3.5.1 h1:hmP3UOw4f+EYexsJjFxvU38+kn+V/s2CclXHanIBkmQ=
mvdan.cc/sh/v3 v3.5.1/go.mod h1:1JcoyAKm1lZw/2bZje/iYKWicU/KMd0rsyJeKHnsK4E=
oras.land/oras-go v1.2.1 h1:/VcGS8FUy3eEXLl/1vC4QypLHwrfSmgW7ygsoklqKK8=
oras.land/oras-go v1.2.1/go.mod h1:3N11Z5E3c4ZzOjroCl1RtAdB4yNAYl7A27j2SVf913A=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
Expand Down
67 changes: 67 additions & 0 deletions internal/exec/shell_utils.go
View file
Open in desktop
@@ -1,13 +1,18 @@
package exec

import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"runtime"
"strings"

u "github.com/cloudposse/atmos/pkg/utils"
"github.com/mvdan/sh/v3/expand"
"mvdan.cc/sh/v3/interp"
"mvdan.cc/sh/v3/syntax"
)

// ExecuteShellCommand prints and executes the provided command with args and flags
Expand All @@ -31,6 +36,68 @@ func ExecuteShellCommand(command string, args []string, dir string, env []string
return cmd.Run()
}

// ExecuteShell uses mvdan.cc/sh/v3's parser and interpreter to run a shell script
func ExecuteShell(command string, name string, dir string, env []string, dryRun bool, verbose bool) error {
parser, err := syntax.NewParser().Parse(strings.NewReader(command), name)
if err != nil {
return err
}

runner, err := interp.New(
interp.Dir(dir),
interp.Env(expand.ListEnviron(append(os.Environ(), env...)...)),
Copy link
Member

@aknysh aknysh Nov 28, 2022 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@stoned you removed

commandToRun := os.ExpandEnv(commandTmpl)

which is used to replace ${var} or $var in the command steps at runtime according to the values of the current environment variables.

Does interp.Env(expand do the same thing?

Also, we prob need to simplify

interp.Env(expand.ListEnviron(append(os.Environ(), env...)...)),

by using a local var b/c it's difficult to read

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exactly, mvdan.cc/sh/v3/interp.Env() will setup the environment to use: there is no need to have os.ExpandEnv() making its own substitutions.
And, yes, I will rewrite the call to interp.Env() to be, hopefully, simpler to read.

interp.StdIO(os.Stdin, os.Stdout, os.Stderr),
)
if err != nil {
return err
}

if verbose {
u.PrintInfo("\nExecuting command:")
fmt.Println(command)
}

if dryRun {
return nil
}

return runner.Run(context.TODO(), parser)
}

// ExecuteShell uses mvdan.cc/sh/v3's parser and interpreter to run a shell script and capture its standard output
func ExecuteShellAndReturnOutput(command string, name string, dir string, env []string, dryRun bool, verbose bool) (string, error) {
var b bytes.Buffer
parser, err := syntax.NewParser().Parse(strings.NewReader(command), name)
if err != nil {
return "", err
}

runner, err := interp.New(
interp.Dir(dir),
interp.Env(expand.ListEnviron(append(os.Environ(), env...)...)),
interp.StdIO(os.Stdin, &b, os.Stderr),
)
if err != nil {
return "", err
}

if verbose {
aknysh marked this conversation as resolved.
Show resolved Hide resolved
u.PrintInfo("\nExecuting command:")
fmt.Println(command)
}

if dryRun {
return "", nil
}

err = runner.Run(context.TODO(), parser)
if err != nil {
return "", err
}

return b.String(), nil
}

// ExecuteShellCommandAndReturnOutput prints and executes the provided command with args and flags and returns the command output
func ExecuteShellCommandAndReturnOutput(command string, args []string, dir string, env []string, dryRun bool, verbose bool) (string, error) {
aknysh marked this conversation as resolved.
Show resolved Hide resolved
cmd := exec.Command(command, args...)
Expand Down