From e495fe38aaec597ca73849f5a59c5cc22e41c817 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Wed, 1 Dec 2021 13:54:09 -0800 Subject: [PATCH] cmd/exec: Add new exec subcommand This is just a skeleton but the basic functionality is there: run OPA in a "one shot" mode against a set of input files and print the results for each. Fixes #3525 Signed-off-by: Torin Sandall --- cmd/exec.go | 220 ++++++++++++++++++++++++++++++++++++++ cmd/exec_test.go | 149 ++++++++++++++++++++++++++ cmd/internal/exec/exec.go | 184 +++++++++++++++++++++++++++++++ 3 files changed, 553 insertions(+) create mode 100644 cmd/exec.go create mode 100644 cmd/exec_test.go create mode 100644 cmd/internal/exec/exec.go diff --git a/cmd/exec.go b/cmd/exec.go new file mode 100644 index 0000000000..c3c9556b93 --- /dev/null +++ b/cmd/exec.go @@ -0,0 +1,220 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/open-policy-agent/opa/cmd/internal/exec" + "github.com/open-policy-agent/opa/internal/config" + internal_logging "github.com/open-policy-agent/opa/internal/logging" + "github.com/open-policy-agent/opa/logging" + "github.com/open-policy-agent/opa/plugins" + "github.com/open-policy-agent/opa/plugins/bundle" + "github.com/open-policy-agent/opa/plugins/discovery" + "github.com/open-policy-agent/opa/plugins/logs" + "github.com/open-policy-agent/opa/plugins/status" + "github.com/open-policy-agent/opa/sdk" + "github.com/open-policy-agent/opa/util" + "github.com/spf13/cobra" +) + +func init() { + + var bundlePaths repeatedStringFlag + + params := exec.NewParams(os.Stdout) + + var cmd = &cobra.Command{ + Use: `exec [ [...]]`, + Short: "Execute against input files", + Long: `Execute against input files. + +The 'exec' command executes OPA against one or more input files. If the paths +refer to directories, OPA will execute against files contained inside those +directories, recursively. + +The 'exec' command accepts a --config-file/-c or series of --set options as +arguments. These options behave the same as way as 'opa run'. Since the 'exec' +command is intended to execute OPA in one-shot, the 'exec' command will +manually trigger plugins before and after policy execution: + +Before: Discovery -> Bundle -> Status +After: Decision Logs + +By default, the 'exec' command executes the "default decision" (specified in +the OPA configuration) against each input file. This can be overridden by +specifying the --decision argument and pointing at a specific policy decision, +e.g., opa exec --decision /foo/bar/baz ...`, + + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + params.Paths = args + params.BundlePaths = bundlePaths.v + if err := runExec(params); err != nil { + logging.Get().WithFields(map[string]interface{}{"err": err}).Error("Unexpected error.") + os.Exit(1) + } + }, + } + + addBundleFlag(cmd.Flags(), &bundlePaths) + addOutputFormat(cmd.Flags(), params.OutputFormat) + addConfigFileFlag(cmd.Flags(), ¶ms.ConfigFile) + addConfigOverrides(cmd.Flags(), ¶ms.ConfigOverrides) + addConfigOverrideFiles(cmd.Flags(), ¶ms.ConfigOverrideFiles) + cmd.Flags().StringVarP(¶ms.Decision, "decision", "", "", "set decision to evaluate") + cmd.Flags().VarP(params.LogLevel, "log-level", "l", "set log level") + cmd.Flags().Var(params.LogFormat, "log-format", "set log format") + + RootCommand.AddCommand(cmd) +} + +func runExec(params *exec.Params) error { + + stdLogger, consoleLogger, err := setupLogging(params.LogLevel.String(), params.LogFormat.String()) + if err != nil { + return fmt.Errorf("config error: %w", err) + } + + config, err := setupConfig(params.ConfigFile, params.ConfigOverrides, params.ConfigOverrideFiles, params.BundlePaths) + if err != nil { + return fmt.Errorf("config error: %w", err) + } + + ctx := context.Background() + ready := make(chan struct{}) + + opa, err := sdk.New(ctx, sdk.Options{ + Config: bytes.NewReader(config), + Logger: stdLogger, + ConsoleLogger: consoleLogger, + Ready: ready, + }) + if err != nil { + return fmt.Errorf("runtime error: %w", err) + } + + if err := triggerPlugins(ctx, opa, []string{discovery.Name, bundle.Name, status.Name}); err != nil { + return fmt.Errorf("runtime error: %w", err) + } + + <-ready + + if err := exec.Exec(ctx, opa, params); err != nil { + return fmt.Errorf("exec error: %w", err) + } + + if err := triggerPlugins(ctx, opa, []string{logs.Name}); err != nil { + return fmt.Errorf("runtime error: %w", err) + } + + return nil +} + +func triggerPlugins(ctx context.Context, opa *sdk.OPA, names []string) error { + for _, name := range names { + if p, ok := opa.Plugin(name).(plugins.Triggerable); ok { + if err := p.Trigger(ctx); err != nil { + return err + } + } + } + return nil +} + +func setupLogging(level, format string) (logging.Logger, logging.Logger, error) { + + lvl, err := internal_logging.GetLevel(level) + if err != nil { + return nil, nil, err + } + + logging.Get().SetFormatter(internal_logging.GetFormatter(format)) + logging.Get().SetLevel(lvl) + + stdLogger := logging.New() + stdLogger.SetLevel(lvl) + stdLogger.SetFormatter(internal_logging.GetFormatter(format)) + + consoleLogger := logging.New() + consoleLogger.SetFormatter(internal_logging.GetFormatter(format)) + + return stdLogger, consoleLogger, nil +} + +func setupConfig(file string, overrides []string, overrideFiles []string, bundlePaths []string) ([]byte, error) { + + bs, err := config.Load(file, overrides, overrideFiles) + if err != nil { + return nil, err + } + + var root map[string]interface{} + + if err := util.Unmarshal(bs, &root); err != nil { + return nil, err + } + + if err := injectExplicitBundles(root, bundlePaths); err != nil { + return nil, err + } + + // NOTE(tsandall): This could be generalized in the future if we need to + // deal with arbitrary plugins. + + // NOTE(tsandall): Overriding the discovery trigger mode to manual means + // that all plugins will inherit the trigger mode by default. If the plugin + // trigger mode is explicitly set to something other than 'manual' this will + // result in a configuration error. + if cfg, ok := root["discovery"].(map[string]interface{}); ok { + cfg["trigger"] = "manual" + } + + if cfg, ok := root["bundles"].(map[string]interface{}); ok { + for _, x := range cfg { + if bcfg, ok := x.(map[string]interface{}); ok { + bcfg["trigger"] = "manual" + } + } + } + + if cfg, ok := root["decision_logs"].(map[string]interface{}); ok { + if rcfg, ok := cfg["reporting"].(map[string]interface{}); ok { + rcfg["trigger"] = "manual" + } + } + + if cfg, ok := root["status"].(map[string]interface{}); ok { + cfg["trigger"] = "manual" + } + + return json.Marshal(root) +} + +func injectExplicitBundles(root map[string]interface{}, paths []string) error { + if len(paths) == 0 { + return nil + } + + bundles, ok := root["bundles"].(map[string]interface{}) + if !ok { + bundles = map[string]interface{}{} + root["bundles"] = bundles + } + + for i := range paths { + abspath, err := filepath.Abs(paths[i]) + if err != nil { + return err + } + bundles[fmt.Sprintf("~%d", i)] = map[string]interface{}{ + "resource": fmt.Sprintf("file://%v", abspath), + } + } + + return nil +} diff --git a/cmd/exec_test.go b/cmd/exec_test.go new file mode 100644 index 0000000000..aa2756394b --- /dev/null +++ b/cmd/exec_test.go @@ -0,0 +1,149 @@ +package cmd + +import ( + "bytes" + "reflect" + "testing" + + "github.com/open-policy-agent/opa/cmd/internal/exec" + sdk_test "github.com/open-policy-agent/opa/sdk/test" + "github.com/open-policy-agent/opa/util" + "github.com/open-policy-agent/opa/util/test" +) + +func TestExecBasic(t *testing.T) { + + files := map[string]string{ + "test.json": `{"foo": 7}`, + "test2.yaml": `bar: 8`, + "test3.yml": `baz: 9`, + "ignore": `garbage`, // do not recognize this filetype + } + + test.WithTempFS(files, func(dir string) { + + s := sdk_test.MustNewServer(sdk_test.MockBundle("/bundles/bundle.tar.gz", map[string]string{ + "test.rego": ` + package system + main["hello"] + `, + })) + + defer s.Stop() + + var buf bytes.Buffer + params := exec.NewParams(&buf) + _ = params.OutputFormat.Set("json") + params.ConfigOverrides = []string{ + "services.test.url=" + s.URL(), + "bundles.test.resource=/bundles/bundle.tar.gz", + } + + params.Paths = append(params.Paths, dir) + err := runExec(params) + if err != nil { + t.Fatal(err) + } + + output := util.MustUnmarshalJSON(bytes.ReplaceAll(buf.Bytes(), []byte(dir), nil)) + + exp := util.MustUnmarshalJSON([]byte(`{"result": [{ + "path": "/test.json", + "result": ["hello"] + }, { + "path": "/test2.yaml", + "result": ["hello"] + }, { + "path": "/test3.yml", + "result": ["hello"] + }]}`)) + + if !reflect.DeepEqual(output, exp) { + t.Fatal("Expected:", exp, "Got:", output) + } + }) + +} + +func TestExecDecisionOption(t *testing.T) { + + files := map[string]string{ + "test.json": `{"foo": 7}`, + } + + test.WithTempFS(files, func(dir string) { + + s := sdk_test.MustNewServer(sdk_test.MockBundle("/bundles/bundle.tar.gz", map[string]string{ + "test.rego": ` + package foo + main["hello"] + `, + })) + + defer s.Stop() + + var buf bytes.Buffer + params := exec.NewParams(&buf) + _ = params.OutputFormat.Set("json") + params.Decision = "foo/main" + params.ConfigOverrides = []string{ + "services.test.url=" + s.URL(), + "bundles.test.resource=/bundles/bundle.tar.gz", + } + + params.Paths = append(params.Paths, dir) + err := runExec(params) + if err != nil { + t.Fatal(err) + } + + output := util.MustUnmarshalJSON(bytes.ReplaceAll(buf.Bytes(), []byte(dir), nil)) + + exp := util.MustUnmarshalJSON([]byte(`{"result": [{ + "path": "/test.json", + "result": ["hello"] + }]}`)) + + if !reflect.DeepEqual(output, exp) { + t.Fatal("Expected:", exp, "Got:", output) + } + + }) + +} + +func TestExecBundleFlag(t *testing.T) { + + files := map[string]string{ + "files/test.json": `{"foo": 7}`, + "bundle/x.rego": `package system + + main["hello"]`, + } + + test.WithTempFS(files, func(dir string) { + + var buf bytes.Buffer + params := exec.NewParams(&buf) + _ = params.OutputFormat.Set("json") + params.BundlePaths = []string{dir + "/bundle/"} + params.Paths = append(params.Paths, dir+"/files/") + + err := runExec(params) + if err != nil { + t.Fatal(err) + } + + output := util.MustUnmarshalJSON(bytes.ReplaceAll(buf.Bytes(), []byte(dir), nil)) + + exp := util.MustUnmarshalJSON([]byte(`{"result": [{ + "path": "/files/test.json", + "result": ["hello"] + }]}`)) + + if !reflect.DeepEqual(output, exp) { + t.Fatal("Expected:", exp, "Got:", output) + } + + }) +} diff --git a/cmd/internal/exec/exec.go b/cmd/internal/exec/exec.go new file mode 100644 index 0000000000..ea12593fed --- /dev/null +++ b/cmd/internal/exec/exec.go @@ -0,0 +1,184 @@ +package exec + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "time" + + "github.com/open-policy-agent/opa/sdk" + "github.com/open-policy-agent/opa/util" +) + +type Params struct { + Paths []string // file paths to execute against + Output io.Writer // output stream to write normal output to + ConfigFile string // OPA configuration file path + ConfigOverrides []string // OPA configuration overrides (--set arguments) + ConfigOverrideFiles []string // OPA configuration overrides (--set-file arguments) + OutputFormat *util.EnumFlag // output format (default: pretty) + LogLevel *util.EnumFlag // log level for plugins + LogFormat *util.EnumFlag // log format for plugins + BundlePaths []string // explicit paths of bundles to inject into the configuration + Decision string // decision to evaluate (overrides default decision set by configuration) +} + +func NewParams(w io.Writer) *Params { + return &Params{ + Output: w, + OutputFormat: util.NewEnumFlag("pretty", []string{"pretty", "json"}), + LogLevel: util.NewEnumFlag("error", []string{"debug", "info", "error"}), + LogFormat: util.NewEnumFlag("json", []string{"text", "json", "json-pretty"}), + } +} + +// Exec executes OPA against the supplied files and outputs each result. +// +// NOTE(tsandall): consider expanding functionality: +// +// * specialized output formats (e.g., pretty/non-JSON outputs) +// * exit codes set by convention or policy (e.g,. non-empty set => error) +// * support for new input file formats beyond JSON and YAML +func Exec(ctx context.Context, opa *sdk.OPA, params *Params) error { + + now := time.Now() + r := &jsonReporter{w: params.Output, buf: make([]result, 0)} + + for item := range listAllPaths(params.Paths) { + + if item.Error != nil { + return item.Error + } + + input, err := parse(item.Path) + + if err != nil { + if err2 := r.Report(result{Path: item.Path, Error: err}); err2 != nil { + return err2 + } + continue + } else if input == nil { + continue + } + + rs, err := opa.Decision(ctx, sdk.DecisionOptions{ + Path: params.Decision, + Now: now, + Input: input, + }) + if err != nil { + if err2 := r.Report(result{Path: item.Path, Error: err}); err2 != nil { + return err2 + } + continue + } + + if err := r.Report(result{Path: item.Path, Result: &rs.Result}); err != nil { + return err + } + } + + return r.Close() +} + +type result struct { + Path string `json:"path"` + Error error `json:"error,omitempty"` + Result *interface{} `json:"result,omitempty"` +} + +type jsonReporter struct { + w io.Writer + buf []result +} + +func (jr *jsonReporter) Report(r result) error { + jr.buf = append(jr.buf, r) + return nil +} + +func (jr *jsonReporter) Close() error { + enc := json.NewEncoder(jr.w) + enc.SetIndent("", " ") + return enc.Encode(struct { + Result []result `json:"result"` + }{ + Result: jr.buf, + }) +} + +type fileListItem struct { + Path string + Error error +} + +func listAllPaths(roots []string) chan fileListItem { + ch := make(chan fileListItem) + go func() { + for _, path := range roots { + err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + ch <- fileListItem{Path: path} + return nil + }) + if err != nil { + ch <- fileListItem{Path: path, Error: err} + } + } + close(ch) + }() + return ch +} + +var parsers = map[string]parser{ + ".json": utilParser{}, + ".yaml": utilParser{}, + ".yml": utilParser{}, +} + +type parser interface { + Parse(io.Reader) (interface{}, error) +} + +type utilParser struct { +} + +func (utilParser) Parse(r io.Reader) (interface{}, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + var x interface{} + return x, util.Unmarshal(bs, &x) +} + +func parse(p string) (*interface{}, error) { + + parser, ok := parsers[path.Ext(p)] + if !ok { + return nil, nil + } + + f, err := os.Open(p) + if err != nil { + return nil, err + } + + defer f.Close() + + val, err := parser.Parse(f) + if err != nil { + return nil, err + } + + return &val, nil +}