Create helpers which integrate with OpenTelemetry for diagnose collec…

…tion (#11454) * Create helpers which integrate with OpenTelemetry for diagnose collection * Go mod vendor * Comments * Update vault/diagnose/helpers.go Co-authored-by: swayne275 <swayne275@gmail.com> * Add unit test/example * tweak output * More comments * add spot check concept * Get unit tests working on Result structs * Fix unit test * Get unit tests working, and make diagnose sessions local rather than global * Comments * Last comments * No need for init * :| * Fix helpers_test Co-authored-by: swayne275 <swayne275@gmail.com>
hashicorp · Apr 29, 2021 · d60057b · d60057b
1 parent aafede4
commit d60057b
Show file tree

Hide file tree

Showing 123 changed files with 16,616 additions and 175 deletions.
diff --git a/command/operator_diagnose.go b/command/operator_diagnose.go
@@ -1,7 +1,7 @@
 package command
 
 import (
-	"fmt"
+	"context"
 	"strings"
 	"sync"
 
@@ -26,6 +26,7 @@ var (
 
 type OperatorDiagnoseCommand struct {
 	*BaseCommand
+	diagnose *diagnose.Session
 
 	flagDebug    bool
 	flagSkips    []string
@@ -129,6 +130,16 @@ func (c *OperatorDiagnoseCommand) RunWithParsedFlags() int {
 	}
 
 	c.UI.Output(version.GetVersion().FullVersionNumber(true))
+	ctx := diagnose.Context(context.Background(), c.diagnose)
+	err := c.offlineDiagnostics(ctx)
+
+	if err != nil {
+		return 1
+	}
+	return 0
+}
+
+func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error {
 	rloadFuncs := make(map[string][]reloadutil.ReloadFunc)
 	server := &ServerCommand{
 		// TODO: set up a different one?
@@ -150,126 +161,106 @@ func (c *OperatorDiagnoseCommand) RunWithParsedFlags() int {
 		reloadFuncsLock: new(sync.RWMutex),
 	}
 
-	phase := "Parse configuration"
+	ctx, span := diagnose.StartSpan(ctx, "initialization")
+	defer span.End()
+
 	server.flagConfigs = c.flagConfigs
 	config, err := server.parseConfig()
 	if err != nil {
-		c.outputErrorsAndWarnings(fmt.Sprintf("Error while reading configuration files: %s", err.Error()), phase, nil)
-		return 1
+		return diagnose.SpotError(ctx, "parse-config", err)
+	} else {
+		diagnose.SpotOk(ctx, "parse-config", "")
 	}
-	c.UI.Output(same_line + status_ok + phase)
-
 	// Check Listener Information
-	phase = "Check Listeners"
 	// TODO: Run Diagnose checks on the actual net.Listeners
-	var warnings []string
-	disableClustering := config.HAStorage.DisableClustering
-	infoKeys := make([]string, 0, 10)
-	info := make(map[string]string)
-	status, lns, _, errMsg := server.InitListeners(config, disableClustering, &infoKeys, &info)
-
-	if status != 0 {
-		c.outputErrorsAndWarnings(fmt.Sprintf("Error parsing listener configuration. %s", errMsg.Error()), phase, nil)
-		return 1
-	}
-
-	// Make sure we close all listeners from this point on
-	listenerCloseFunc := func() {
-		for _, ln := range lns {
-			ln.Listener.Close()
-		}
-	}
 
-	defer c.cleanupGuard.Do(listenerCloseFunc)
+	if err := diagnose.Test(ctx, "init-listeners", func(ctx context.Context) error {
+		disableClustering := config.HAStorage.DisableClustering
+		infoKeys := make([]string, 0, 10)
+		info := make(map[string]string)
+		status, lns, _, errMsg := server.InitListeners(config, disableClustering, &infoKeys, &info)
 
-	sanitizedListeners := make([]listenerutil.Listener, 0, len(config.Listeners))
-	for _, ln := range lns {
-		if ln.Config.TLSDisable {
-			warnings = append(warnings, "WARNING! TLS is disabled in a Listener config stanza.")
-			continue
-		}
-		if ln.Config.TLSDisableClientCerts {
-			warnings = append(warnings, "WARNING! TLS for a listener is turned on without requiring client certs.")
+		if status != 0 {
+			return errMsg
 		}
 
-		// Check ciphersuite and load ca/cert/key files
-		// TODO: TLSConfig returns a reloadFunc and a TLSConfig. We can use this to
-		// perform an active probe.
-		_, _, err := listenerutil.TLSConfig(ln.Config, make(map[string]string), c.UI)
-		if err != nil {
-			c.outputErrorsAndWarnings(fmt.Sprintf("Error creating TLS Configuration out of config file:  %s", err.Error()), phase, warnings)
-			return 1
+		// Make sure we close all listeners from this point on
+		listenerCloseFunc := func() {
+			for _, ln := range lns {
+				ln.Listener.Close()
+			}
 		}
 
-		sanitizedListeners = append(sanitizedListeners, listenerutil.Listener{
-			Listener: ln.Listener,
-			Config:   ln.Config,
-		})
-	}
-	err = diagnose.ListenerChecks(sanitizedListeners)
-	if err != nil {
-		c.outputErrorsAndWarnings(fmt.Sprintf("Diagnose caught configuration errors:  %s", err.Error()), phase, warnings)
-		return 1
-	}
+		defer c.cleanupGuard.Do(listenerCloseFunc)
 
-	c.UI.Output(same_line + status_ok + phase)
+		sanitizedListeners := make([]listenerutil.Listener, 0, len(config.Listeners))
+		for _, ln := range lns {
+			if ln.Config.TLSDisable {
+				diagnose.Warn(ctx, "TLS is disabled in a Listener config stanza.")
+				continue
+			}
+			if ln.Config.TLSDisableClientCerts {
+				diagnose.Warn(ctx, "TLS for a listener is turned on without requiring client certs.")
+			}
+
+			// Check ciphersuite and load ca/cert/key files
+			// TODO: TLSConfig returns a reloadFunc and a TLSConfig. We can use this to
+			// perform an active probe.
+			_, _, err := listenerutil.TLSConfig(ln.Config, make(map[string]string), c.UI)
+			if err != nil {
+				return err
+			}
+
+			sanitizedListeners = append(sanitizedListeners, listenerutil.Listener{
+				Listener: ln.Listener,
+				Config:   ln.Config,
+			})
+		}
+		return diagnose.ListenerChecks(sanitizedListeners)
+	}); err != nil {
+		return err
+	}
 
 	// Errors in these items could stop Vault from starting but are not yet covered:
 	// TODO: logging configuration
 	// TODO: SetupTelemetry
 	// TODO: check for storage backend
 
-	phase = "Access storage"
-	warnings = nil
-
-	_, err = server.setupStorage(config)
-	if err != nil {
-		c.outputErrorsAndWarnings(fmt.Sprintf(err.Error()), phase, warnings)
-		return 1
-	}
+	if err := diagnose.Test(ctx, "storage", func(ctx context.Context) error {
 
-	if config.Storage != nil && config.Storage.Type == storageTypeConsul {
-		err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.Storage.Config, server.logger, true)
+		_, err = server.setupStorage(config)
 		if err != nil {
-			c.outputErrorsAndWarnings(fmt.Sprintf(err.Error()), phase, warnings)
-			return 1
+			return err
 		}
-	}
 
-	if config.HAStorage != nil && config.HAStorage.Type == storageTypeConsul {
-		err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.HAStorage.Config, server.logger, true)
-		if err != nil {
-			c.outputErrorsAndWarnings(fmt.Sprintf(err.Error()), phase, warnings)
-			return 1
+		if config.Storage != nil && config.Storage.Type == storageTypeConsul {
+			err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.Storage.Config, server.logger, true)
+			if err != nil {
+				return err
+			}
 		}
-	}
-
-	c.UI.Output(same_line + status_ok + phase)
 
-	// Initialize the Service Discovery, if there is one
-	phase = "Service discovery"
-	if config.ServiceRegistration != nil && config.ServiceRegistration.Type == "consul" {
-		// SetupSecureTLS for service discovery uses the same cert and key to set up physical
-		// storage. See the consul package in physical for details.
-		err = srconsul.SetupSecureTLS(api.DefaultConfig(), config.ServiceRegistration.Config, server.logger, true)
-		if err != nil {
-			c.outputErrorsAndWarnings(fmt.Sprintf(err.Error()), phase, warnings)
-			return 1
+		if config.HAStorage != nil && config.HAStorage.Type == storageTypeConsul {
+			err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.HAStorage.Config, server.logger, true)
+			if err != nil {
+				return err
+			}
 		}
+		return nil
+	}); err != nil {
+		return err
 	}
-	c.UI.Output(same_line + status_ok + phase)
-
-	return 0
-}
-
-func (c *OperatorDiagnoseCommand) outputErrorsAndWarnings(err, phase string, warnings []string) {
-
-	c.UI.Output(same_line + status_failed + phase)
 
-	if warnings != nil && len(warnings) > 0 {
-		for _, warning := range warnings {
-			c.UI.Warn(warning)
+	return diagnose.Test(ctx, "service-discovery", func(ctx context.Context) error {
+		// Initialize the Service Discovery, if there is one
+		if config.ServiceRegistration != nil && config.ServiceRegistration.Type == "consul" {
+			// SetupSecureTLS for service discovery uses the same cert and key to set up physical
+			// storage. See the consul package in physical for details.
+			err = srconsul.SetupSecureTLS(api.DefaultConfig(), config.ServiceRegistration.Config, server.logger, true)
+			if err != nil {
+				return err
+			}
 		}
-	}
-	c.UI.Error(err)
+		return nil
+	})
 }