From b2f7d6f7bdbdba6bb69337eb9568da24b61aa27c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 7 Dec 2021 17:52:04 -0500 Subject: [PATCH] pkg/cmd: add serve package --- cmd/spicedb/main.go | 45 +++---- ...st.go => servetesting_integration_test.go} | 0 .../{devsvc/devsvc.go => serve/devtools.go} | 6 +- {cmd/spicedb => pkg/cmd/serve}/serve.go | 115 ++++++++++-------- .../testserver.go => pkg/cmd/serve/testing.go | 21 ++-- 5 files changed, 95 insertions(+), 92 deletions(-) rename cmd/spicedb/{testserver_integration_test.go => servetesting_integration_test.go} (100%) rename pkg/cmd/{devsvc/devsvc.go => serve/devtools.go} (97%) rename {cmd/spicedb => pkg/cmd/serve}/serve.go (66%) rename cmd/spicedb/testserver.go => pkg/cmd/serve/testing.go (95%) diff --git a/cmd/spicedb/main.go b/cmd/spicedb/main.go index ab555aeae2..7d918f2e22 100644 --- a/cmd/spicedb/main.go +++ b/cmd/spicedb/main.go @@ -2,21 +2,16 @@ package main import ( "math/rand" - "net/http" - "net/http/pprof" "time" "github.com/cespare/xxhash" - "github.com/jzelinskie/cobrautil" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/rs/zerolog" "github.com/sercand/kuberesolver/v3" "google.golang.org/grpc/balancer" consistentbalancer "github.com/authzed/spicedb/pkg/balancer" - "github.com/authzed/spicedb/pkg/cmd/devsvc" "github.com/authzed/spicedb/pkg/cmd/migrate" "github.com/authzed/spicedb/pkg/cmd/root" + "github.com/authzed/spicedb/pkg/cmd/serve" "github.com/authzed/spicedb/pkg/cmd/version" ) @@ -25,23 +20,6 @@ const ( backendsPerKey = 1 ) -var defaultPreRunE = cobrautil.CommandStack( - cobrautil.SyncViperPreRunE("spicedb"), - cobrautil.ZeroLogPreRunE("log", zerolog.InfoLevel), - cobrautil.OpenTelemetryPreRunE("otel", zerolog.InfoLevel), -) - -func metricsHandler() http.Handler { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - return mux -} - func main() { // Set up a seed for randomness rand.Seed(time.Now().UnixNano()) @@ -50,7 +28,11 @@ func main() { kuberesolver.RegisterInCluster() // Enable consistent hashring gRPC load balancer - balancer.Register(consistentbalancer.NewConsistentHashringBuilder(xxhash.Sum64, hashringReplicationFactor, backendsPerKey)) + balancer.Register(consistentbalancer.NewConsistentHashringBuilder( + xxhash.Sum64, + hashringReplicationFactor, + backendsPerKey, + )) // Create a root command rootCmd := root.NewCommand() @@ -71,12 +53,17 @@ func main() { rootCmd.AddCommand(headCmd) // Add server commands - devSvcCmd := devsvc.NewCommand(rootCmd.Use) - devsvc.RegisterFlags(devSvcCmd) - rootCmd.AddCommand(devSvcCmd) + serveCmd := serve.NewServeCommand(rootCmd.Use) + serve.RegisterServeFlags(serveCmd) + rootCmd.AddCommand(serveCmd) + + devtoolsCmd := serve.NewDevtoolsCommand(rootCmd.Use) + serve.RegisterDevtoolsFlags(devtoolsCmd) + rootCmd.AddCommand(devtoolsCmd) - registerServeCmd(rootCmd) - registerTestserverCmd(rootCmd) + testingCmd := serve.NewTestingCommand(rootCmd.Use) + serve.RegisterTestingFlags(testingCmd) + rootCmd.AddCommand(testingCmd) _ = rootCmd.Execute() } diff --git a/cmd/spicedb/testserver_integration_test.go b/cmd/spicedb/servetesting_integration_test.go similarity index 100% rename from cmd/spicedb/testserver_integration_test.go rename to cmd/spicedb/servetesting_integration_test.go diff --git a/pkg/cmd/devsvc/devsvc.go b/pkg/cmd/serve/devtools.go similarity index 97% rename from pkg/cmd/devsvc/devsvc.go rename to pkg/cmd/serve/devtools.go index b06b890431..bbd224dad7 100644 --- a/pkg/cmd/devsvc/devsvc.go +++ b/pkg/cmd/serve/devtools.go @@ -1,4 +1,4 @@ -package devsvc +package serve import ( "context" @@ -28,7 +28,7 @@ import ( defaults "github.com/authzed/spicedb/pkg/cmd" ) -func RegisterFlags(cmd *cobra.Command) { +func RegisterDevtoolsFlags(cmd *cobra.Command) { cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true) cobrautil.RegisterHttpServerFlags(cmd.Flags(), "metrics", "metrics", ":9090", true) cobrautil.RegisterHttpServerFlags(cmd.Flags(), "http", "download", ":8443", false) @@ -42,7 +42,7 @@ func RegisterFlags(cmd *cobra.Command) { cmd.Flags().String("s3-region", "auto", "s3 region for s3 share store") } -func NewCommand(programName string) *cobra.Command { +func NewDevtoolsCommand(programName string) *cobra.Command { return &cobra.Command{ Use: "serve-devtools", Short: "runs the developer tools service", diff --git a/cmd/spicedb/serve.go b/pkg/cmd/serve/serve.go similarity index 66% rename from cmd/spicedb/serve.go rename to pkg/cmd/serve/serve.go index eb3cf9d13e..669b26dd1f 100644 --- a/cmd/spicedb/serve.go +++ b/pkg/cmd/serve/serve.go @@ -1,4 +1,4 @@ -package main +package serve import ( "context" @@ -22,6 +22,7 @@ import ( "github.com/authzed/spicedb/internal/auth" "github.com/authzed/spicedb/internal/dashboard" + "github.com/authzed/spicedb/internal/datastore/common" "github.com/authzed/spicedb/internal/datastore/proxy" combineddispatch "github.com/authzed/spicedb/internal/dispatch/combined" "github.com/authzed/spicedb/internal/gateway" @@ -29,87 +30,101 @@ import ( "github.com/authzed/spicedb/internal/namespace" "github.com/authzed/spicedb/internal/services" v1alpha1svc "github.com/authzed/spicedb/internal/services/v1alpha1" - cmdlib "github.com/authzed/spicedb/pkg/cmd" + defaults "github.com/authzed/spicedb/pkg/cmd" logmw "github.com/authzed/spicedb/pkg/middleware/logging" "github.com/authzed/spicedb/pkg/middleware/requestid" "github.com/authzed/spicedb/pkg/validationfile" ) -func registerServeCmd(rootCmd *cobra.Command) { - var datastoreOptions cmdlib.DatastoreConfig - serveCmd := &cobra.Command{ - Use: "serve", - Short: "serve the permissions database", - Long: "A database that stores, computes, and validates application permissions", - PreRunE: defaultPreRunE, - Run: func(cmd *cobra.Command, args []string) { - serveRun(cmd, args, datastoreOptions) - }, - Example: fmt.Sprintf(` %s: - spicedb serve --grpc-preshared-key "somerandomkeyhere" - - %s: - spicedb serve --grpc-preshared-key "realkeyhere" --grpc-tls-cert-path path/to/tls/cert --grpc-tls-key-path path/to/tls/key \ - --http-tls-cert-path path/to/tls/cert --http-tls-key-path path/to/tls/key \ - --datastore-engine postgres --datastore-conn-uri "postgres-connection-string-here" -`, color.YellowString("No TLS and in-memory"), color.GreenString("TLS and a real datastore")), - } - +func RegisterServeFlags(cmd *cobra.Command) { // Flags for the gRPC API server - cobrautil.RegisterGrpcServerFlags(serveCmd.Flags(), "grpc", "gRPC", ":50051", true) - serveCmd.Flags().String("grpc-preshared-key", "", "preshared key to require for authenticated requests") - serveCmd.Flags().Duration("grpc-shutdown-grace-period", 0*time.Second, "amount of time after receiving sigint to continue serving") - if err := serveCmd.MarkFlagRequired("grpc-preshared-key"); err != nil { + cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true) + cmd.Flags().String("grpc-preshared-key", "", "preshared key to require for authenticated requests") + cmd.Flags().Duration("grpc-shutdown-grace-period", 0*time.Second, "amount of time after receiving sigint to continue serving") + if err := cmd.MarkFlagRequired("grpc-preshared-key"); err != nil { panic("failed to mark flag as required: " + err.Error()) } // Flags for the datastore - cmdlib.RegisterDatastoreFlags(serveCmd, &datastoreOptions) - serveCmd.Flags().Bool("datastore-readonly", false, "set the service to read-only mode") - serveCmd.Flags().StringSlice("datastore-bootstrap-files", []string{}, "bootstrap data yaml files to load") - serveCmd.Flags().Bool("datastore-bootstrap-overwrite", false, "overwrite any existing data with bootstrap data") - - serveCmd.Flags().Bool("datastore-request-hedging", true, "enable request hedging") - serveCmd.Flags().Duration("datastore-request-hedging-initial-slow-value", 10*time.Millisecond, "initial value to use for slow datastore requests, before statistics have been collected") - serveCmd.Flags().Uint64("datastore-request-hedging-max-requests", 1_000_000, "maximum number of historical requests to consider") - serveCmd.Flags().Float64("datastore-request-hedging-quantile", 0.95, "quantile of historical datastore request time over which a request will be considered slow") + cmd.Flags().String("datastore-engine", "memory", `type of datastore to initialize ("memory", "postgres", "cockroachdb")`) + cmd.Flags().String("datastore-conn-uri", "", `connection string used by remote datastores (e.g. "postgres://postgres:password@localhost:5432/spicedb")`) + cmd.Flags().Bool("datastore-readonly", false, "set the service to read-only mode") + cmd.Flags().Int("datastore-conn-max-open", 20, "number of concurrent connections open in a remote datastore's connection pool") + cmd.Flags().Int("datastore-conn-min-open", 10, "number of minimum concurrent connections open in a remote datastore's connection pool") + cmd.Flags().Duration("datastore-conn-max-lifetime", 30*time.Minute, "maximum amount of time a connection can live in a remote datastore's connection pool") + cmd.Flags().Duration("datastore-conn-max-idletime", 30*time.Minute, "maximum amount of time a connection can idle in a remote datastore's connection pool") + cmd.Flags().Duration("datastore-conn-healthcheck-interval", 30*time.Second, "time between a remote datastore's connection pool health checks") + cmd.Flags().Duration("datastore-gc-window", 24*time.Hour, "amount of time before revisions are garbage collected") + cmd.Flags().Duration("datastore-gc-interval", 3*time.Minute, "amount of time between passes of garbage collection (postgres driver only)") + cmd.Flags().Duration("datastore-gc-max-operation-time", 1*time.Minute, "maximum amount of time a garbage collection pass can operate before timing out (postgres driver only)") + cmd.Flags().Duration("datastore-revision-fuzzing-duration", 5*time.Second, "amount of time to advertize stale revisions") + // See crdb doc for info about follower reads and how it is configured: https://www.cockroachlabs.com/docs/stable/follower-reads.html + cmd.Flags().Duration("datastore-follower-read-delay-duration", 4_800*time.Millisecond, "amount of time to subtract from non-sync revision timestamps to ensure they are sufficiently in the past to enable follower reads (cockroach driver only)") + cmd.Flags().String("datastore-query-split-size", common.DefaultSplitAtEstimatedQuerySize.String(), "estimated number of bytes at which a query is split when using a remote datastore") + cmd.Flags().StringSlice("datastore-bootstrap-files", []string{}, "bootstrap data yaml files to load") + cmd.Flags().Bool("datastore-bootstrap-overwrite", false, "overwrite any existing data with bootstrap data") + cmd.Flags().Int("datastore-max-tx-retries", 50, "number of times a retriable transaction should be retried (cockroach driver only)") + cmd.Flags().String("datastore-tx-overlap-strategy", "static", `strategy to generate transaction overlap keys ("prefix", "static", "insecure") (cockroach driver only)`) + cmd.Flags().String("datastore-tx-overlap-key", "key", "static key to touch when writing to ensure transactions overlap (only used if --datastore-tx-overlap-strategy=static is set; cockroach driver only)") + + cmd.Flags().Bool("datastore-request-hedging", true, "enable request hedging") + cmd.Flags().Duration("datastore-request-hedging-initial-slow-value", 10*time.Millisecond, "initial value to use for slow datastore requests, before statistics have been collected") + cmd.Flags().Uint64("datastore-request-hedging-max-requests", 1_000_000, "maximum number of historical requests to consider") + cmd.Flags().Float64("datastore-request-hedging-quantile", 0.95, "quantile of historical datastore request time over which a request will be considered slow") // Flags for the namespace manager - serveCmd.Flags().Duration("ns-cache-expiration", 1*time.Minute, "amount of time a namespace entry should remain cached") + cmd.Flags().Duration("ns-cache-expiration", 1*time.Minute, "amount of time a namespace entry should remain cached") // Flags for parsing and validating schemas. - serveCmd.Flags().Bool("schema-prefixes-required", false, "require prefixes on all object definitions in schemas") + cmd.Flags().Bool("schema-prefixes-required", false, "require prefixes on all object definitions in schemas") // Flags for HTTP gateway - cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "http", "http", ":8443", false) + cobrautil.RegisterHttpServerFlags(cmd.Flags(), "http", "http", ":8443", false) // Flags for configuring the dispatch server - cobrautil.RegisterGrpcServerFlags(serveCmd.Flags(), "dispatch-cluster", "dispatch", ":50053", false) + cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "dispatch-cluster", "dispatch", ":50053", false) // Flags for configuring dispatch requests - serveCmd.Flags().Uint32("dispatch-max-depth", 50, "maximum recursion depth for nested calls") - serveCmd.Flags().String("dispatch-upstream-addr", "", "upstream grpc address to dispatch to") - serveCmd.Flags().String("dispatch-upstream-ca-path", "", "local path to the TLS CA used when connecting to the dispatch cluster") + cmd.Flags().Uint32("dispatch-max-depth", 50, "maximum recursion depth for nested calls") + cmd.Flags().String("dispatch-upstream-addr", "", "upstream grpc address to dispatch to") + cmd.Flags().String("dispatch-upstream-ca-path", "", "local path to the TLS CA used when connecting to the dispatch cluster") // Flags for configuring API behavior - serveCmd.Flags().Bool("disable-v1-schema-api", false, "disables the V1 schema API") + cmd.Flags().Bool("disable-v1-schema-api", false, "disables the V1 schema API") // Flags for misc services - cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "dashboard", "dashboard", ":8080", true) - cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "metrics", "metrics", ":9090", true) + cobrautil.RegisterHttpServerFlags(cmd.Flags(), "dashboard", "dashboard", ":8080", true) + cobrautil.RegisterHttpServerFlags(cmd.Flags(), "metrics", "metrics", ":9090", true) +} - // Required flags. +func NewServeCommand(programName string) *cobra.Command { + var datastoreOptions defaults.DatastoreConfig + return &cobra.Command{ + Use: "serve", + Short: "serve the permissions database", + Long: "A database that stores, computes, and validates application permissions", + PreRunE: defaults.DefaultPreRunE(programName), + Run: func(cmd *cobra.Command, args []string) { + serveRun(cmd, args, datastoreOptions) + }, + Example: fmt.Sprintf(` %s: + spicedb serve --grpc-preshared-key "somerandomkeyhere" - rootCmd.AddCommand(serveCmd) + %s: + spicedb serve --grpc-preshared-key "realkeyhere" --grpc-tls-cert-path path/to/tls/cert --grpc-tls-key-path path/to/tls/key \ + --http-tls-cert-path path/to/tls/cert --http-tls-key-path path/to/tls/key \ + --datastore-engine postgres --datastore-conn-uri "postgres-connection-string-here" +`, color.YellowString("No TLS and in-memory"), color.GreenString("TLS and a real datastore")), + } } -func serveRun(cmd *cobra.Command, args []string, datastoreOpts cmdlib.DatastoreConfig) { +func serveRun(cmd *cobra.Command, args []string, datastoreOpts defaults.DatastoreConfig) { token := cobrautil.MustGetStringExpanded(cmd, "grpc-preshared-key") if len(token) < 1 { log.Fatal().Msg("a preshared key must be provided via --grpc-preshared-key to authenticate API requests") } - ds, err := cmdlib.NewDatastore(datastoreOpts.ToOption()) + ds, err := defaults.NewDatastore(datastoreOpts.ToOption()) if err != nil { log.Fatal().Err(err).Msg("failed to init datastore") } @@ -259,7 +274,7 @@ func serveRun(cmd *cobra.Command, args []string, datastoreOpts cmdlib.DatastoreC // Start the metrics endpoint. metricsSrv := cobrautil.HttpServerFromFlags(cmd, "metrics") - metricsSrv.Handler = metricsHandler() + metricsSrv.Handler = defaults.MetricsHandler() go func() { if err := cobrautil.HttpListenFromFlags(cmd, "metrics", metricsSrv, zerolog.InfoLevel); err != nil { log.Fatal().Err(err).Msg("failed while serving metrics") diff --git a/cmd/spicedb/testserver.go b/pkg/cmd/serve/testing.go similarity index 95% rename from cmd/spicedb/testserver.go rename to pkg/cmd/serve/testing.go index bea4888fb9..ece6329ae9 100644 --- a/cmd/spicedb/testserver.go +++ b/pkg/cmd/serve/testing.go @@ -1,4 +1,4 @@ -package main +package serve import ( "context" @@ -33,6 +33,7 @@ import ( "github.com/authzed/spicedb/internal/namespace" "github.com/authzed/spicedb/internal/services" v1alpha1svc "github.com/authzed/spicedb/internal/services/v1alpha1" + defaults "github.com/authzed/spicedb/pkg/cmd" "github.com/authzed/spicedb/pkg/validationfile" ) @@ -43,20 +44,20 @@ const ( revisionFuzzingDuration = 10 * time.Millisecond ) -func registerTestserverCmd(rootCmd *cobra.Command) { - testserveCmd := &cobra.Command{ +func RegisterTestingFlags(cmd *cobra.Command) { + cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true) + cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "readonly-grpc", "read-only gRPC", ":50052", true) + cmd.Flags().StringSlice("load-configs", []string{}, "configuration yaml files to load") +} + +func NewTestingCommand(programName string) *cobra.Command { + return &cobra.Command{ Use: "serve-testing", Short: "test server with an in-memory datastore", Long: "An in-memory spicedb server which serves completely isolated datastores per client-supplied auth token used.", - PreRunE: defaultPreRunE, + PreRunE: defaults.DefaultPreRunE(programName), Run: runTestServer, } - - cobrautil.RegisterGrpcServerFlags(testserveCmd.Flags(), "grpc", "gRPC", ":50051", true) - cobrautil.RegisterGrpcServerFlags(testserveCmd.Flags(), "readonly-grpc", "read-only gRPC", ":50052", true) - testserveCmd.Flags().StringSlice("load-configs", []string{}, "configuration yaml files to load") - - rootCmd.AddCommand(testserveCmd) } func runTestServer(cmd *cobra.Command, args []string) {