Skip to content

Commit

Permalink
pkg/cmd: add serve package
Browse files Browse the repository at this point in the history
  • Loading branch information
@jzelinskie
jzelinskie committed Dec 7, 2021
1 parent e92e70e commit 7a0eee8
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 101 deletions.
45 changes: 16 additions & 29 deletions cmd/spicedb/main.go
View file
Expand Up @@ -2,21 +2,16 @@ package main

import (
"math/rand"
"net/http"
"net/http/pprof"
"time"

"github.com/cespare/xxhash"
"github.com/jzelinskie/cobrautil"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/zerolog"
"github.com/sercand/kuberesolver/v3"
"google.golang.org/grpc/balancer"

consistentbalancer "github.com/authzed/spicedb/pkg/balancer"
"github.com/authzed/spicedb/pkg/cmd/devsvc"
"github.com/authzed/spicedb/pkg/cmd/migrate"
"github.com/authzed/spicedb/pkg/cmd/root"
"github.com/authzed/spicedb/pkg/cmd/serve"
"github.com/authzed/spicedb/pkg/cmd/version"
)

Expand All @@ -25,23 +20,6 @@ const (
backendsPerKey = 1
)

var defaultPreRunE = cobrautil.CommandStack(
cobrautil.SyncViperPreRunE("spicedb"),
cobrautil.ZeroLogPreRunE("log", zerolog.InfoLevel),
cobrautil.OpenTelemetryPreRunE("otel", zerolog.InfoLevel),
)

func metricsHandler() http.Handler {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
return mux
}

func main() {
// Set up a seed for randomness
rand.Seed(time.Now().UnixNano())
Expand All @@ -50,7 +28,11 @@ func main() {
kuberesolver.RegisterInCluster()

// Enable consistent hashring gRPC load balancer
balancer.Register(consistentbalancer.NewConsistentHashringBuilder(xxhash.Sum64, hashringReplicationFactor, backendsPerKey))
balancer.Register(consistentbalancer.NewConsistentHashringBuilder(
xxhash.Sum64,
hashringReplicationFactor,
backendsPerKey,
))

// Create a root command
rootCmd := root.NewCommand()
Expand All @@ -71,12 +53,17 @@ func main() {
rootCmd.AddCommand(headCmd)

// Add server commands
devSvcCmd := devsvc.NewCommand(rootCmd.Use)
devsvc.RegisterFlags(devSvcCmd)
rootCmd.AddCommand(devSvcCmd)
serveCmd := serve.NewServeCommand(rootCmd.Use)
serve.RegisterServeFlags(serveCmd)
rootCmd.AddCommand(serveCmd)

devtoolsCmd := serve.NewDevtoolsCommand(rootCmd.Use)
serve.RegisterDevtoolsFlags(devtoolsCmd)
rootCmd.AddCommand(devtoolsCmd)

registerServeCmd(rootCmd)
registerTestserverCmd(rootCmd)
testingCmd := serve.NewTestingCommand(rootCmd.Use)
serve.RegisterTestingFlags(testingCmd)
rootCmd.AddCommand(testingCmd)

_ = rootCmd.Execute()
}
0 cmd/spicedb/testserver_integration_test.go → cmd/spicedb/servetesting_integration_test.go
View file
File renamed without changes.
6 changes: 3 additions & 3 deletions pkg/cmd/devsvc/devsvc.go → pkg/cmd/serve/devtools.go
View file
@@ -1,4 +1,4 @@
package devsvc
package serve

import (
"context"
Expand Down Expand Up @@ -28,7 +28,7 @@ import (
defaults "github.com/authzed/spicedb/pkg/cmd"
)

func RegisterFlags(cmd *cobra.Command) {
func RegisterDevtoolsFlags(cmd *cobra.Command) {
cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true)
cobrautil.RegisterHttpServerFlags(cmd.Flags(), "metrics", "metrics", ":9090", true)
cobrautil.RegisterHttpServerFlags(cmd.Flags(), "http", "download", ":8443", false)
Expand All @@ -42,7 +42,7 @@ func RegisterFlags(cmd *cobra.Command) {
cmd.Flags().String("s3-region", "auto", "s3 region for s3 share store")
}

func NewCommand(programName string) *cobra.Command {
func NewDevtoolsCommand(programName string) *cobra.Command {
return &cobra.Command{
Use: "serve-devtools",
Short: "runs the developer tools service",
Expand Down
117 changes: 58 additions & 59 deletions cmd/spicedb/serve.go → pkg/cmd/serve/serve.go
View file
@@ -1,4 +1,4 @@
package main
package serve

import (
"context"
Expand Down Expand Up @@ -35,90 +35,89 @@ import (
"github.com/authzed/spicedb/internal/namespace"
"github.com/authzed/spicedb/internal/services"
v1alpha1svc "github.com/authzed/spicedb/internal/services/v1alpha1"
defaults "github.com/authzed/spicedb/pkg/cmd"
logmw "github.com/authzed/spicedb/pkg/middleware/logging"
"github.com/authzed/spicedb/pkg/middleware/requestid"
"github.com/authzed/spicedb/pkg/validationfile"
)

func registerServeCmd(rootCmd *cobra.Command) {
serveCmd := &cobra.Command{
Use: "serve",
Short: "serve the permissions database",
Long: "A database that stores, computes, and validates application permissions",
PreRunE: defaultPreRunE,
Run: serveRun,
Example: fmt.Sprintf(` %s:
spicedb serve --grpc-preshared-key "somerandomkeyhere"
%s:
spicedb serve --grpc-preshared-key "realkeyhere" --grpc-tls-cert-path path/to/tls/cert --grpc-tls-key-path path/to/tls/key \
--http-tls-cert-path path/to/tls/cert --http-tls-key-path path/to/tls/key \
--datastore-engine postgres --datastore-conn-uri "postgres-connection-string-here"
`, color.YellowString("No TLS and in-memory"), color.GreenString("TLS and a real datastore")),
}

func RegisterServeFlags(cmd *cobra.Command) {
// Flags for the gRPC API server
cobrautil.RegisterGrpcServerFlags(serveCmd.Flags(), "grpc", "gRPC", ":50051", true)
serveCmd.Flags().String("grpc-preshared-key", "", "preshared key to require for authenticated requests")
serveCmd.Flags().Duration("grpc-shutdown-grace-period", 0*time.Second, "amount of time after receiving sigint to continue serving")
if err := serveCmd.MarkFlagRequired("grpc-preshared-key"); err != nil {
cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true)
cmd.Flags().String("grpc-preshared-key", "", "preshared key to require for authenticated requests")
cmd.Flags().Duration("grpc-shutdown-grace-period", 0*time.Second, "amount of time after receiving sigint to continue serving")
if err := cmd.MarkFlagRequired("grpc-preshared-key"); err != nil {
panic("failed to mark flag as required: " + err.Error())
}

// Flags for the datastore
serveCmd.Flags().String("datastore-engine", "memory", `type of datastore to initialize ("memory", "postgres", "cockroachdb")`)
serveCmd.Flags().String("datastore-conn-uri", "", `connection string used by remote datastores (e.g. "postgres://postgres:password@localhost:5432/spicedb")`)
serveCmd.Flags().Bool("datastore-readonly", false, "set the service to read-only mode")
serveCmd.Flags().Int("datastore-conn-max-open", 20, "number of concurrent connections open in a remote datastore's connection pool")
serveCmd.Flags().Int("datastore-conn-min-open", 10, "number of minimum concurrent connections open in a remote datastore's connection pool")
serveCmd.Flags().Duration("datastore-conn-max-lifetime", 30*time.Minute, "maximum amount of time a connection can live in a remote datastore's connection pool")
serveCmd.Flags().Duration("datastore-conn-max-idletime", 30*time.Minute, "maximum amount of time a connection can idle in a remote datastore's connection pool")
serveCmd.Flags().Duration("datastore-conn-healthcheck-interval", 30*time.Second, "time between a remote datastore's connection pool health checks")
serveCmd.Flags().Duration("datastore-gc-window", 24*time.Hour, "amount of time before revisions are garbage collected")
serveCmd.Flags().Duration("datastore-gc-interval", 3*time.Minute, "amount of time between passes of garbage collection (postgres driver only)")
serveCmd.Flags().Duration("datastore-gc-max-operation-time", 1*time.Minute, "maximum amount of time a garbage collection pass can operate before timing out (postgres driver only)")
serveCmd.Flags().Duration("datastore-revision-fuzzing-duration", 5*time.Second, "amount of time to advertize stale revisions")
cmd.Flags().String("datastore-engine", "memory", `type of datastore to initialize ("memory", "postgres", "cockroachdb")`)
cmd.Flags().String("datastore-conn-uri", "", `connection string used by remote datastores (e.g. "postgres://postgres:password@localhost:5432/spicedb")`)
cmd.Flags().Bool("datastore-readonly", false, "set the service to read-only mode")
cmd.Flags().Int("datastore-conn-max-open", 20, "number of concurrent connections open in a remote datastore's connection pool")
cmd.Flags().Int("datastore-conn-min-open", 10, "number of minimum concurrent connections open in a remote datastore's connection pool")
cmd.Flags().Duration("datastore-conn-max-lifetime", 30*time.Minute, "maximum amount of time a connection can live in a remote datastore's connection pool")
cmd.Flags().Duration("datastore-conn-max-idletime", 30*time.Minute, "maximum amount of time a connection can idle in a remote datastore's connection pool")
cmd.Flags().Duration("datastore-conn-healthcheck-interval", 30*time.Second, "time between a remote datastore's connection pool health checks")
cmd.Flags().Duration("datastore-gc-window", 24*time.Hour, "amount of time before revisions are garbage collected")
cmd.Flags().Duration("datastore-gc-interval", 3*time.Minute, "amount of time between passes of garbage collection (postgres driver only)")
cmd.Flags().Duration("datastore-gc-max-operation-time", 1*time.Minute, "maximum amount of time a garbage collection pass can operate before timing out (postgres driver only)")
cmd.Flags().Duration("datastore-revision-fuzzing-duration", 5*time.Second, "amount of time to advertize stale revisions")
// See crdb doc for info about follower reads and how it is configured: https://www.cockroachlabs.com/docs/stable/follower-reads.html
serveCmd.Flags().Duration("datastore-follower-read-delay-duration", 4_800*time.Millisecond, "amount of time to subtract from non-sync revision timestamps to ensure they are sufficiently in the past to enable follower reads (cockroach driver only)")
serveCmd.Flags().String("datastore-query-split-size", common.DefaultSplitAtEstimatedQuerySize.String(), "estimated number of bytes at which a query is split when using a remote datastore")
serveCmd.Flags().StringSlice("datastore-bootstrap-files", []string{}, "bootstrap data yaml files to load")
serveCmd.Flags().Bool("datastore-bootstrap-overwrite", false, "overwrite any existing data with bootstrap data")
serveCmd.Flags().Int("datastore-max-tx-retries", 50, "number of times a retriable transaction should be retried (cockroach driver only)")
serveCmd.Flags().String("datastore-tx-overlap-strategy", "static", `strategy to generate transaction overlap keys ("prefix", "static", "insecure") (cockroach driver only)`)
serveCmd.Flags().String("datastore-tx-overlap-key", "key", "static key to touch when writing to ensure transactions overlap (only used if --datastore-tx-overlap-strategy=static is set; cockroach driver only)")

serveCmd.Flags().Bool("datastore-request-hedging", true, "enable request hedging")
serveCmd.Flags().Duration("datastore-request-hedging-initial-slow-value", 10*time.Millisecond, "initial value to use for slow datastore requests, before statistics have been collected")
serveCmd.Flags().Uint64("datastore-request-hedging-max-requests", 1_000_000, "maximum number of historical requests to consider")
serveCmd.Flags().Float64("datastore-request-hedging-quantile", 0.95, "quantile of historical datastore request time over which a request will be considered slow")
cmd.Flags().Duration("datastore-follower-read-delay-duration", 4_800*time.Millisecond, "amount of time to subtract from non-sync revision timestamps to ensure they are sufficiently in the past to enable follower reads (cockroach driver only)")
cmd.Flags().String("datastore-query-split-size", common.DefaultSplitAtEstimatedQuerySize.String(), "estimated number of bytes at which a query is split when using a remote datastore")
cmd.Flags().StringSlice("datastore-bootstrap-files", []string{}, "bootstrap data yaml files to load")
cmd.Flags().Bool("datastore-bootstrap-overwrite", false, "overwrite any existing data with bootstrap data")
cmd.Flags().Int("datastore-max-tx-retries", 50, "number of times a retriable transaction should be retried (cockroach driver only)")
cmd.Flags().String("datastore-tx-overlap-strategy", "static", `strategy to generate transaction overlap keys ("prefix", "static", "insecure") (cockroach driver only)`)
cmd.Flags().String("datastore-tx-overlap-key", "key", "static key to touch when writing to ensure transactions overlap (only used if --datastore-tx-overlap-strategy=static is set; cockroach driver only)")

cmd.Flags().Bool("datastore-request-hedging", true, "enable request hedging")
cmd.Flags().Duration("datastore-request-hedging-initial-slow-value", 10*time.Millisecond, "initial value to use for slow datastore requests, before statistics have been collected")
cmd.Flags().Uint64("datastore-request-hedging-max-requests", 1_000_000, "maximum number of historical requests to consider")
cmd.Flags().Float64("datastore-request-hedging-quantile", 0.95, "quantile of historical datastore request time over which a request will be considered slow")

// Flags for the namespace manager
serveCmd.Flags().Duration("ns-cache-expiration", 1*time.Minute, "amount of time a namespace entry should remain cached")
cmd.Flags().Duration("ns-cache-expiration", 1*time.Minute, "amount of time a namespace entry should remain cached")

// Flags for parsing and validating schemas.
serveCmd.Flags().Bool("schema-prefixes-required", false, "require prefixes on all object definitions in schemas")
cmd.Flags().Bool("schema-prefixes-required", false, "require prefixes on all object definitions in schemas")

// Flags for HTTP gateway
cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "http", "http", ":8443", false)
cobrautil.RegisterHttpServerFlags(cmd.Flags(), "http", "http", ":8443", false)

// Flags for configuring the dispatch server
cobrautil.RegisterGrpcServerFlags(serveCmd.Flags(), "dispatch-cluster", "dispatch", ":50053", false)
cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "dispatch-cluster", "dispatch", ":50053", false)

// Flags for configuring dispatch requests
serveCmd.Flags().Uint32("dispatch-max-depth", 50, "maximum recursion depth for nested calls")
serveCmd.Flags().String("dispatch-upstream-addr", "", "upstream grpc address to dispatch to")
serveCmd.Flags().String("dispatch-upstream-ca-path", "", "local path to the TLS CA used when connecting to the dispatch cluster")
cmd.Flags().Uint32("dispatch-max-depth", 50, "maximum recursion depth for nested calls")
cmd.Flags().String("dispatch-upstream-addr", "", "upstream grpc address to dispatch to")
cmd.Flags().String("dispatch-upstream-ca-path", "", "local path to the TLS CA used when connecting to the dispatch cluster")

// Flags for configuring API behavior
serveCmd.Flags().Bool("disable-v1-schema-api", false, "disables the V1 schema API")
cmd.Flags().Bool("disable-v1-schema-api", false, "disables the V1 schema API")

// Flags for misc services
cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "dashboard", "dashboard", ":8080", true)
cobrautil.RegisterHttpServerFlags(serveCmd.Flags(), "metrics", "metrics", ":9090", true)
cobrautil.RegisterHttpServerFlags(cmd.Flags(), "dashboard", "dashboard", ":8080", true)
cobrautil.RegisterHttpServerFlags(cmd.Flags(), "metrics", "metrics", ":9090", true)
}

// Required flags.
func NewServeCommand(programName string) *cobra.Command {
return &cobra.Command{
Use: "serve",
Short: "serve the permissions database",
Long: "A database that stores, computes, and validates application permissions",
PreRunE: defaults.DefaultPreRunE(programName),
Run: serveRun,
Example: fmt.Sprintf(` %s:
spicedb serve --grpc-preshared-key "somerandomkeyhere"
rootCmd.AddCommand(serveCmd)
%s:
spicedb serve --grpc-preshared-key "realkeyhere" --grpc-tls-cert-path path/to/tls/cert --grpc-tls-key-path path/to/tls/key \
--http-tls-cert-path path/to/tls/cert --http-tls-key-path path/to/tls/key \
--datastore-engine postgres --datastore-conn-uri "postgres-connection-string-here"
`, color.YellowString("No TLS and in-memory"), color.GreenString("TLS and a real datastore")),
}
}

func serveRun(cmd *cobra.Command, args []string) {
Expand Down Expand Up @@ -337,7 +336,7 @@ func serveRun(cmd *cobra.Command, args []string) {

// Start the metrics endpoint.
metricsSrv := cobrautil.HttpServerFromFlags(cmd, "metrics")
metricsSrv.Handler = metricsHandler()
metricsSrv.Handler = defaults.MetricsHandler()
go func() {
if err := cobrautil.HttpListenFromFlags(cmd, "metrics", metricsSrv, zerolog.InfoLevel); err != nil {
log.Fatal().Err(err).Msg("failed while serving metrics")
Expand Down
21 changes: 11 additions & 10 deletions cmd/spicedb/testserver.go → pkg/cmd/serve/testing.go
View file
@@ -1,4 +1,4 @@
package main
package serve

import (
"context"
Expand Down Expand Up @@ -33,6 +33,7 @@ import (
"github.com/authzed/spicedb/internal/namespace"
"github.com/authzed/spicedb/internal/services"
v1alpha1svc "github.com/authzed/spicedb/internal/services/v1alpha1"
defaults "github.com/authzed/spicedb/pkg/cmd"
"github.com/authzed/spicedb/pkg/validationfile"
)

Expand All @@ -43,20 +44,20 @@ const (
revisionFuzzingDuration = 10 * time.Millisecond
)

func registerTestserverCmd(rootCmd *cobra.Command) {
testserveCmd := &cobra.Command{
func RegisterTestingFlags(cmd *cobra.Command) {
cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "grpc", "gRPC", ":50051", true)
cobrautil.RegisterGrpcServerFlags(cmd.Flags(), "readonly-grpc", "read-only gRPC", ":50052", true)
cmd.Flags().StringSlice("load-configs", []string{}, "configuration yaml files to load")
}

func NewTestingCommand(programName string) *cobra.Command {
return &cobra.Command{
Use: "serve-testing",
Short: "test server with an in-memory datastore",
Long: "An in-memory spicedb server which serves completely isolated datastores per client-supplied auth token used.",
PreRunE: defaultPreRunE,
PreRunE: defaults.DefaultPreRunE(programName),
Run: runTestServer,
}

cobrautil.RegisterGrpcServerFlags(testserveCmd.Flags(), "grpc", "gRPC", ":50051", true)
cobrautil.RegisterGrpcServerFlags(testserveCmd.Flags(), "readonly-grpc", "read-only gRPC", ":50052", true)
testserveCmd.Flags().StringSlice("load-configs", []string{}, "configuration yaml files to load")

rootCmd.AddCommand(testserveCmd)
}

func runTestServer(cmd *cobra.Command, args []string) {
Expand Down

0 comments on commit 7a0eee8

Please sign in to comment.