From 3bbcb1c432257b3a0086d2d84b3fa0d89b64a12d Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Tue, 7 Dec 2021 10:02:44 -0500 Subject: [PATCH] support both config object (for cobra bindings) and functional options for datastore --- cmd/spicedb/serve.go | 31 +--- go.mod | 1 + go.sum | 6 + internal/datastore/crdb/crdb.go | 28 ---- internal/datastore/datastore.go | 3 - internal/datastore/memdb/memdb.go | 13 -- internal/datastore/postgres/postgres.go | 26 ---- pkg/cmd/serve/datastore.go | 195 +++++++++++------------- pkg/cmd/serve/datastore_options.go | 135 ++++++++++++++++ tools.go | 3 + 10 files changed, 238 insertions(+), 203 deletions(-) create mode 100644 pkg/cmd/serve/datastore_options.go create mode 100644 tools.go diff --git a/cmd/spicedb/serve.go b/cmd/spicedb/serve.go index fe1d32810f..3012bfdb9c 100644 --- a/cmd/spicedb/serve.go +++ b/cmd/spicedb/serve.go @@ -22,7 +22,6 @@ import ( "github.com/authzed/spicedb/internal/auth" "github.com/authzed/spicedb/internal/dashboard" - "github.com/authzed/spicedb/internal/datastore" "github.com/authzed/spicedb/internal/datastore/proxy" combineddispatch "github.com/authzed/spicedb/internal/dispatch/combined" "github.com/authzed/spicedb/internal/gateway" @@ -37,12 +36,15 @@ import ( ) func registerServeCmd(rootCmd *cobra.Command) { + var datastoreOptions serve.DatastoreConfig serveCmd := &cobra.Command{ Use: "serve", Short: "serve the permissions database", Long: "A database that stores, computes, and validates application permissions", PreRunE: defaultPreRunE, - Run: serveRun, + Run: func(cmd *cobra.Command, args []string) { + serveRun(cmd, args, datastoreOptions) + }, Example: fmt.Sprintf(` %s: spicedb serve --grpc-preshared-key "somerandomkeyhere" @@ -62,7 +64,6 @@ func registerServeCmd(rootCmd *cobra.Command) { } // Flags for the datastore - var datastoreOptions serve.Options serve.RegisterDatastoreFlags(serveCmd, &datastoreOptions) serveCmd.Flags().Bool("datastore-readonly", false, "set the service to read-only mode") serveCmd.Flags().StringSlice("datastore-bootstrap-files", []string{}, "bootstrap data yaml files to load") @@ -102,31 +103,13 @@ func registerServeCmd(rootCmd *cobra.Command) { rootCmd.AddCommand(serveCmd) } -func serveRun(cmd *cobra.Command, args []string) { +func serveRun(cmd *cobra.Command, args []string, datastoreOpts serve.DatastoreConfig) { token := cobrautil.MustGetStringExpanded(cmd, "grpc-preshared-key") if len(token) < 1 { log.Fatal().Msg("a preshared key must be provided via --grpc-preshared-key to authenticate API requests") } - datastoreEngine := cobrautil.MustGetStringExpanded(cmd, "datastore-engine") - ds, err := serve.NewDatastore( - datastore.Engine(datastoreEngine), - serve.WithRevisionQuantization(cobrautil.MustGetDuration(cmd, "datastore-revision-fuzzing-duration")), - serve.WithGCWindow(cobrautil.MustGetDuration(cmd, "datastore-gc-window")), - serve.WithURI(cobrautil.MustGetStringExpanded(cmd, "datastore-conn-uri")), - serve.WithMaxIdleTime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-idletime")), - serve.WithMaxLifetime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-lifetime")), - serve.WithMaxOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-max-open")), - serve.WithMinOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-min-open")), - serve.WithSplitQuerySize(cobrautil.MustGetStringExpanded(cmd, "datastore-query-split-size")), - serve.WithFollowerReadDelay(cobrautil.MustGetDuration(cmd, "datastore-follower-read-delay-duration")), - serve.WithMaxRetries(cobrautil.MustGetInt(cmd, "datastore-max-tx-retries")), - serve.WithOverlapKey(cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-key")), - serve.WithOverlapStrategy(cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-strategy")), - serve.WithHealthCheckPeriod(cobrautil.MustGetDuration(cmd, "datastore-conn-healthcheck-interval")), - serve.WithGCInterval(cobrautil.MustGetDuration(cmd, "datastore-gc-interval")), - serve.WithGCMaxOperationTime(cobrautil.MustGetDuration(cmd, "datastore-gc-max-operation-time")), - ) + ds, err := serve.NewDatastore(datastoreOpts.ToOption()) if err != nil { log.Fatal().Err(err).Msg("failed to init datastore") } @@ -288,7 +271,7 @@ func serveRun(cmd *cobra.Command, args []string) { dashboardSrv.Handler = dashboard.NewHandler( cobrautil.MustGetStringExpanded(cmd, "grpc-addr"), cobrautil.MustGetStringExpanded(cmd, "grpc-tls-cert-path") != "" && cobrautil.MustGetStringExpanded(cmd, "grpc-tls-key-path") != "", - datastoreEngine, + datastoreOpts.Engine, ds, ) go func() { diff --git a/go.mod b/go.mod index bc0a0e6db2..105b65ec15 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/containerd/continuity v0.2.1 // indirect github.com/dgraph-io/ristretto v0.1.0 github.com/docker/docker v20.10.9+incompatible // indirect + github.com/ecordell/optgen v0.0.4 github.com/emirpasic/gods v1.12.0 github.com/envoyproxy/protoc-gen-validate v0.6.2 github.com/fatih/color v1.13.0 diff --git a/go.sum b/go.sum index 243133dcd8..3e1ac30845 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/dave/jennifer v1.4.0 h1:tNJFJmLDVTLu+v05mVZ88RINa3vQqnyyWkTKWYz0CwE= +github.com/dave/jennifer v1.4.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -154,6 +156,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/ecordell/optgen v0.0.4 h1:0ytRxj6f/LBl7laGDxiUjjR+ioAqr4WMLpS84D8AD10= +github.com/ecordell/optgen v0.0.4/go.mod h1:bAPkLVWcBlTX5EkXW0UTPRj3+yjq2I6VLgH8OasuQEM= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -767,6 +771,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -999,6 +1004,7 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= diff --git a/internal/datastore/crdb/crdb.go b/internal/datastore/crdb/crdb.go index 42e91a84c3..6334c57b2c 100644 --- a/internal/datastore/crdb/crdb.go +++ b/internal/datastore/crdb/crdb.go @@ -18,7 +18,6 @@ import ( "github.com/authzed/spicedb/internal/datastore" "github.com/authzed/spicedb/internal/datastore/crdb/migrations" - "github.com/authzed/spicedb/pkg/cmd/serve" ) var ( @@ -50,35 +49,8 @@ const ( querySelectNow = "SELECT cluster_logical_timestamp()" queryReturningTimestamp = "RETURNING cluster_logical_timestamp()" queryShowZoneConfig = "SHOW ZONE CONFIGURATION FOR RANGE default;" - - cockroachEngine datastore.Engine = "cockroach" ) -func init() { - serve.RegisterEngine(cockroachEngine, newFromDatastoreOptions) -} - -func newFromDatastoreOptions(opts serve.Options) (datastore.Datastore, error) { - splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize) - if err != nil { - return nil, fmt.Errorf("failed to parse split query size: %w", err) - } - return NewCRDBDatastore( - opts.URI, - GCWindow(opts.GCWindow), - RevisionQuantization(opts.RevisionQuantization), - ConnMaxIdleTime(opts.MaxIdleTime), - ConnMaxLifetime(opts.MaxLifetime), - MaxOpenConns(opts.MaxOpenConns), - MinOpenConns(opts.MinOpenConns), - SplitAtEstimatedQuerySize(splitQuerySize), - FollowerReadDelay(opts.FollowerReadDelay), - MaxRetries(opts.MaxRetries), - OverlapKey(opts.OverlapKey), - OverlapStrategy(opts.OverlapStrategy), - ) -} - // NewCRDBDatastore initializes a SpiceDB datastore that uses a CockroachDB // database while leveraging its AOST functionality. func NewCRDBDatastore(url string, options ...Option) (datastore.Datastore, error) { diff --git a/internal/datastore/datastore.go b/internal/datastore/datastore.go index b47541280b..80196ed7ae 100644 --- a/internal/datastore/datastore.go +++ b/internal/datastore/datastore.go @@ -150,6 +150,3 @@ type Revision = decimal.Decimal // revision type in the future a bit easier if necessary. Implementations // should use any time they want to signal an empty/error revision. var NoRevision Revision - -// Engine represents the type of a datastore engine, i.e. Cockroach or Postgres -type Engine string diff --git a/internal/datastore/memdb/memdb.go b/internal/datastore/memdb/memdb.go index 530ea7c688..0c258895c1 100644 --- a/internal/datastore/memdb/memdb.go +++ b/internal/datastore/memdb/memdb.go @@ -11,24 +11,11 @@ import ( v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" "github.com/hashicorp/go-memdb" "github.com/jzelinskie/stringz" - "github.com/rs/zerolog/log" "github.com/shopspring/decimal" "github.com/authzed/spicedb/internal/datastore" - "github.com/authzed/spicedb/pkg/cmd/serve" ) -const memoryEngine datastore.Engine = "memory" - -func init() { - serve.RegisterEngine(memoryEngine, newFromDatastoreOptions) -} - -func newFromDatastoreOptions(opts serve.Options) (datastore.Datastore, error) { - log.Warn().Msg("in-memory datastore is not persistent and not feasible to run in a high availability fashion") - return NewMemdbDatastore(0, opts.RevisionQuantization, opts.GCWindow, 0) -} - // DisableGC is a convenient constant for setting the garbage collection // interval high enough that it will never run. const DisableGC = time.Duration(math.MaxInt64) diff --git a/internal/datastore/postgres/postgres.go b/internal/datastore/postgres/postgres.go index 3f82eea94e..de2da21ed0 100644 --- a/internal/datastore/postgres/postgres.go +++ b/internal/datastore/postgres/postgres.go @@ -24,7 +24,6 @@ import ( "github.com/authzed/spicedb/internal/datastore" "github.com/authzed/spicedb/internal/datastore/postgres/migrations" - "github.com/authzed/spicedb/pkg/cmd/serve" ) const ( @@ -56,8 +55,6 @@ const ( tracingDriverName = "postgres-tracing" batchDeleteSize = 1000 - - postgresEngine datastore.Engine = "postgres" ) var ( @@ -86,29 +83,6 @@ var ( func init() { dbsql.Register(tracingDriverName, sqlmw.Driver(stdlib.GetDefaultDriver(), new(traceInterceptor))) - serve.RegisterEngine(postgresEngine, newFromDatastoreOptions) -} - -func newFromDatastoreOptions(opts serve.Options) (datastore.Datastore, error) { - splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize) - if err != nil { - return nil, fmt.Errorf("failed to parse split query size: %w", err) - } - return NewPostgresDatastore( - opts.URI, - GCWindow(opts.GCWindow), - RevisionFuzzingTimedelta(opts.RevisionQuantization), - ConnMaxIdleTime(opts.MaxIdleTime), - ConnMaxLifetime(opts.MaxLifetime), - MaxOpenConns(opts.MaxOpenConns), - MinOpenConns(opts.MinOpenConns), - SplitAtEstimatedQuerySize(splitQuerySize), - HealthCheckPeriod(opts.HealthCheckPeriod), - GCInterval(opts.GCInterval), - GCMaxOperationTime(opts.GCMaxOperationTime), - EnablePrometheusStats(), - EnableTracing(), - ) } var ( diff --git a/pkg/cmd/serve/datastore.go b/pkg/cmd/serve/datastore.go index c235d12fee..033fcd31f1 100644 --- a/pkg/cmd/serve/datastore.go +++ b/pkg/cmd/serve/datastore.go @@ -4,28 +4,30 @@ import ( "fmt" "time" + "github.com/alecthomas/units" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/authzed/spicedb/internal/datastore" "github.com/authzed/spicedb/internal/datastore/common" + "github.com/authzed/spicedb/internal/datastore/crdb" + "github.com/authzed/spicedb/internal/datastore/memdb" + "github.com/authzed/spicedb/internal/datastore/postgres" ) -type engineBuilderFunc func(options Options) (datastore.Datastore, error) +type engineBuilderFunc func(options DatastoreConfig) (datastore.Datastore, error) -func RegisterEngine(key datastore.Engine, builder engineBuilderFunc) { - if builderForEngine == nil { - builderForEngine = make(map[datastore.Engine]engineBuilderFunc) - } - if _, ok := builderForEngine[key]; ok { - panic("cannot register two datastore engines with the same name: " + key) - } - builderForEngine[key] = builder +var builderForEngine = map[string]engineBuilderFunc{ + "cockroach": newCRDBDatastore, + "postgres": newPostgresDatastore, + "memory": newMemoryDatstore, } -var builderForEngine map[datastore.Engine]engineBuilderFunc +type Option func(*DatastoreConfig) -type Options struct { +//go:generate go run github.com/ecordell/optgen -output datastore_options.go . DatastoreConfig +type DatastoreConfig struct { + Engine string URI string GCWindow time.Duration RevisionQuantization time.Duration @@ -48,9 +50,29 @@ type Options struct { GCMaxOperationTime time.Duration } +func (o *DatastoreConfig) ToOption() Option { + return func(to *DatastoreConfig) { + to.Engine = o.Engine + to.URI = o.URI + to.GCWindow = o.GCWindow + to.RevisionQuantization = o.RevisionQuantization + to.MaxIdleTime = o.MaxIdleTime + to.MaxOpenConns = o.MaxOpenConns + to.MinOpenConns = o.MinOpenConns + to.SplitQuerySize = o.SplitQuerySize + to.FollowerReadDelay = o.FollowerReadDelay + to.MaxRetries = o.MaxRetries + to.OverlapKey = o.OverlapKey + to.OverlapStrategy = o.OverlapStrategy + to.HealthCheckPeriod = o.HealthCheckPeriod + to.GCInterval = o.GCInterval + to.GCMaxOperationTime = o.GCMaxOperationTime + } +} + // RegisterDatastoreFlags adds datastore flags to a cobra command -func RegisterDatastoreFlags(cmd *cobra.Command, opts *Options) { - cmd.Flags().String("datastore-engine", "memory", `type of datastore to initialize ("memory", "postgres", "cockroachdb")`) +func RegisterDatastoreFlags(cmd *cobra.Command, opts *DatastoreConfig) { + cmd.Flags().StringVar(&opts.Engine, "datastore-engine", "memory", `type of datastore to initialize ("memory", "postgres", "cockroachdb")`) cmd.Flags().StringVar(&opts.URI, "datastore-conn-uri", "", `connection string used by remote datastores (e.g. "postgres://postgres:password@localhost:5432/spicedb")`) cmd.Flags().IntVar(&opts.MaxOpenConns, "datastore-conn-max-open", 20, "number of concurrent connections open in a remote datastore's connection pool") cmd.Flags().IntVar(&opts.MinOpenConns, "datastore-conn-min-open", 10, "number of minimum concurrent connections open in a remote datastore's connection pool") @@ -67,113 +89,68 @@ func RegisterDatastoreFlags(cmd *cobra.Command, opts *Options) { cmd.Flags().IntVar(&opts.MaxRetries, "datastore-max-tx-retries", 50, "number of times a retriable transaction should be retried (cockroach driver only)") cmd.Flags().StringVar(&opts.OverlapStrategy, "datastore-tx-overlap-strategy", "static", `strategy to generate transaction overlap keys ("prefix", "static", "insecure") (cockroach driver only)`) cmd.Flags().StringVar(&opts.OverlapKey, "datastore-tx-overlap-key", "key", "static key to touch when writing to ensure transactions overlap (only used if --datastore-tx-overlap-strategy=static is set; cockroach driver only)") - return } -type Option func(*Options) - // NewDatastore initializes a datastore given the options -func NewDatastore(kind datastore.Engine, options ...Option) (datastore.Datastore, error) { - var opts Options +func NewDatastore(options ...Option) (datastore.Datastore, error) { + var opts DatastoreConfig for _, o := range options { o(&opts) } - dsBuilder, ok := builderForEngine[kind] + dsBuilder, ok := builderForEngine[opts.Engine] if !ok { - return nil, fmt.Errorf("unknown datastore engine type: %s", kind) + return nil, fmt.Errorf("unknown datastore engine type: %s", opts.Engine) } - log.Info().Msgf("using %s datastore engine", kind) + log.Info().Msgf("using %s datastore engine", opts.Engine) return dsBuilder(opts) } -func WithRevisionQuantization(revisionQuantization time.Duration) Option { - return func(c *Options) { - c.RevisionQuantization = revisionQuantization - } -} - -func WithGCWindow(gcWindow time.Duration) Option { - return func(c *Options) { - c.GCWindow = gcWindow - } -} - -func WithURI(uri string) Option { - return func(c *Options) { - c.URI = uri - } -} - -func WithMaxIdleTime(maxIdleTime time.Duration) Option { - return func(c *Options) { - c.MaxIdleTime = maxIdleTime - } -} - -func WithMaxLifetime(maxLifetime time.Duration) Option { - return func(c *Options) { - c.MaxLifetime = maxLifetime - } -} - -func WithMaxOpenConns(maxOpenConns int) Option { - return func(c *Options) { - c.MaxOpenConns = maxOpenConns - } -} - -func WithMinOpenConns(minOpenConns int) Option { - return func(c *Options) { - c.MinOpenConns = minOpenConns - } -} - -func WithSplitQuerySize(splitQuerySize string) Option { - return func(c *Options) { - c.SplitQuerySize = splitQuerySize - } -} - -func WithFollowerReadDelay(followerDelay time.Duration) Option { - return func(c *Options) { - c.FollowerReadDelay = followerDelay - } -} - -func WithMaxRetries(retries int) Option { - return func(c *Options) { - c.MaxRetries = retries - } -} - -func WithOverlapKey(key string) Option { - return func(c *Options) { - c.OverlapKey = key - } -} - -func WithOverlapStrategy(strategy string) Option { - return func(c *Options) { - c.OverlapStrategy = strategy - } -} - -func WithHealthCheckPeriod(interval time.Duration) Option { - return func(c *Options) { - c.HealthCheckPeriod = interval - } -} - -func WithGCInterval(interval time.Duration) Option { - return func(c *Options) { - c.GCInterval = interval - } -} - -func WithGCMaxOperationTime(interval time.Duration) Option { - return func(c *Options) { - c.GCMaxOperationTime = interval - } +func newCRDBDatastore(opts DatastoreConfig) (datastore.Datastore, error) { + splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize) + if err != nil { + return nil, fmt.Errorf("failed to parse split query size: %w", err) + } + return crdb.NewCRDBDatastore( + opts.URI, + crdb.GCWindow(opts.GCWindow), + crdb.RevisionQuantization(opts.RevisionQuantization), + crdb.ConnMaxIdleTime(opts.MaxIdleTime), + crdb.ConnMaxLifetime(opts.MaxLifetime), + crdb.MaxOpenConns(opts.MaxOpenConns), + crdb.MinOpenConns(opts.MinOpenConns), + crdb.SplitAtEstimatedQuerySize(splitQuerySize), + crdb.FollowerReadDelay(opts.FollowerReadDelay), + crdb.MaxRetries(opts.MaxRetries), + crdb.OverlapKey(opts.OverlapKey), + crdb.OverlapStrategy(opts.OverlapStrategy), + ) +} + +func newPostgresDatastore(opts DatastoreConfig) (datastore.Datastore, error) { + splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize) + if err != nil { + return nil, fmt.Errorf("failed to parse split query size: %w", err) + } + return postgres.NewPostgresDatastore( + opts.URI, + postgres.GCWindow(opts.GCWindow), + postgres.RevisionFuzzingTimedelta(opts.RevisionQuantization), + postgres.ConnMaxIdleTime(opts.MaxIdleTime), + postgres.ConnMaxLifetime(opts.MaxLifetime), + postgres.MaxOpenConns(opts.MaxOpenConns), + postgres.MinOpenConns(opts.MinOpenConns), + postgres.SplitAtEstimatedQuerySize(splitQuerySize), + postgres.HealthCheckPeriod(opts.HealthCheckPeriod), + postgres.GCInterval(opts.GCInterval), + postgres.GCMaxOperationTime(opts.GCMaxOperationTime), + postgres.EnablePrometheusStats(), + postgres.EnableTracing(), + ) +} + +func newMemoryDatstore(opts DatastoreConfig) (datastore.Datastore, error) { + log.Warn().Msg("in-memory datastore is not persistent and not feasible to run in a high availability fashion") + return memdb.NewMemdbDatastore(0, opts.RevisionQuantization, opts.GCWindow, 0) } diff --git a/pkg/cmd/serve/datastore_options.go b/pkg/cmd/serve/datastore_options.go new file mode 100644 index 0000000000..50f4bf51d4 --- /dev/null +++ b/pkg/cmd/serve/datastore_options.go @@ -0,0 +1,135 @@ +// Code generated by github.com/ecordell/optgen. DO NOT EDIT. +package serve + +import "time" + +type DatastoreConfigOption func(d *DatastoreConfig) + +// NewDatastoreConfigWithOptions creates a new DatastoreConfig with the passed in options set +func NewDatastoreConfigWithOptions(opts ...DatastoreConfigOption) *DatastoreConfig { + d := &DatastoreConfig{} + for _, o := range opts { + o(d) + } + return d +} + +// DatastoreConfigWithOptions configures an existing DatastoreConfig with the passed in options set +func DatastoreConfigWithOptions(d *DatastoreConfig, opts ...DatastoreConfigOption) *DatastoreConfig { + for _, o := range opts { + o(d) + } + return d +} + +// WithEngine returns an option that can set Engine on a DatastoreConfig +func WithEngine(engine string) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.Engine = engine + } +} + +// WithURI returns an option that can set URI on a DatastoreConfig +func WithURI(uRI string) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.URI = uRI + } +} + +// WithGCWindow returns an option that can set GCWindow on a DatastoreConfig +func WithGCWindow(gCWindow time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.GCWindow = gCWindow + } +} + +// WithRevisionQuantization returns an option that can set RevisionQuantization on a DatastoreConfig +func WithRevisionQuantization(revisionQuantization time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.RevisionQuantization = revisionQuantization + } +} + +// WithMaxIdleTime returns an option that can set MaxIdleTime on a DatastoreConfig +func WithMaxIdleTime(maxIdleTime time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.MaxIdleTime = maxIdleTime + } +} + +// WithMaxLifetime returns an option that can set MaxLifetime on a DatastoreConfig +func WithMaxLifetime(maxLifetime time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.MaxLifetime = maxLifetime + } +} + +// WithMaxOpenConns returns an option that can set MaxOpenConns on a DatastoreConfig +func WithMaxOpenConns(maxOpenConns int) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.MaxOpenConns = maxOpenConns + } +} + +// WithMinOpenConns returns an option that can set MinOpenConns on a DatastoreConfig +func WithMinOpenConns(minOpenConns int) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.MinOpenConns = minOpenConns + } +} + +// WithSplitQuerySize returns an option that can set SplitQuerySize on a DatastoreConfig +func WithSplitQuerySize(splitQuerySize string) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.SplitQuerySize = splitQuerySize + } +} + +// WithFollowerReadDelay returns an option that can set FollowerReadDelay on a DatastoreConfig +func WithFollowerReadDelay(followerReadDelay time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.FollowerReadDelay = followerReadDelay + } +} + +// WithMaxRetries returns an option that can set MaxRetries on a DatastoreConfig +func WithMaxRetries(maxRetries int) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.MaxRetries = maxRetries + } +} + +// WithOverlapKey returns an option that can set OverlapKey on a DatastoreConfig +func WithOverlapKey(overlapKey string) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.OverlapKey = overlapKey + } +} + +// WithOverlapStrategy returns an option that can set OverlapStrategy on a DatastoreConfig +func WithOverlapStrategy(overlapStrategy string) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.OverlapStrategy = overlapStrategy + } +} + +// WithHealthCheckPeriod returns an option that can set HealthCheckPeriod on a DatastoreConfig +func WithHealthCheckPeriod(healthCheckPeriod time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.HealthCheckPeriod = healthCheckPeriod + } +} + +// WithGCInterval returns an option that can set GCInterval on a DatastoreConfig +func WithGCInterval(gCInterval time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.GCInterval = gCInterval + } +} + +// WithGCMaxOperationTime returns an option that can set GCMaxOperationTime on a DatastoreConfig +func WithGCMaxOperationTime(gCMaxOperationTime time.Duration) DatastoreConfigOption { + return func(d *DatastoreConfig) { + d.GCMaxOperationTime = gCMaxOperationTime + } +} diff --git a/tools.go b/tools.go new file mode 100644 index 0000000000..797b75befa --- /dev/null +++ b/tools.go @@ -0,0 +1,3 @@ +package spicedb + +import _ "github.com/ecordell/optgen"