Skip to content

Commit

Permalink
model datastore cli options
Browse files Browse the repository at this point in the history
  • Loading branch information
ecordell committed Dec 6, 2021
1 parent 1015f37 commit 2055021
Show file tree
Hide file tree
Showing 5 changed files with 238 additions and 65 deletions.
84 changes: 19 additions & 65 deletions cmd/spicedb/serve.go
Expand Up @@ -8,7 +8,6 @@ import (
"syscall"
"time"

"github.com/alecthomas/units"
"github.com/fatih/color"
grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
grpczerolog "github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2"
Expand All @@ -25,9 +24,6 @@ import (
"github.com/authzed/spicedb/internal/dashboard"
"github.com/authzed/spicedb/internal/datastore"
"github.com/authzed/spicedb/internal/datastore/common"
"github.com/authzed/spicedb/internal/datastore/crdb"
"github.com/authzed/spicedb/internal/datastore/memdb"
"github.com/authzed/spicedb/internal/datastore/postgres"
"github.com/authzed/spicedb/internal/datastore/proxy"
combineddispatch "github.com/authzed/spicedb/internal/dispatch/combined"
"github.com/authzed/spicedb/internal/gateway"
Expand Down Expand Up @@ -128,68 +124,26 @@ func serveRun(cmd *cobra.Command, args []string) {
}

datastoreEngine := cobrautil.MustGetStringExpanded(cmd, "datastore-engine")
datastoreURI := cobrautil.MustGetStringExpanded(cmd, "datastore-conn-uri")

revisionFuzzingTimedelta := cobrautil.MustGetDuration(cmd, "datastore-revision-fuzzing-duration")
gcWindow := cobrautil.MustGetDuration(cmd, "datastore-gc-window")
maxRetries := cobrautil.MustGetInt(cmd, "datastore-max-tx-retries")
overlapKey := cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-key")
overlapStrategy := cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-strategy")

splitQuerySize, err := units.ParseBase2Bytes(cobrautil.MustGetStringExpanded(cmd, "datastore-query-split-size"))
ds, err := datastore.NewDatastore(
datastore.Engine(datastoreEngine),
datastore.WithRevisionQuantization(cobrautil.MustGetDuration(cmd, "datastore-revision-fuzzing-duration")),
datastore.WithGCWindow(cobrautil.MustGetDuration(cmd, "datastore-gc-window")),
datastore.WithURI(cobrautil.MustGetStringExpanded(cmd, "datastore-conn-uri")),
datastore.WithMaxIdleTime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-idletime")),
datastore.WithMaxLifetime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-lifetime")),
datastore.WithMaxOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-max-open")),
datastore.WithMinOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-min-open")),
datastore.WithSplitQuerySize(cobrautil.MustGetStringExpanded(cmd, "datastore-query-split-size")),
datastore.WithFollowerReadDelay(cobrautil.MustGetDuration(cmd, "datastore-follower-read-delay-duration")),
datastore.WithMaxRetries(cobrautil.MustGetInt(cmd, "datastore-max-tx-retries")),
datastore.WithOverlapKey(cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-key")),
datastore.WithOverlapStrategy(cobrautil.MustGetStringExpanded(cmd, "datastore-tx-overlap-strategy")),
datastore.WithHealthCheckPeriod(cobrautil.MustGetDuration(cmd, "datastore-conn-healthcheck-interval")),
datastore.WithGCInterval(cobrautil.MustGetDuration(cmd, "datastore-gc-interval")),
datastore.WithGCMaxOperationTime(cobrautil.MustGetDuration(cmd, "datastore-gc-max-operation-time")),
)
if err != nil {
log.Fatal().Err(err).Msg("failed to parse datastore-query-split-size")
}

var ds datastore.Datastore
if datastoreEngine == "memory" {
log.Info().Msg("using in-memory datastore")
log.Warn().Msg("in-memory datastore is not persistent and not feasible to run in a high availability fashion")
ds, err = memdb.NewMemdbDatastore(0, revisionFuzzingTimedelta, gcWindow, 0)
if err != nil {
log.Fatal().Err(err).Msg("failed to init datastore")
}
} else if datastoreEngine == "cockroachdb" {
log.Info().Msg("using cockroachdb datastore")
ds, err = crdb.NewCRDBDatastore(
datastoreURI,
crdb.ConnMaxIdleTime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-idletime")),
crdb.ConnMaxLifetime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-lifetime")),
crdb.MaxOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-max-open")),
crdb.MinOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-min-open")),
crdb.RevisionQuantization(revisionFuzzingTimedelta),
crdb.FollowerReadDelay(cobrautil.MustGetDuration(cmd, "datastore-follower-read-delay-duration")),
crdb.GCWindow(gcWindow),
crdb.MaxRetries(maxRetries),
crdb.SplitAtEstimatedQuerySize(splitQuerySize),
crdb.OverlapKey(overlapKey),
crdb.OverlapStrategy(overlapStrategy),
)
if err != nil {
log.Fatal().Err(err).Msg("failed to init datastore")
}
} else if datastoreEngine == "postgres" {
log.Info().Msg("using postgres datastore")
ds, err = postgres.NewPostgresDatastore(
datastoreURI,
postgres.ConnMaxIdleTime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-idletime")),
postgres.ConnMaxLifetime(cobrautil.MustGetDuration(cmd, "datastore-conn-max-lifetime")),
postgres.HealthCheckPeriod(cobrautil.MustGetDuration(cmd, "datastore-conn-healthcheck-interval")),
postgres.MaxOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-max-open")),
postgres.MinOpenConns(cobrautil.MustGetInt(cmd, "datastore-conn-min-open")),
postgres.RevisionFuzzingTimedelta(revisionFuzzingTimedelta),
postgres.GCInterval(cobrautil.MustGetDuration(cmd, "datastore-gc-interval")),
postgres.GCMaxOperationTime(cobrautil.MustGetDuration(cmd, "datastore-gc-max-operation-time")),
postgres.GCWindow(gcWindow),
postgres.EnablePrometheusStats(),
postgres.EnableTracing(),
postgres.SplitAtEstimatedQuerySize(splitQuerySize),
)
if err != nil {
log.Fatal().Err(err).Msg("failed to init datastore")
}
} else {
log.Fatal().Str("datastore-engine", datastoreEngine).Msg("unknown datastore engine type")
log.Fatal().Err(err).Msg("failed to init datastore")
}

bootstrapFilePaths := cobrautil.MustGetStringSlice(cmd, "datastore-bootstrap-files")
Expand Down
27 changes: 27 additions & 0 deletions internal/datastore/crdb/crdb.go
Expand Up @@ -49,8 +49,35 @@ const (
querySelectNow = "SELECT cluster_logical_timestamp()"
queryReturningTimestamp = "RETURNING cluster_logical_timestamp()"
queryShowZoneConfig = "SHOW ZONE CONFIGURATION FOR RANGE default;"

cockroachEngine datastore.Engine = "cockroach"
)

func init() {
datastore.RegisterEngine(cockroachEngine, newFromDatastoreOptions)
}

func newFromDatastoreOptions(opts datastore.Options) (datastore.Datastore, error) {
splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize)
if err != nil {
return nil, fmt.Errorf("failed to parse split query size: %w", err)
}
return NewCRDBDatastore(
opts.URI,
GCWindow(opts.GCWindow),
RevisionQuantization(opts.RevisionQuantization),
ConnMaxIdleTime(opts.MaxIdleTime),
ConnMaxLifetime(opts.MaxLifetime),
MaxOpenConns(opts.MaxOpenConns),
MinOpenConns(opts.MinOpenConns),
SplitAtEstimatedQuerySize(splitQuerySize),
FollowerReadDelay(opts.FollowerReadDelay),
MaxRetries(opts.MaxRetries),
OverlapKey(opts.OverlapKey),
OverlapStrategy(opts.OverlapStrategy),
)
}

// NewCRDBDatastore initializes a SpiceDB datastore that uses a CockroachDB
// database while leveraging its AOST functionality.
func NewCRDBDatastore(url string, options ...Option) (datastore.Datastore, error) {
Expand Down
12 changes: 12 additions & 0 deletions internal/datastore/memdb/memdb.go
Expand Up @@ -11,11 +11,23 @@ import (
v1 "github.com/authzed/authzed-go/proto/authzed/api/v1"
"github.com/hashicorp/go-memdb"
"github.com/jzelinskie/stringz"
"github.com/rs/zerolog/log"
"github.com/shopspring/decimal"

"github.com/authzed/spicedb/internal/datastore"
)

const memoryEngine datastore.Engine = "memory"

func init() {
datastore.RegisterEngine(memoryEngine, newFromDatastoreOptions)
}

func newFromDatastoreOptions(opts datastore.Options) (datastore.Datastore, error) {
log.Warn().Msg("in-memory datastore is not persistent and not feasible to run in a high availability fashion")
return NewMemdbDatastore(0, opts.RevisionQuantization, opts.GCWindow, 0)
}

// DisableGC is a convenient constant for setting the garbage collection
// interval high enough that it will never run.
const DisableGC = time.Duration(math.MaxInt64)
Expand Down
155 changes: 155 additions & 0 deletions internal/datastore/options.go
@@ -0,0 +1,155 @@
package datastore

import (
"fmt"
"time"

"github.com/rs/zerolog/log"
)

type Engine string

type engineBuilderFunc func(options Options) (Datastore, error)

func RegisterEngine(key Engine, builder engineBuilderFunc) {
if builderForEngine == nil {
builderForEngine = make(map[Engine]engineBuilderFunc)
}
if _, ok := builderForEngine[key]; ok {
panic("cannot register two datastore engines with the same name: " + key)
}
builderForEngine[key] = builder
}

var builderForEngine map[Engine]engineBuilderFunc

type Options struct {
URI string
GCWindow time.Duration
RevisionQuantization time.Duration

MaxIdleTime time.Duration
MaxLifetime time.Duration
MaxOpenConns int
MinOpenConns int
SplitQuerySize string

// CRDB
FollowerReadDelay time.Duration
MaxRetries int
OverlapKey string
OverlapStrategy string

// Postgres
HealthCheckPeriod time.Duration
GCInterval time.Duration
GCMaxOperationTime time.Duration
}

type Option func(*Options)

// NewDatastore initializes a datastore given the options
func NewDatastore(kind Engine, options ...Option) (Datastore, error) {
var opts Options
for _, o := range options {
o(&opts)
}

dsBuilder, ok := builderForEngine[kind]
if !ok {
return nil, fmt.Errorf("unknown datastore engine type: %s", kind)
}
log.Info().Msgf("using %s datastore engine", kind)

return dsBuilder(opts)
}

func WithRevisionQuantization(revisionQuantization time.Duration) Option {
return func(c *Options) {
c.RevisionQuantization = revisionQuantization
}
}

func WithGCWindow(gcWindow time.Duration) Option {
return func(c *Options) {
c.GCWindow = gcWindow
}
}

func WithURI(uri string) Option {
return func(c *Options) {
c.URI = uri
}
}

func WithMaxIdleTime(maxIdleTime time.Duration) Option {
return func(c *Options) {
c.MaxIdleTime = maxIdleTime
}
}

func WithMaxLifetime(maxLifetime time.Duration) Option {
return func(c *Options) {
c.MaxLifetime = maxLifetime
}
}

func WithMaxOpenConns(maxOpenConns int) Option {
return func(c *Options) {
c.MaxOpenConns = maxOpenConns
}
}

func WithMinOpenConns(minOpenConns int) Option {
return func(c *Options) {
c.MinOpenConns = minOpenConns
}
}

func WithSplitQuerySize(splitQuerySize string) Option {
return func(c *Options) {
c.SplitQuerySize = splitQuerySize
}
}

func WithFollowerReadDelay(followerDelay time.Duration) Option {
return func(c *Options) {
c.FollowerReadDelay = followerDelay
}
}

func WithMaxRetries(retries int) Option {
return func(c *Options) {
c.MaxRetries = retries
}
}

func WithOverlapKey(key string) Option {
return func(c *Options) {
c.OverlapKey = key
}
}

func WithOverlapStrategy(strategy string) Option {
return func(c *Options) {
c.OverlapStrategy = strategy
}
}

func WithHealthCheckPeriod(interval time.Duration) Option {
return func(c *Options) {
c.HealthCheckPeriod = interval
}
}

func WithGCInterval(interval time.Duration) Option {
return func(c *Options) {
c.GCInterval = interval
}
}

func WithGCMaxOperationTime(interval time.Duration) Option {
return func(c *Options) {
c.GCMaxOperationTime = interval
}
}
25 changes: 25 additions & 0 deletions internal/datastore/postgres/postgres.go
Expand Up @@ -55,6 +55,8 @@ const (
tracingDriverName = "postgres-tracing"

batchDeleteSize = 1000

postgresEngine datastore.Engine = "postgres"
)

var (
Expand Down Expand Up @@ -83,6 +85,29 @@ var (

func init() {
dbsql.Register(tracingDriverName, sqlmw.Driver(stdlib.GetDefaultDriver(), new(traceInterceptor)))
datastore.RegisterEngine(postgresEngine, newFromDatastoreOptions)
}

func newFromDatastoreOptions(opts datastore.Options) (datastore.Datastore, error) {
splitQuerySize, err := units.ParseBase2Bytes(opts.SplitQuerySize)
if err != nil {
return nil, fmt.Errorf("failed to parse split query size: %w", err)
}
return NewPostgresDatastore(
opts.URI,
GCWindow(opts.GCWindow),
RevisionFuzzingTimedelta(opts.RevisionQuantization),
ConnMaxIdleTime(opts.MaxIdleTime),
ConnMaxLifetime(opts.MaxLifetime),
MaxOpenConns(opts.MaxOpenConns),
MinOpenConns(opts.MinOpenConns),
SplitAtEstimatedQuerySize(splitQuerySize),
HealthCheckPeriod(opts.HealthCheckPeriod),
GCInterval(opts.GCInterval),
GCMaxOperationTime(opts.GCMaxOperationTime),
EnablePrometheusStats(),
EnableTracing(),
)
}

var (
Expand Down

0 comments on commit 2055021

Please sign in to comment.