From 76818a31c0bb1af2fc61d222d81f8619d737634b Mon Sep 17 00:00:00 2001 From: Josh Black Date: Fri, 11 Jun 2021 14:53:15 -0700 Subject: [PATCH 1/6] Switch boltdb freelist type to MapType and enable NofreelistSync --- physical/raft/fsm.go | 6 +++++- physical/raft/raft.go | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index 934c5726c8d5b..ecbbdbf25dffb 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -154,7 +154,11 @@ func (f *FSM) openDBFile(dbPath string) error { return errors.New("can not open empty filename") } - boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{Timeout: 1 * time.Second}) + boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{ + Timeout: 1 * time.Second, + FreelistType: bolt.FreelistMapType, + NoFreelistSync: true, + }) if err != nil { return err } diff --git a/physical/raft/raft.go b/physical/raft/raft.go index f7065ef7831de..56efcfb210108 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -364,7 +364,14 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } // Create the backend raft store for logs and stable storage. - store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db")) + raftOptions := raftboltdb.Options{ + Path: filepath.Join(path, "raft.db"), + BoltOptions: &bolt.Options{ + FreelistType: bolt.FreelistMapType, + NoFreelistSync: true, + }, + } + store, err := raftboltdb.New(raftOptions) if err != nil { return nil, err } From fca3ceb6159fb5f182a89507d292baacadd7e093 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 17 Jun 2021 18:06:51 -0700 Subject: [PATCH 2/6] add log line with elapsed db open time --- physical/raft/fsm.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index ecbbdbf25dffb..64d826fd32f81 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -154,6 +154,7 @@ func (f *FSM) openDBFile(dbPath string) error { return errors.New("can not open empty filename") } + start := time.Now() boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{ Timeout: 1 * time.Second, FreelistType: bolt.FreelistMapType, @@ -162,6 +163,7 @@ func (f *FSM) openDBFile(dbPath string) error { if err != nil { return err } + f.logger.Debug("time to open database", "elapsed", time.Now().Sub(start), "path", dbPath) err = boltDB.Update(func(tx *bolt.Tx) error { // make sure we have the necessary buckets created From b64545386bfda364a560d1caaa47ac3ee8f98935 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Fri, 18 Jun 2021 14:43:31 -0700 Subject: [PATCH 3/6] PR feedback --- physical/raft/fsm.go | 9 ++++++--- physical/raft/raft.go | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index 64d826fd32f81..b1d548964a5a1 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -154,16 +154,19 @@ func (f *FSM) openDBFile(dbPath string) error { return errors.New("can not open empty filename") } + freelistType, noFreelistSync := freelistOptions() start := time.Now() boltDB, err := bolt.Open(dbPath, 0o666, &bolt.Options{ Timeout: 1 * time.Second, - FreelistType: bolt.FreelistMapType, - NoFreelistSync: true, + FreelistType: freelistType, + NoFreelistSync: noFreelistSync, }) if err != nil { return err } - f.logger.Debug("time to open database", "elapsed", time.Now().Sub(start), "path", dbPath) + elapsed := time.Now().Sub(start) + f.logger.Debug("time to open database", "elapsed", elapsed, "path", dbPath) + metrics.MeasureSince([]string{"raft_storage", "fsm", "open_db_file"}, start) err = boltDB.Update(func(tx *bolt.Tx) error { // make sure we have the necessary buckets created diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 56efcfb210108..2787ad04a80fd 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -364,11 +364,12 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } // Create the backend raft store for logs and stable storage. + freelistType, noFreelistSync := freelistOptions() raftOptions := raftboltdb.Options{ Path: filepath.Join(path, "raft.db"), BoltOptions: &bolt.Options{ - FreelistType: bolt.FreelistMapType, - NoFreelistSync: true, + FreelistType: freelistType, + NoFreelistSync: noFreelistSync, }, } store, err := raftboltdb.New(raftOptions) @@ -1580,3 +1581,21 @@ func (s sealer) Open(ctx context.Context, ct []byte) ([]byte, error) { return s.access.Decrypt(ctx, &eblob, nil) } + +// freelistOptions returns the freelist type and nofreelistsync values to use +// when opening boltdb files, based on our preferred defaults, and the possible +// presence of overriding environment variables. +func freelistOptions() (bolt.FreelistType, bool) { + freelistType := bolt.FreelistMapType + noFreelistSync := true + + if os.Getenv("VAULT_FREELIST_TYPE") == "array" { + freelistType = bolt.FreelistArrayType + } + + if os.Getenv("VAULT_NO_FREELIST_SYNC") == "false" { + noFreelistSync = false + } + + return freelistType, noFreelistSync +} From 86d9e1b1a12cf2e85e77c48eec6ed1b423e66dff Mon Sep 17 00:00:00 2001 From: Josh Black Date: Fri, 18 Jun 2021 14:45:43 -0700 Subject: [PATCH 4/6] add changelog entry --- changelog/11895.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/11895.txt diff --git a/changelog/11895.txt b/changelog/11895.txt new file mode 100644 index 0000000000000..e4bb2a59219f5 --- /dev/null +++ b/changelog/11895.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +raft: change freelist type to map and set nofreelistsync to true +``` From d62832e48ad5fa0cecf23f75d39c114bb05a6056 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Fri, 18 Jun 2021 15:11:52 -0700 Subject: [PATCH 5/6] fix changelog type --- changelog/11895.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/11895.txt b/changelog/11895.txt index e4bb2a59219f5..2a8837f112beb 100644 --- a/changelog/11895.txt +++ b/changelog/11895.txt @@ -1,3 +1,3 @@ -```release-note:enhancement +```release-note:improvement raft: change freelist type to map and set nofreelistsync to true ``` From c188d54ad1170e641f699c9cc65ca8d0d39771ca Mon Sep 17 00:00:00 2001 From: Josh Black Date: Mon, 21 Jun 2021 10:56:25 -0700 Subject: [PATCH 6/6] fix up env vars --- physical/raft/raft.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/physical/raft/raft.go b/physical/raft/raft.go index 2787ad04a80fd..6ca75043f3455 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -1589,11 +1589,11 @@ func freelistOptions() (bolt.FreelistType, bool) { freelistType := bolt.FreelistMapType noFreelistSync := true - if os.Getenv("VAULT_FREELIST_TYPE") == "array" { + if os.Getenv("VAULT_RAFT_FREELIST_TYPE") == "array" { freelistType = bolt.FreelistArrayType } - if os.Getenv("VAULT_NO_FREELIST_SYNC") == "false" { + if os.Getenv("VAULT_RAFT_FREELIST_SYNC") != "" { noFreelistSync = false }