From 8fa10514690f214f300f6e6c9a385b780ff6d243 Mon Sep 17 00:00:00 2001 From: Artur Wierzbicki Date: Wed, 21 Sep 2022 01:09:55 +0200 Subject: [PATCH 1/2] Search: Add search index configuration options (#55525) * Search: externalize config * Search: update config descriptions * Search: fix value * Search: fix * update * Search: revert config values * Search: rename copy/paste * Search: fix tests (cherry picked from commit c3ca5405cedf19e71ae644bbe62f7c1231eea3ec) --- conf/defaults.ini | 16 +++++++++++++ pkg/services/searchV2/allowed_actions_test.go | 3 ++- pkg/services/searchV2/index.go | 23 ++++++++++++------- pkg/services/searchV2/index_test.go | 3 ++- pkg/services/searchV2/service.go | 3 ++- pkg/setting/setting.go | 3 +++ pkg/setting/setting_search.go | 23 +++++++++++++++++++ 7 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 pkg/setting/setting_search.go diff --git a/conf/defaults.ini b/conf/defaults.ini index f91fe98508e0..294b941f1684 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1282,3 +1282,19 @@ scheduler_interval = [storage] # Allow uploading SVG files without sanitization. allow_unsanitized_svg_upload = false + + +#################################### Search ################################################ + +[search] +# Defines the number of dashboards loaded at once in a batch during a full reindex. +# This is a temporary settings that might be removed in the future. +dashboard_loading_batch_size = 200 + +# Defines the frequency of a full search reindex. +# This is a temporary settings that might be removed in the future. +full_reindex_interval = 5m + +# Defines the frequency of partial index updates based on recent changes such as dashboard updates. +# This is a temporary settings that might be removed in the future. +index_update_interval = 10s diff --git a/pkg/services/searchV2/allowed_actions_test.go b/pkg/services/searchV2/allowed_actions_test.go index ce0ced9f5afc..b694b6e7541a 100644 --- a/pkg/services/searchV2/allowed_actions_test.go +++ b/pkg/services/searchV2/allowed_actions_test.go @@ -14,6 +14,7 @@ import ( accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -80,7 +81,7 @@ var ( ) func service(t *testing.T) *StandardSearchService { - service, ok := ProvideService(nil, nil, nil, accesscontrolmock.New()).(*StandardSearchService) + service, ok := ProvideService(&setting.Cfg{Search: setting.SearchSettings{}}, nil, nil, accesscontrolmock.New()).(*StandardSearchService) require.True(t, ok) return service } diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index 4a7e9209b6ec..3e1eda2f00e8 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -19,6 +19,8 @@ import ( "github.com/grafana/grafana/pkg/services/searchV2/extract" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/store" + "github.com/grafana/grafana/pkg/setting" + "go.opentelemetry.io/otel/attribute" "github.com/blugelabs/bluge" ) @@ -94,9 +96,10 @@ type searchIndex struct { extender DocumentExtender folderIdLookup folderUIDLookup syncCh chan chan struct{} + settings setting.SearchSettings } -func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup) *searchIndex { +func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup, settings setting.SearchSettings) *searchIndex { return &searchIndex{ loader: dashLoader, eventStore: evStore, @@ -107,6 +110,7 @@ func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender Doc extender: extender, folderIdLookup: folderIDs, syncCh: make(chan chan struct{}), + settings: settings, } } @@ -170,11 +174,13 @@ func (i *searchIndex) sync(ctx context.Context) error { } func (i *searchIndex) run(ctx context.Context, orgIDs []int64, reIndexSignalCh chan struct{}) error { - reIndexInterval := 5 * time.Minute + i.logger.Info("Initializing SearchV2", "dashboardLoadingBatchSize", i.settings.DashboardLoadingBatchSize, "fullReindexInterval", i.settings.FullReindexInterval, "indexUpdateInterval", i.settings.IndexUpdateInterval) + + reIndexInterval := i.settings.FullReindexInterval fullReIndexTimer := time.NewTimer(reIndexInterval) defer fullReIndexTimer.Stop() - partialUpdateInterval := 5 * time.Second + partialUpdateInterval := i.settings.IndexUpdateInterval partialUpdateTimer := time.NewTimer(partialUpdateInterval) defer partialUpdateTimer.Stop() @@ -748,12 +754,13 @@ func (i *searchIndex) updateDashboard(ctx context.Context, orgID int64, index *o } type sqlDashboardLoader struct { - sql *sqlstore.SQLStore - logger log.Logger + sql *sqlstore.SQLStore + logger log.Logger + settings setting.SearchSettings } -func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader { - return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")} +func newSQLDashboardLoader(sql *sqlstore.SQLStore, settings setting.SearchSettings) *sqlDashboardLoader { + return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader"), settings: settings} } func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) { @@ -762,7 +769,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das limit := 1 if dashboardUID == "" { - limit = 200 + limit = l.settings.DashboardLoadingBatchSize dashboards = make([]dashboard, 0, limit+1) // Add the root folder ID (does not exist in SQL). diff --git a/pkg/services/searchV2/index_test.go b/pkg/services/searchV2/index_test.go index 079605209624..96560f56f045 100644 --- a/pkg/services/searchV2/index_test.go +++ b/pkg/services/searchV2/index_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/searchV2/extract" @@ -64,7 +65,7 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten dashboardLoader, &store.MockEntityEventsService{}, extender, - func(ctx context.Context, folderId int64) (string, error) { return "x", nil }) + func(ctx context.Context, folderId int64) (string, error) { return "x", nil }, setting.SearchSettings{}) require.NotNil(t, index) numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID) require.NoError(t, err) diff --git a/pkg/services/searchV2/service.go b/pkg/services/searchV2/service.go index 7b00e68cdc47..54ea8bc38cfd 100644 --- a/pkg/services/searchV2/service.go +++ b/pkg/services/searchV2/service.go @@ -77,10 +77,11 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s ac: ac, }, dashboardIndex: newSearchIndex( - newSQLDashboardLoader(sql), + newSQLDashboardLoader(sql, cfg.Search), entityEventStore, extender.GetDocumentExtender(), newFolderIDLookup(sql), + cfg.Search, ), logger: log.New("searchV2"), extender: extender, diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 3fc8a51bffa0..276c464d75f9 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -449,6 +449,8 @@ type Cfg struct { Storage StorageSettings + Search SearchSettings + // Access Control RBACEnabled bool RBACPermissionCache bool @@ -1021,6 +1023,7 @@ func (cfg *Cfg) Load(args CommandLineArgs) error { cfg.DashboardPreviews = readDashboardPreviewsSettings(iniFile) cfg.Storage = readStorageSettings(iniFile) + cfg.Search = readSearchSettings(iniFile) if VerifyEmailEnabled && !cfg.Smtp.Enabled { cfg.Logger.Warn("require_email_validation is enabled but smtp is disabled") diff --git a/pkg/setting/setting_search.go b/pkg/setting/setting_search.go new file mode 100644 index 000000000000..b889fdd6319c --- /dev/null +++ b/pkg/setting/setting_search.go @@ -0,0 +1,23 @@ +package setting + +import ( + "time" + + "gopkg.in/ini.v1" +) + +type SearchSettings struct { + FullReindexInterval time.Duration + IndexUpdateInterval time.Duration + DashboardLoadingBatchSize int +} + +func readSearchSettings(iniFile *ini.File) SearchSettings { + s := SearchSettings{} + + searchSection := iniFile.Section("search") + s.DashboardLoadingBatchSize = searchSection.Key("dashboard_loading_batch_size").MustInt(200) + s.FullReindexInterval = searchSection.Key("full_reindex_interval").MustDuration(5 * time.Minute) + s.IndexUpdateInterval = searchSection.Key("index_update_interval").MustDuration(10 * time.Second) + return s +} From 4121da912157ed3738330b00efee4e6e035cde24 Mon Sep 17 00:00:00 2001 From: Artur Wierzbicki Date: Wed, 21 Sep 2022 01:42:14 +0200 Subject: [PATCH 2/2] fix merge --- pkg/services/searchV2/index.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index 3e1eda2f00e8..4864f9fc7b47 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -20,7 +20,6 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/setting" - "go.opentelemetry.io/otel/attribute" "github.com/blugelabs/bluge" )