Search: Add search index configuration options (#55525)

conf/defaults.ini

@@ -1282,3 +1282,19 @@ scheduler_interval =
 [storage]
 # Allow uploading SVG files without sanitization.
 allow_unsanitized_svg_upload = false
+
+
+#################################### Search ################################################
+
+[search]
+# Defines the number of dashboards loaded at once in a batch during a full reindex.
+# This is a temporary settings that might be removed in the future.
+dashboard_loading_batch_size = 200
+
+# Defines the frequency of a full search reindex.
+# This is a temporary settings that might be removed in the future.
+full_reindex_interval = 5m
+
+# Defines the frequency of partial index updates based on recent changes such as dashboard updates.
+# This is a temporary settings that might be removed in the future.
+index_update_interval = 10s

pkg/services/searchV2/allowed_actions_test.go

@@ -14,6 +14,7 @@ import (
 	accesscontrolmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
 	"github.com/grafana/grafana/pkg/services/dashboards"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/require"
 )
 
@@ -80,7 +81,7 @@ var (
 )
 
 func service(t *testing.T) *StandardSearchService {
-	service, ok := ProvideService(nil, nil, nil, accesscontrolmock.New()).(*StandardSearchService)
+	service, ok := ProvideService(&setting.Cfg{Search: setting.SearchSettings{}}, nil, nil, accesscontrolmock.New()).(*StandardSearchService)
 	require.True(t, ok)
 	return service
 }

pkg/services/searchV2/index.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/searchV2/extract"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/services/store"
+	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/blugelabs/bluge"
 )
@@ -94,9 +95,10 @@ type searchIndex struct {
 	extender                DocumentExtender
 	folderIdLookup          folderUIDLookup
 	syncCh                  chan chan struct{}
+	settings                setting.SearchSettings
 }
 
-func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup) *searchIndex {
+func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup, settings setting.SearchSettings) *searchIndex {
 	return &searchIndex{
 		loader:          dashLoader,
 		eventStore:      evStore,
@@ -107,6 +109,7 @@ func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender Doc
 		extender:        extender,
 		folderIdLookup:  folderIDs,
 		syncCh:          make(chan chan struct{}),
+		settings:        settings,
 	}
 }
 
@@ -170,11 +173,13 @@ func (i *searchIndex) sync(ctx context.Context) error {
 }
 
 func (i *searchIndex) run(ctx context.Context, orgIDs []int64, reIndexSignalCh chan struct{}) error {
-	reIndexInterval := 5 * time.Minute
+	i.logger.Info("Initializing SearchV2", "dashboardLoadingBatchSize", i.settings.DashboardLoadingBatchSize, "fullReindexInterval", i.settings.FullReindexInterval, "indexUpdateInterval", i.settings.IndexUpdateInterval)
+
+	reIndexInterval := i.settings.FullReindexInterval
 	fullReIndexTimer := time.NewTimer(reIndexInterval)
 	defer fullReIndexTimer.Stop()
 
-	partialUpdateInterval := 5 * time.Second
+	partialUpdateInterval := i.settings.IndexUpdateInterval
 	partialUpdateTimer := time.NewTimer(partialUpdateInterval)
 	defer partialUpdateTimer.Stop()
 
@@ -748,12 +753,13 @@ func (i *searchIndex) updateDashboard(ctx context.Context, orgID int64, index *o
 }
 
 type sqlDashboardLoader struct {
-	sql    *sqlstore.SQLStore
-	logger log.Logger
+	sql      *sqlstore.SQLStore
+	logger   log.Logger
+	settings setting.SearchSettings
 }
 
-func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader {
-	return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")}
+func newSQLDashboardLoader(sql *sqlstore.SQLStore, settings setting.SearchSettings) *sqlDashboardLoader {
+	return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader"), settings: settings}
 }
 
 func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
@@ -762,7 +768,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 	limit := 1
 
 	if dashboardUID == "" {
-		limit = 200
+		limit = l.settings.DashboardLoadingBatchSize
 		dashboards = make([]dashboard, 0, limit+1)
 
 		// Add the root folder ID (does not exist in SQL).

pkg/services/searchV2/index_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/searchV2/extract"
@@ -64,7 +65,7 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten
 		dashboardLoader,
 		&store.MockEntityEventsService{},
 		extender,
-		func(ctx context.Context, folderId int64) (string, error) { return "x", nil })
+		func(ctx context.Context, folderId int64) (string, error) { return "x", nil }, setting.SearchSettings{})
 	require.NotNil(t, index)
 	numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID)
 	require.NoError(t, err)

pkg/services/searchV2/service.go

@@ -77,10 +77,11 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
 			ac:  ac,
 		},
 		dashboardIndex: newSearchIndex(
-			newSQLDashboardLoader(sql),
+			newSQLDashboardLoader(sql, cfg.Search),
 			entityEventStore,
 			extender.GetDocumentExtender(),
 			newFolderIDLookup(sql),
+			cfg.Search,
 		),
 		logger:    log.New("searchV2"),
 		extender:  extender,

pkg/setting/setting.go

@@ -449,6 +449,8 @@ type Cfg struct {
 
 	Storage StorageSettings
 
+	Search SearchSettings
+
 	// Access Control
 	RBACEnabled         bool
 	RBACPermissionCache bool
@@ -1021,6 +1023,7 @@ func (cfg *Cfg) Load(args CommandLineArgs) error {
 
 	cfg.DashboardPreviews = readDashboardPreviewsSettings(iniFile)
 	cfg.Storage = readStorageSettings(iniFile)
+	cfg.Search = readSearchSettings(iniFile)
 
 	if VerifyEmailEnabled && !cfg.Smtp.Enabled {
 		cfg.Logger.Warn("require_email_validation is enabled but smtp is disabled")

pkg/setting/setting_search.go

@@ -0,0 +1,23 @@
+package setting
+
+import (
+	"time"
+
+	"gopkg.in/ini.v1"
+)
+
+type SearchSettings struct {
+	FullReindexInterval       time.Duration
+	IndexUpdateInterval       time.Duration
+	DashboardLoadingBatchSize int
+}
+
+func readSearchSettings(iniFile *ini.File) SearchSettings {
+	s := SearchSettings{}
+
+	searchSection := iniFile.Section("search")
+	s.DashboardLoadingBatchSize = searchSection.Key("dashboard_loading_batch_size").MustInt(200)
+	s.FullReindexInterval = searchSection.Key("full_reindex_interval").MustDuration(5 * time.Minute)
+	s.IndexUpdateInterval = searchSection.Key("index_update_interval").MustDuration(10 * time.Second)
+	return s
+}