diff --git a/services/search/pkg/config/config.go b/services/search/pkg/config/config.go index 3d27ee7e4..c00196b5f 100644 --- a/services/search/pkg/config/config.go +++ b/services/search/pkg/config/config.go @@ -28,6 +28,7 @@ type Config struct { Engine Engine `yaml:"engine"` Extractor Extractor `yaml:"extractor"` ContentExtractionSizeLimit uint64 `yaml:"content_extraction_size_limit" env:"SEARCH_CONTENT_EXTRACTION_SIZE_LIMIT" desc:"Maximum file size in bytes that is allowed for content extraction." introductionVersion:"1.0.0"` + BatchSize int `yaml:"batch_size" env:"SEARCH_BATCH_SIZE" desc:"The number of documents to process in a single batch. Defaults to 500." introductionVersion:"1.0.0"` ServiceAccount ServiceAccount `yaml:"service_account"` diff --git a/services/search/pkg/config/defaults/defaultconfig.go b/services/search/pkg/config/defaults/defaultconfig.go index ba9aefaf1..ff3bc0711 100644 --- a/services/search/pkg/config/defaults/defaultconfig.go +++ b/services/search/pkg/config/defaults/defaultconfig.go @@ -58,6 +58,7 @@ func DefaultConfig() *config.Config { AckWait: 1 * time.Minute, }, ContentExtractionSizeLimit: 20 * 1024 * 1024, // Limit content extraction to <20MB files by default + BatchSize: 500, } } diff --git a/services/search/pkg/search/service.go b/services/search/pkg/search/service.go index 25182d2ae..673d94ec7 100644 --- a/services/search/pkg/search/service.go +++ b/services/search/pkg/search/service.go @@ -41,7 +41,6 @@ const ( _spaceTypeProject = "project" _spaceTypeGrant = "grant" _slowQueryDuration = 500 * time.Millisecond - _batchSize = 500 ) // Searcher is the interface to the SearchService @@ -65,6 +64,8 @@ type Service struct { serviceAccountID string serviceAccountSecret string + + batchSize int } var errSkipSpace error @@ -80,6 +81,8 @@ func NewService(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], eng e serviceAccountID: cfg.ServiceAccount.ServiceAccountID, serviceAccountSecret: cfg.ServiceAccount.ServiceAccountSecret, + + batchSize: cfg.BatchSize, } return s @@ -460,7 +463,7 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error { }() w := walker.NewWalker(s.gatewaySelector) - s.engine.StartBatch(_batchSize) + s.engine.StartBatch(s.batchSize) defer func() { if err := s.engine.EndBatch(); err != nil { s.logger.Error().Err(err).Msg("failed to end batch")