Make batch size configurable

services/search/pkg/config/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	Engine                     Engine                `yaml:"engine"`
 	Extractor                  Extractor             `yaml:"extractor"`
 	ContentExtractionSizeLimit uint64                `yaml:"content_extraction_size_limit" env:"SEARCH_CONTENT_EXTRACTION_SIZE_LIMIT" desc:"Maximum file size in bytes that is allowed for content extraction." introductionVersion:"1.0.0"`
+	BatchSize                  int                   `yaml:"batch_size" env:"SEARCH_BATCH_SIZE" desc:"The number of documents to process in a single batch. Defaults to 500." introductionVersion:"1.0.0"`
 
 	ServiceAccount ServiceAccount `yaml:"service_account"`
 

services/search/pkg/config/defaults/defaultconfig.go

@@ -58,6 +58,7 @@ func DefaultConfig() *config.Config {
 			AckWait:          1 * time.Minute,
 		},
 		ContentExtractionSizeLimit: 20 * 1024 * 1024, // Limit content extraction to <20MB files by default
+		BatchSize:                  500,
 	}
 }
 

services/search/pkg/search/service.go

@@ -41,7 +41,6 @@ const (
 	_spaceTypeProject    = "project"
 	_spaceTypeGrant      = "grant"
 	_slowQueryDuration   = 500 * time.Millisecond
-	_batchSize           = 500
 )
 
 // Searcher is the interface to the SearchService
@@ -65,6 +64,8 @@ type Service struct {
 
 	serviceAccountID     string
 	serviceAccountSecret string
+
+	batchSize int
 }
 
 var errSkipSpace error
@@ -80,6 +81,8 @@ func NewService(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], eng e
 
 		serviceAccountID:     cfg.ServiceAccount.ServiceAccountID,
 		serviceAccountSecret: cfg.ServiceAccount.ServiceAccountSecret,
+
+		batchSize: cfg.BatchSize,
 	}
 
 	return s
@@ -460,7 +463,7 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
 	}()
 
 	w := walker.NewWalker(s.gatewaySelector)
-	s.engine.StartBatch(_batchSize)
+	s.engine.StartBatch(s.batchSize)
 	defer func() {
 		if err := s.engine.EndBatch(); err != nil {
 			s.logger.Error().Err(err).Msg("failed to end batch")