build(deps): bump github.com/blevesearch/bleve/v2 from 2.4.4 to 2.5.0

Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.4.4 to 2.5.0.
- [Release notes](https://github.com/blevesearch/bleve/releases)
- [Commits](https://github.com/blevesearch/bleve/compare/v2.4.4...v2.5.0)

---
updated-dependencies:
- dependency-name: github.com/blevesearch/bleve/v2
  dependency-version: 2.5.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

vendor/github.com/blevesearch/bleve/v2/.travis.yml

@@ -3,9 +3,9 @@ sudo: false
 language: go
 
 go:
- - "1.12.x"
- - "1.13.x"
- - "1.14.x"
+ - "1.21.x"
+ - "1.22.x"
+ - "1.23.x"
 
 script:
   - go get golang.org/x/tools/cmd/cover
@@ -17,9 +17,9 @@ script:
   - go vet $(go list ./... | grep -v vendor/)
   - go test ./test -v -indexType scorch
   - errcheck -ignorepkg fmt $(go list ./... | grep -v vendor/);
-  - docs/project-code-coverage.sh
-  - docs/build_children.sh
+  - scripts/project-code-coverage.sh
+  - scripts/build_children.sh
 
 notifications:
   email:
-    - marty.schoch@gmail.com
+    - fts-team@couchbase.com

vendor/github.com/blevesearch/bleve/v2/README.md

@@ -1,11 +1,11 @@
 # ![bleve](docs/bleve.png) bleve
 
-[![Tests](https://github.com/blevesearch/bleve/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions?query=workflow%3ATests+event%3Apush+branch%3Amaster)
+[![Tests](https://github.com/blevesearch/bleve/actions/workflows/tests.yml/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions/workflows/tests.yml?query=event%3Apush+branch%3Amaster)
 [![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master)
-[![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve)
-[![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Go Reference](https://pkg.go.dev/badge/github.com/blevesearch/bleve/v2.svg)](https://pkg.go.dev/github.com/blevesearch/bleve/v2)
+[![Join the chat](https://badges.gitter.im/join_chat.svg)](https://app.gitter.im/#/room/#blevesearch_bleve:gitter.im)
 [![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve)
-[![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve)
+[![Go Report Card](https://goreportcard.com/badge/github.com/blevesearch/bleve/v2)](https://goreportcard.com/report/github.com/blevesearch/bleve/v2)
 [![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
@@ -24,7 +24,8 @@ A modern indexing + search library in GO
     * [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/)
     * [geo spatial search](https://github.com/blevesearch/bleve/blob/master/geo/README.md)
     * approximate k-nearest neighbors via [vector search](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md)
-* [tf-idf](https://en.wikipedia.org/wiki/Tf-idf) scoring
+    * [synonym search](https://github.com/blevesearch/bleve/blob/master/docs/synonyms.md)
+* [tf-idf](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#tf-idf) / [bm25](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#bm25) scoring models
 * Hybrid search: exact + semantic
 * Query time boosting
 * Search result match highlighting with document fragments
@@ -42,7 +43,7 @@ message := struct{
 	Body string
 }{
 	Id:   "example",
-	From: "marty.schoch@gmail.com",
+	From: "xyz@couchbase.com",
 	Body: "bleve indexing is easy",
 }
 

vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/custom/custom.go

@@ -101,7 +101,10 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (
 }
 
 func init() {
-	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	err := registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) {

vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/keyword/keyword.go

@@ -34,5 +34,8 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (
 }
 
 func init() {
-	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	err := registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/analyzer/standard/standard.go

@@ -48,5 +48,8 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (
 }
 
 func init() {
-	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	err := registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go

@@ -60,5 +60,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go

@@ -43,5 +43,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds/microseconds.go

@@ -48,5 +48,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds/milliseconds.go

@@ -48,5 +48,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds/nanoseconds.go

@@ -48,5 +48,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds/seconds.go

@@ -48,5 +48,8 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
 }
 
 func init() {
-	registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/analyzer_en.go

@@ -66,5 +66,8 @@ func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (
 }
 
 func init() {
-	registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)
+	err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/plural_stemmer.go

@@ -63,7 +63,10 @@ func EnglishPluralStemmerFilterConstructor(config map[string]interface{}, cache
 }
 
 func init() {
-	registry.RegisterTokenFilter(PluralStemmerName, EnglishPluralStemmerFilterConstructor)
+	err := registry.RegisterTokenFilter(PluralStemmerName, EnglishPluralStemmerFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }
 
 // ----------------------------------------------------------------------------

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/possessive_filter_en.go

@@ -63,5 +63,8 @@ func PossessiveFilterConstructor(config map[string]interface{}, cache *registry.
 }
 
 func init() {
-	registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor)
+	err := registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stemmer_en_snowball.go

@@ -45,5 +45,8 @@ func EnglishStemmerFilterConstructor(config map[string]interface{}, cache *regis
 }
 
 func init() {
-	registry.RegisterTokenFilter(SnowballStemmerName, EnglishStemmerFilterConstructor)
+	err := registry.RegisterTokenFilter(SnowballStemmerName, EnglishStemmerFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_filter_en.go

@@ -29,5 +29,8 @@ func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.C
 }
 
 func init() {
-	registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)
+	err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/lang/en/stop_words_en.go

@@ -340,5 +340,8 @@ func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (
 }
 
 func init() {
-	registry.RegisterTokenMap(StopName, TokenMapConstructor)
+	err := registry.RegisterTokenMap(StopName, TokenMapConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/token/lowercase/lowercase.go

@@ -47,7 +47,10 @@ func LowerCaseFilterConstructor(config map[string]interface{}, cache *registry.C
 }
 
 func init() {
-	registry.RegisterTokenFilter(Name, LowerCaseFilterConstructor)
+	err := registry.RegisterTokenFilter(Name, LowerCaseFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }
 
 // toLowerDeferredCopy will function exactly like

vendor/github.com/blevesearch/bleve/v2/analysis/token/porter/porter.go

@@ -49,5 +49,8 @@ func PorterStemmerConstructor(config map[string]interface{}, cache *registry.Cac
 }
 
 func init() {
-	registry.RegisterTokenFilter(Name, PorterStemmerConstructor)
+	err := registry.RegisterTokenFilter(Name, PorterStemmerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/token/stop/stop.go

@@ -66,5 +66,8 @@ func StopTokensFilterConstructor(config map[string]interface{}, cache *registry.
 }
 
 func init() {
-	registry.RegisterTokenFilter(Name, StopTokensFilterConstructor)
+	err := registry.RegisterTokenFilter(Name, StopTokensFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/single/single.go

@@ -45,5 +45,8 @@ func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *regis
 }
 
 func init() {
-	registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor)
+	err := registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode/unicode.go

@@ -115,7 +115,10 @@ func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.
 }
 
 func init() {
-	registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor)
+	err := registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func convertType(segmentWordType int) analysis.TokenType {

vendor/github.com/blevesearch/bleve/v2/analysis/type.go

@@ -106,6 +106,15 @@ type DateTimeParser interface {
 	ParseDateTime(string) (time.Time, string, error)
 }
 
+const SynonymSourceType = "synonym"
+
+type SynonymSourceVisitor func(name string, item SynonymSource) error
+
+type SynonymSource interface {
+	Analyzer() string
+	Collection() string
+}
+
 type ByteArrayConverter interface {
 	Convert([]byte) (interface{}, error)
 }

vendor/github.com/blevesearch/bleve/v2/doc.go

@@ -19,7 +19,7 @@ Example Opening New Index, Indexing Data
 
 	message := struct{
 	    Id:   "example"
-	    From: "marty.schoch@gmail.com",
+	    From: "xyz@couchbase.com",
 	    Body: "bleve indexing is easy",
 	}
 

vendor/github.com/blevesearch/bleve/v2/document/document.go

@@ -34,6 +34,7 @@ type Document struct {
 	Fields           []Field `json:"fields"`
 	CompositeFields  []*CompositeField
 	StoredFieldsSize uint64
+	indexed          bool
 }
 
 func (d *Document) StoredFieldsBytes() uint64 {
@@ -48,6 +49,13 @@ func NewDocument(id string) *Document {
 	}
 }
 
+func NewSynonymDocument(id string) *Document {
+	return &Document{
+		id:     id,
+		Fields: make([]Field, 0),
+	}
+}
+
 func (d *Document) Size() int {
 	sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr +
 		len(d.id)
@@ -133,3 +141,19 @@ func (d *Document) VisitComposite(visitor index.CompositeFieldVisitor) {
 func (d *Document) HasComposite() bool {
 	return len(d.CompositeFields) > 0
 }
+
+func (d *Document) VisitSynonymFields(visitor index.SynonymFieldVisitor) {
+	for _, f := range d.Fields {
+		if sf, ok := f.(index.SynonymField); ok {
+			visitor(sf)
+		}
+	}
+}
+
+func (d *Document) SetIndexed() {
+	d.indexed = true
+}
+
+func (d *Document) Indexed() bool {
+	return d.indexed
+}

vendor/github.com/blevesearch/bleve/v2/document/field_boolean.go

@@ -116,13 +116,13 @@ func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte
 		name:              name,
 		arrayPositions:    arrayPositions,
 		value:             value,
-		options:           DefaultNumericIndexingOptions,
+		options:           DefaultBooleanIndexingOptions,
 		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
 func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField {
-	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions)
+	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultBooleanIndexingOptions)
 }
 
 func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField {

vendor/github.com/blevesearch/bleve/v2/document/field_geoshape.go

@@ -87,13 +87,6 @@ func (n *GeoShapeField) AnalyzedTokenFrequencies() index.TokenFrequencies {
 func (n *GeoShapeField) Analyze() {
 	// compute the bytes representation for the coordinates
 	tokens := make(analysis.TokenStream, 0)
-	tokens = append(tokens, &analysis.Token{
-		Start:    0,
-		End:      len(n.encodedValue),
-		Term:     n.encodedValue,
-		Position: 1,
-		Type:     analysis.AlphaNumeric,
-	})
 
 	rti := geo.GetSpatialAnalyzerPlugin("s2")
 	terms := rti.GetIndexTokens(n.shape)
@@ -126,6 +119,10 @@ func (n *GeoShapeField) NumPlainTextBytes() uint64 {
 	return n.numPlainTextBytes
 }
 
+func (n *GeoShapeField) EncodedShape() []byte {
+	return n.encodedValue
+}
+
 func NewGeoShapeField(name string, arrayPositions []uint64,
 	coordinates [][][][]float64, typ string) *GeoShapeField {
 	return NewGeoShapeFieldWithIndexingOptions(name, arrayPositions,

vendor/github.com/blevesearch/bleve/v2/document/field_ip.go

@@ -31,7 +31,7 @@ func init() {
 	reflectStaticSizeIPField = int(reflect.TypeOf(f).Size())
 }
 
-const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues | index.IncludeTermVectors
+const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues
 
 type IPField struct {
 	name              string
@@ -115,7 +115,7 @@ func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IP
 		name:              name,
 		arrayPositions:    arrayPositions,
 		value:             value,
-		options:           DefaultNumericIndexingOptions,
+		options:           DefaultIPIndexingOptions,
 		numPlainTextBytes: uint64(len(value)),
 	}
 }

vendor/github.com/blevesearch/bleve/v2/document/field_synonym.go

@@ -0,0 +1,149 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+var reflectStaticSizeSynonymField int
+
+func init() {
+	var f SynonymField
+	reflectStaticSizeSynonymField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultSynonymIndexingOptions = index.IndexField
+
+type SynonymField struct {
+	name              string
+	analyzer          analysis.Analyzer
+	options           index.FieldIndexingOptions
+	input             []string
+	synonyms          []string
+	numPlainTextBytes uint64
+
+	// populated during analysis
+	synonymMap map[string][]string
+}
+
+func (s *SynonymField) Size() int {
+	return reflectStaticSizeSynonymField + size.SizeOfPtr +
+		len(s.name)
+}
+
+func (s *SynonymField) Name() string {
+	return s.name
+}
+
+func (s *SynonymField) ArrayPositions() []uint64 {
+	return nil
+}
+
+func (s *SynonymField) Options() index.FieldIndexingOptions {
+	return s.options
+}
+
+func (s *SynonymField) NumPlainTextBytes() uint64 {
+	return s.numPlainTextBytes
+}
+
+func (s *SynonymField) AnalyzedLength() int {
+	return 0
+}
+
+func (s *SynonymField) EncodedFieldType() byte {
+	return 'y'
+}
+
+func (s *SynonymField) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return nil
+}
+
+func (s *SynonymField) Analyze() {
+	var analyzedInput []string
+	if len(s.input) > 0 {
+		analyzedInput = make([]string, 0, len(s.input))
+		for _, term := range s.input {
+			analyzedTerm := analyzeSynonymTerm(term, s.analyzer)
+			if analyzedTerm != "" {
+				analyzedInput = append(analyzedInput, analyzedTerm)
+			}
+		}
+	}
+	analyzedSynonyms := make([]string, 0, len(s.synonyms))
+	for _, syn := range s.synonyms {
+		analyzedTerm := analyzeSynonymTerm(syn, s.analyzer)
+		if analyzedTerm != "" {
+			analyzedSynonyms = append(analyzedSynonyms, analyzedTerm)
+		}
+	}
+	s.synonymMap = processSynonymData(analyzedInput, analyzedSynonyms)
+}
+
+func (s *SynonymField) Value() []byte {
+	return nil
+}
+
+func (s *SynonymField) IterateSynonyms(visitor func(term string, synonyms []string)) {
+	for term, synonyms := range s.synonymMap {
+		visitor(term, synonyms)
+	}
+}
+
+func NewSynonymField(name string, analyzer analysis.Analyzer, input []string, synonyms []string) *SynonymField {
+	return &SynonymField{
+		name:     name,
+		analyzer: analyzer,
+		options:  DefaultSynonymIndexingOptions,
+		input:    input,
+		synonyms: synonyms,
+	}
+}
+
+func processSynonymData(input []string, synonyms []string) map[string][]string {
+	var synonymMap map[string][]string
+	if len(input) > 0 {
+		// Map each term to the same list of synonyms.
+		synonymMap = make(map[string][]string, len(input))
+		for _, term := range input {
+			synonymMap[term] = synonyms
+		}
+	} else {
+		synonymMap = make(map[string][]string, len(synonyms))
+		// Precompute a map where each synonym points to all other synonyms.
+		for i, elem := range synonyms {
+			synonymMap[elem] = make([]string, 0, len(synonyms)-1)
+			for j, otherElem := range synonyms {
+				if i != j {
+					synonymMap[elem] = append(synonymMap[elem], otherElem)
+				}
+			}
+		}
+	}
+	return synonymMap
+}
+
+func analyzeSynonymTerm(term string, analyzer analysis.Analyzer) string {
+	tokenStream := analyzer.Analyze([]byte(term))
+	if len(tokenStream) == 1 {
+		return string(tokenStream[0].Term)
+	}
+	return ""
+}

vendor/github.com/blevesearch/bleve/v2/error.go

@@ -27,6 +27,7 @@ const (
 	ErrorEmptyID
 	ErrorIndexReadInconsistency
 	ErrorTwoPhaseSearchInconsistency
+	ErrorSynonymSearchNotSupported
 )
 
 // Error represents a more strongly typed bleve error for detecting
@@ -49,4 +50,5 @@ var errorMessages = map[Error]string{
 	ErrorEmptyID:                     "document ID cannot be empty",
 	ErrorIndexReadInconsistency:      "index read inconsistency detected",
 	ErrorTwoPhaseSearchInconsistency: "2-phase search failed, likely due to an overlapping topology change",
+	ErrorSynonymSearchNotSupported:   "synonym search not supported",
 }

vendor/github.com/blevesearch/bleve/v2/geo/README.md

@@ -1,4 +1,4 @@
-# geo support in bleve
+# Geo spatial search support in bleve
 
 Latest bleve spatial capabilities are powered by spatial hierarchical tokens generated from s2geometry.
 You can find more details about the [s2geometry basics here](http://s2geometry.io/), and explore the 

vendor/github.com/blevesearch/bleve/v2/geo/geo.go

@@ -139,7 +139,7 @@ func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, f
 
 	var minLonL, maxLonL float64
 	if minLatL > minLatRad && maxLatL < maxLatRad {
-		deltaLon := asin(sin(radDistance) / cos(radLat))
+		deltaLon := math.Asin(math.Sin(radDistance) / math.Cos(radLat))
 		minLonL = radLon - deltaLon
 		if minLonL < minLonRad {
 			minLonL += 2 * math.Pi

vendor/github.com/blevesearch/bleve/v2/geo/geo_dist.go

@@ -88,11 +88,11 @@ func ParseDistanceUnit(u string) (float64, error) {
 func Haversin(lon1, lat1, lon2, lat2 float64) float64 {
 	x1 := lat1 * degreesToRadian
 	x2 := lat2 * degreesToRadian
-	h1 := 1 - cos(x1-x2)
-	h2 := 1 - cos((lon1-lon2)*degreesToRadian)
-	h := (h1 + cos(x1)*cos(x2)*h2) / 2
+	h1 := 1 - math.Cos(x1-x2)
+	h2 := 1 - math.Cos((lon1-lon2)*degreesToRadian)
+	h := (h1 + math.Cos(x1)*math.Cos(x2)*h2) / 2
 	avgLat := (x1 + x2) / 2
 	diameter := earthDiameter(avgLat)
 
-	return diameter * asin(math.Min(1, math.Sqrt(h)))
+	return diameter * math.Asin(math.Min(1, math.Sqrt(h)))
 }

vendor/github.com/blevesearch/bleve/v2/geo/parse.go

@@ -236,14 +236,19 @@ func extract2DCoordinates(thing interface{}) [][]float64 {
 
 func extract3DCoordinates(thing interface{}) (c [][][]float64) {
 	coords := reflect.ValueOf(thing)
-	for i := 0; i < coords.Len(); i++ {
-		vals := coords.Index(i)
+	if !coords.IsValid() {
+		return nil
+	}
 
-		edges := vals.Interface()
-		if es, ok := edges.([]interface{}); ok {
-			loop := extract2DCoordinates(es)
-			if len(loop) > 0 {
-				c = append(c, loop)
+	if coords.Kind() == reflect.Slice {
+		for i := 0; i < coords.Len(); i++ {
+			vals := coords.Index(i)
+			edges := vals.Interface()
+			if es, ok := edges.([]interface{}); ok {
+				loop := extract2DCoordinates(es)
+				if len(loop) > 0 {
+					c = append(c, loop)
+				}
 			}
 		}
 	}

vendor/github.com/blevesearch/bleve/v2/geo/sloppy.go

@@ -19,104 +19,16 @@ import (
 )
 
 var earthDiameterPerLatitude []float64
-var sinTab []float64
-var cosTab []float64
-var asinTab []float64
-var asinDer1DivF1Tab []float64
-var asinDer2DivF2Tab []float64
-var asinDer3DivF3Tab []float64
-var asinDer4DivF4Tab []float64
 
-const radiusTabsSize = (1 << 10) + 1
-const radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1)
-const radiusIndexer = 1 / radiusDelta
-const sinCosTabsSize = (1 << 11) + 1
-const asinTabsSize = (1 << 13) + 1
-const oneDivF2 = 1 / 2.0
-const oneDivF3 = 1 / 6.0
-const oneDivF4 = 1 / 24.0
-
-// 1.57079632673412561417e+00 first 33 bits of pi/2
-var pio2Hi = math.Float64frombits(0x3FF921FB54400000)
-
-// 6.07710050650619224932e-11 pi/2 - PIO2_HI
-var pio2Lo = math.Float64frombits(0x3DD0B4611A626331)
-
-var asinPio2Hi = math.Float64frombits(0x3FF921FB54442D18) // 1.57079632679489655800e+00
-var asinPio2Lo = math.Float64frombits(0x3C91A62633145C07) // 6.12323399573676603587e-17
-var asinPs0 = math.Float64frombits(0x3fc5555555555555)    //  1.66666666666666657415e-01
-var asinPs1 = math.Float64frombits(0xbfd4d61203eb6f7d)    // -3.25565818622400915405e-01
-var asinPs2 = math.Float64frombits(0x3fc9c1550e884455)    //  2.01212532134862925881e-01
-var asinPs3 = math.Float64frombits(0xbfa48228b5688f3b)    // -4.00555345006794114027e-02
-var asinPs4 = math.Float64frombits(0x3f49efe07501b288)    //  7.91534994289814532176e-04
-var asinPs5 = math.Float64frombits(0x3f023de10dfdf709)    //  3.47933107596021167570e-05
-var asinQs1 = math.Float64frombits(0xc0033a271c8a2d4b)    // -2.40339491173441421878e+00
-var asinQs2 = math.Float64frombits(0x40002ae59c598ac8)    //  2.02094576023350569471e+00
-var asinQs3 = math.Float64frombits(0xbfe6066c1b8d0159)    // -6.88283971605453293030e-01
-var asinQs4 = math.Float64frombits(0x3fb3b8c5b12e9282)    //  7.70381505559019352791e-02
-
-var twoPiHi = 4 * pio2Hi
-var twoPiLo = 4 * pio2Lo
-var sinCosDeltaHi = twoPiHi/sinCosTabsSize - 1
-var sinCosDeltaLo = twoPiLo/sinCosTabsSize - 1
-var sinCosIndexer = 1 / (sinCosDeltaHi + sinCosDeltaLo)
-var sinCosMaxValueForIntModulo = ((math.MaxInt64 >> 9) / sinCosIndexer) * 0.99
-var asinMaxValueForTabs = math.Sin(73.0 * degreesToRadian)
-
-var asinDelta = asinMaxValueForTabs / (asinTabsSize - 1)
-var asinIndexer = 1 / asinDelta
+const (
+	radiusTabsSize = (1 << 10) + 1
+	radiusDelta    = (math.Pi / 2) / (radiusTabsSize - 1)
+	radiusIndexer  = 1 / radiusDelta
+)
 
 func init() {
 	// initializes the tables used for the sloppy math functions
 
-	// sin and cos
-	sinTab = make([]float64, sinCosTabsSize)
-	cosTab = make([]float64, sinCosTabsSize)
-	sinCosPiIndex := (sinCosTabsSize - 1) / 2
-	sinCosPiMul2Index := 2 * sinCosPiIndex
-	sinCosPiMul05Index := sinCosPiIndex / 2
-	sinCosPiMul15Index := 3 * sinCosPiIndex / 2
-	for i := 0; i < sinCosTabsSize; i++ {
-		// angle: in [0,2*PI].
-		angle := float64(i)*sinCosDeltaHi + float64(i)*sinCosDeltaLo
-		sinAngle := math.Sin(angle)
-		cosAngle := math.Cos(angle)
-		// For indexes corresponding to null cosine or sine, we make sure the value is zero
-		// and not an epsilon. This allows for a much better accuracy for results close to zero.
-		if i == sinCosPiIndex {
-			sinAngle = 0.0
-		} else if i == sinCosPiMul2Index {
-			sinAngle = 0.0
-		} else if i == sinCosPiMul05Index {
-			sinAngle = 0.0
-		} else if i == sinCosPiMul15Index {
-			sinAngle = 0.0
-		}
-		sinTab[i] = sinAngle
-		cosTab[i] = cosAngle
-	}
-
-	// asin
-	asinTab = make([]float64, asinTabsSize)
-	asinDer1DivF1Tab = make([]float64, asinTabsSize)
-	asinDer2DivF2Tab = make([]float64, asinTabsSize)
-	asinDer3DivF3Tab = make([]float64, asinTabsSize)
-	asinDer4DivF4Tab = make([]float64, asinTabsSize)
-	for i := 0; i < asinTabsSize; i++ {
-		// x: in [0,ASIN_MAX_VALUE_FOR_TABS].
-		x := float64(i) * asinDelta
-		asinTab[i] = math.Asin(x)
-		oneMinusXSqInv := 1.0 / (1 - x*x)
-		oneMinusXSqInv05 := math.Sqrt(oneMinusXSqInv)
-		oneMinusXSqInv15 := oneMinusXSqInv05 * oneMinusXSqInv
-		oneMinusXSqInv25 := oneMinusXSqInv15 * oneMinusXSqInv
-		oneMinusXSqInv35 := oneMinusXSqInv25 * oneMinusXSqInv
-		asinDer1DivF1Tab[i] = oneMinusXSqInv05
-		asinDer2DivF2Tab[i] = (x * oneMinusXSqInv15) * oneDivF2
-		asinDer3DivF3Tab[i] = ((1 + 2*x*x) * oneMinusXSqInv25) * oneDivF3
-		asinDer4DivF4Tab[i] = ((5 + 2*x*(2+x*(5-2*x))) * oneMinusXSqInv35) * oneDivF4
-	}
-
 	// earth radius
 	a := 6378137.0
 	b := 6356752.31420
@@ -145,68 +57,3 @@ func earthDiameter(lat float64) float64 {
 	}
 	return earthDiameterPerLatitude[int(index)]
 }
-
-var pio2 = math.Pi / 2
-
-func sin(a float64) float64 {
-	return cos(a - pio2)
-}
-
-// cos is a sloppy math (faster) implementation of math.Cos
-func cos(a float64) float64 {
-	if a < 0.0 {
-		a = -a
-	}
-	if a > sinCosMaxValueForIntModulo {
-		return math.Cos(a)
-	}
-	// index: possibly outside tables range.
-	index := int(a*sinCosIndexer + 0.5)
-	delta := (a - float64(index)*sinCosDeltaHi) - float64(index)*sinCosDeltaLo
-	// Making sure index is within tables range.
-	// Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo.
-	index &= (sinCosTabsSize - 2) // index % (SIN_COS_TABS_SIZE-1)
-	indexCos := cosTab[index]
-	indexSin := sinTab[index]
-	return indexCos + delta*(-indexSin+delta*(-indexCos*oneDivF2+delta*(indexSin*oneDivF3+delta*indexCos*oneDivF4)))
-}
-
-// asin is a sloppy math (faster) implementation of math.Asin
-func asin(a float64) float64 {
-	var negateResult bool
-	if a < 0 {
-		a = -a
-		negateResult = true
-	}
-	if a <= asinMaxValueForTabs {
-		index := int(a*asinIndexer + 0.5)
-		delta := a - float64(index)*asinDelta
-		result := asinTab[index] + delta*(asinDer1DivF1Tab[index]+delta*(asinDer2DivF2Tab[index]+delta*(asinDer3DivF3Tab[index]+delta*asinDer4DivF4Tab[index])))
-		if negateResult {
-			return -result
-		}
-		return result
-	}
-	// value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN
-	// This part is derived from fdlibm.
-	if a < 1 {
-		t := (1.0 - a) * 0.5
-		p := t * (asinPs0 + t*(asinPs1+t*(asinPs2+t*(asinPs3+t*(asinPs4+t+asinPs5)))))
-		q := 1.0 + t*(asinQs1+t*(asinQs2+t*(asinQs3+t*asinQs4)))
-		s := math.Sqrt(t)
-		z := s + s*(p/q)
-		result := asinPio2Hi - ((z + z) - asinPio2Lo)
-		if negateResult {
-			return -result
-		}
-		return result
-	}
-	// value >= 1.0, or value is NaN
-	if a == 1.0 {
-		if negateResult {
-			return -math.Pi / 2
-		}
-		return math.Pi / 2
-	}
-	return math.NaN()
-}

vendor/github.com/blevesearch/bleve/v2/index.go

@@ -16,6 +16,7 @@ package bleve
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/blevesearch/bleve/v2/index/upsidedown"
 
@@ -63,6 +64,36 @@ func (b *Batch) Index(id string, data interface{}) error {
 	return nil
 }
 
+func (b *Batch) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {
+	if id == "" {
+		return ErrorEmptyID
+	}
+	if eventIndex, ok := b.index.(index.EventIndex); ok {
+		eventIndex.FireIndexEvent()
+	}
+	synMap, ok := b.index.Mapping().(mapping.SynonymMapping)
+	if !ok {
+		return ErrorSynonymSearchNotSupported
+	}
+
+	if err := definition.Validate(); err != nil {
+		return err
+	}
+
+	doc := document.NewSynonymDocument(id)
+	err := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms)
+	if err != nil {
+		return err
+	}
+	b.internal.Update(doc)
+
+	b.lastDocSize = uint64(doc.Size() +
+		len(id) + size.SizeOfString) // overhead from internal
+	b.totalSize += b.lastDocSize
+
+	return nil
+}
+
 func (b *Batch) LastDocSize() uint64 {
 	return b.lastDocSize
 }
@@ -323,3 +354,35 @@ type IndexCopyable interface {
 // FileSystemDirectory is the default implementation for the
 // index.Directory interface.
 type FileSystemDirectory string
+
+// SynonymDefinition represents a synonym mapping in Bleve.
+// Each instance associates one or more input terms with a list of synonyms,
+// defining how terms are treated as equivalent in searches.
+type SynonymDefinition struct {
+	// Input is an optional list of terms for unidirectional synonym mapping.
+	// When terms are specified in Input, they will map to the terms in Synonyms,
+	// making the relationship unidirectional (each Input maps to all Synonyms).
+	// If Input is omitted, the relationship is bidirectional among all Synonyms.
+	Input []string `json:"input,omitempty"`
+
+	// Synonyms is a list of terms that are considered equivalent.
+	// If Input is specified, each term in Input will map to each term in Synonyms.
+	// If Input is not specified, the Synonyms list will be treated bidirectionally,
+	// meaning each term in Synonyms is treated as synonymous with all others.
+	Synonyms []string `json:"synonyms"`
+}
+
+func (sd *SynonymDefinition) Validate() error {
+	if len(sd.Synonyms) == 0 {
+		return fmt.Errorf("synonym definition must have at least one synonym")
+	}
+	return nil
+}
+
+// SynonymIndex supports indexing synonym definitions alongside regular documents.
+// Synonyms, grouped by collection name, define term relationships for query expansion in searches.
+type SynonymIndex interface {
+	Index
+	// IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection.
+	IndexSynonym(id string, collection string, definition *SynonymDefinition) error
+}

vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go

@@ -19,7 +19,7 @@ import (
 	"os"
 	"sync"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 	bolt "go.etcd.io/bbolt"
@@ -303,7 +303,7 @@ func (o *Builder) Close() error {
 	}
 
 	// fill the root bolt with this fake index snapshot
-	_, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil)
+	_, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil, nil)
 	if err != nil {
 		_ = tx.Rollback()
 		_ = rootBolt.Close()

vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go

@@ -19,7 +19,7 @@ import (
 	"path/filepath"
 	"sync/atomic"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 )
@@ -352,33 +352,39 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
 		creator:  "introduceMerge",
 	}
 
-	// iterate through current segments
-	newSegmentDeleted := roaring.NewBitmap()
 	var running, docsToPersistCount, memSegments, fileSegments uint64
 	var droppedSegmentFiles []string
+	newSegmentDeleted := make([]*roaring.Bitmap, len(nextMerge.new))
+	for i := range newSegmentDeleted {
+		// create a bitmaps to track the obsoletes per newly merged segments
+		newSegmentDeleted[i] = roaring.NewBitmap()
+	}
+
+	// iterate through current segments
 	for i := range root.segment {
 		segmentID := root.segment[i].id
-		if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok {
+		if segSnapAtMerge, ok := nextMerge.mergedSegHistory[segmentID]; ok {
 			// this segment is going away, see if anything else was deleted since we started the merge
 			if segSnapAtMerge != nil && root.segment[i].deleted != nil {
 				// assume all these deletes are new
 				deletedSince := root.segment[i].deleted
 				// if we already knew about some of them, remove
-				if segSnapAtMerge.deleted != nil {
-					deletedSince = roaring.AndNot(root.segment[i].deleted, segSnapAtMerge.deleted)
+				if segSnapAtMerge.oldSegment.deleted != nil {
+					deletedSince = roaring.AndNot(root.segment[i].deleted, segSnapAtMerge.oldSegment.deleted)
 				}
 				deletedSinceItr := deletedSince.Iterator()
 				for deletedSinceItr.HasNext() {
 					oldDocNum := deletedSinceItr.Next()
-					newDocNum := nextMerge.oldNewDocNums[segmentID][oldDocNum]
-					newSegmentDeleted.Add(uint32(newDocNum))
+					newDocNum := segSnapAtMerge.oldNewDocIDs[oldDocNum]
+					newSegmentDeleted[segSnapAtMerge.workerID].Add(uint32(newDocNum))
 				}
 			}
+
 			// clean up the old segment map to figure out the
 			// obsolete segments wrt root in meantime, whatever
 			// segments left behind in old map after processing
 			// the root segments would be the obsolete segment set
-			delete(nextMerge.old, segmentID)
+			delete(nextMerge.mergedSegHistory, segmentID)
 		} else if root.segment[i].LiveSize() > 0 {
 			// this segment is staying
 			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
@@ -410,52 +416,59 @@ func (s *Scorch) introduceMerge(nextMerge *segmentMerge) {
 	// before the newMerge introduction, need to clean the newly
 	// merged segment wrt the current root segments, hence
 	// applying the obsolete segment contents to newly merged segment
-	for segID, ss := range nextMerge.old {
-		obsoleted := ss.DocNumbersLive()
+	for _, ss := range nextMerge.mergedSegHistory {
+		obsoleted := ss.oldSegment.DocNumbersLive()
 		if obsoleted != nil {
 			obsoletedIter := obsoleted.Iterator()
 			for obsoletedIter.HasNext() {
 				oldDocNum := obsoletedIter.Next()
-				newDocNum := nextMerge.oldNewDocNums[segID][oldDocNum]
-				newSegmentDeleted.Add(uint32(newDocNum))
+				newDocNum := ss.oldNewDocIDs[oldDocNum]
+				newSegmentDeleted[ss.workerID].Add(uint32(newDocNum))
 			}
 		}
 	}
-	var skipped bool
-	// In case where all the docs in the newly merged segment getting
-	// deleted by the time we reach here, can skip the introduction.
-	if nextMerge.new != nil &&
-		nextMerge.new.Count() > newSegmentDeleted.GetCardinality() {
 
-		stats := newFieldStats()
-		if fsr, ok := nextMerge.new.(segment.FieldStatsReporter); ok {
-			fsr.UpdateFieldStats(stats)
+	skipped := true
+	// make the newly merged segments part of the newSnapshot being constructed
+	for i, newMergedSegment := range nextMerge.new {
+		// checking if this newly merged segment is worth keeping based on
+		// obsoleted doc count since the merge intro started
+		if newMergedSegment != nil &&
+			newMergedSegment.Count() > newSegmentDeleted[i].GetCardinality() {
+			stats := newFieldStats()
+			if fsr, ok := newMergedSegment.(segment.FieldStatsReporter); ok {
+				fsr.UpdateFieldStats(stats)
+			}
+
+			// put the merged segment at the end of newSnapshot
+			newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
+				id:         nextMerge.id[i],
+				segment:    newMergedSegment, // take ownership for nextMerge.new's ref-count
+				deleted:    newSegmentDeleted[i],
+				stats:      stats,
+				cachedDocs: &cachedDocs{cache: nil},
+				cachedMeta: &cachedMeta{meta: nil},
+				creator:    "introduceMerge",
+				mmaped:     nextMerge.mmaped,
+			})
+			newSnapshot.offsets = append(newSnapshot.offsets, running)
+			running += newMergedSegment.Count()
+
+			switch newMergedSegment.(type) {
+			case segment.PersistedSegment:
+				fileSegments++
+			default:
+				docsToPersistCount += newMergedSegment.Count() - newSegmentDeleted[i].GetCardinality()
+				memSegments++
+			}
+			skipped = false
 		}
+	}
 
-		// put new segment at end
-		newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{
-			id:         nextMerge.id,
-			segment:    nextMerge.new, // take ownership for nextMerge.new's ref-count
-			deleted:    newSegmentDeleted,
-			stats:      stats,
-			cachedDocs: &cachedDocs{cache: nil},
-			cachedMeta: &cachedMeta{meta: nil},
-			creator:    "introduceMerge",
-			mmaped:     nextMerge.mmaped,
-		})
-		newSnapshot.offsets = append(newSnapshot.offsets, running)
-		atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, 1)
-
-		switch nextMerge.new.(type) {
-		case segment.PersistedSegment:
-			fileSegments++
-		default:
-			docsToPersistCount += nextMerge.new.Count() - newSegmentDeleted.GetCardinality()
-			memSegments++
-		}
-	} else {
-		skipped = true
+	if skipped {
 		atomic.AddUint64(&s.stats.TotFileMergeIntroductionsObsoleted, 1)
+	} else {
+		atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, uint64(len(nextMerge.new)))
 	}
 
 	atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount)

vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go

@@ -19,10 +19,11 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/index/scorch/mergeplan"
 	"github.com/blevesearch/bleve/v2/util"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
@@ -208,6 +209,17 @@ func (s *Scorch) ForceMerge(ctx context.Context,
 func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions,
 	error) {
 	mergePlannerOptions := mergeplan.DefaultMergePlanOptions
+
+	po, err := s.parsePersisterOptions()
+	if err != nil {
+		return nil, err
+	}
+	// by default use the MaxSizeInMemoryMergePerWorker from the persister option
+	// as the FloorSegmentFileSize for the merge planner which would be the
+	// first tier size in the planning. If the value is 0, then we don't use the
+	// file size in the planning.
+	mergePlannerOptions.FloorSegmentFileSize = int64(po.MaxSizeInMemoryMergePerWorker)
+
 	if v, ok := s.config["scorchMergePlanOptions"]; ok {
 		b, err := util.MarshalJSON(v)
 		if err != nil {
@@ -305,14 +317,20 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context,
 		newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)
 		segmentsToMerge := make([]segment.Segment, 0, len(task.Segments))
 		docsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments))
+		mergedSegHistory := make(map[uint64]*mergedSegmentHistory, len(task.Segments))
 
 		for _, planSegment := range task.Segments {
 			if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok {
 				oldMap[segSnapshot.id] = segSnapshot
+				mergedSegHistory[segSnapshot.id] = &mergedSegmentHistory{
+					workerID:   0,
+					oldSegment: segSnapshot,
+				}
 				if persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok {
 					if segSnapshot.LiveSize() == 0 {
 						atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1)
 						oldMap[segSnapshot.id] = nil
+						delete(mergedSegHistory, segSnapshot.id)
 					} else {
 						segmentsToMerge = append(segmentsToMerge, segSnapshot.segment)
 						docsToDrop = append(docsToDrop, segSnapshot.deleted)
@@ -327,7 +345,6 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context,
 			}
 		}
 
-		var oldNewDocNums map[uint64][]uint64
 		var seg segment.Segment
 		var filename string
 		if len(segmentsToMerge) > 0 {
@@ -368,21 +385,22 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context,
 			totalBytesRead := seg.BytesRead() + prevBytesReadTotal
 			seg.ResetBytesRead(totalBytesRead)
 
-			oldNewDocNums = make(map[uint64][]uint64, len(newDocNums))
 			for i, segNewDocNums := range newDocNums {
-				oldNewDocNums[task.Segments[i].Id()] = segNewDocNums
+				if mergedSegHistory[task.Segments[i].Id()] != nil {
+					mergedSegHistory[task.Segments[i].Id()].oldNewDocIDs = segNewDocNums
+				}
 			}
 
 			atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge)))
 		}
 
 		sm := &segmentMerge{
-			id:            newSegmentID,
-			old:           oldMap,
-			oldNewDocNums: oldNewDocNums,
-			new:           seg,
-			notifyCh:      make(chan *mergeTaskIntroStatus),
-			mmaped:        1,
+			id:               []uint64{newSegmentID},
+			mergedSegHistory: mergedSegHistory,
+			new:              []segment.Segment{seg},
+			newCount:         seg.Count(),
+			notifyCh:         make(chan *mergeTaskIntroStatus),
+			mmaped:           1,
 		}
 
 		s.fireEvent(EventKindMergeTaskIntroductionStart, 0)
@@ -435,13 +453,22 @@ type mergeTaskIntroStatus struct {
 	skipped       bool
 }
 
+// this is important when it comes to introducing multiple merged segments in a
+// single introducer channel push. That way there is a check to ensure that the
+// file count doesn't explode during the index's lifetime.
+type mergedSegmentHistory struct {
+	workerID     uint64
+	oldNewDocIDs []uint64
+	oldSegment   *SegmentSnapshot
+}
+
 type segmentMerge struct {
-	id            uint64
-	old           map[uint64]*SegmentSnapshot
-	oldNewDocNums map[uint64][]uint64
-	new           segment.Segment
-	notifyCh      chan *mergeTaskIntroStatus
-	mmaped        uint32
+	id               []uint64
+	new              []segment.Segment
+	mergedSegHistory map[uint64]*mergedSegmentHistory
+	notifyCh         chan *mergeTaskIntroStatus
+	mmaped           uint32
+	newCount         uint64
 }
 
 func cumulateBytesRead(sbs []segment.Segment) uint64 {
@@ -452,24 +479,85 @@ func cumulateBytesRead(sbs []segment.Segment) uint64 {
 	return rv
 }
 
-// perform a merging of the given SegmentBase instances into a new,
-// persisted segment, and synchronously introduce that new segment
-// into the root
-func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot,
-	sbs []segment.Segment, sbsDrops []*roaring.Bitmap,
-	sbsIndexes []int) (*IndexSnapshot, uint64, error) {
+func closeNewMergedSegments(segs []segment.Segment) error {
+	for _, seg := range segs {
+		if seg != nil {
+			_ = seg.DecRef()
+		}
+	}
+	return nil
+}
+
+func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObjs []*flushable) (*IndexSnapshot, []uint64, error) {
 	atomic.AddUint64(&s.stats.TotMemMergeBeg, 1)
 
 	memMergeZapStartTime := time.Now()
 
 	atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1)
 
-	newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)
-	filename := zapFileName(newSegmentID)
-	path := s.path + string(os.PathSeparator) + filename
+	var wg sync.WaitGroup
+	// we're tracking the merged segments and their doc number per worker
+	// to be able to introduce them all at once, so the first dimension of the
+	// slices here correspond to workerID
+	newDocIDsSet := make([][][]uint64, len(flushableObjs))
+	newMergedSegments := make([]segment.Segment, len(flushableObjs))
+	newMergedSegmentIDs := make([]uint64, len(flushableObjs))
+	numFlushes := len(flushableObjs)
+	var numSegments, newMergedCount uint64
+	var em sync.Mutex
+	var errs []error
 
-	newDocNums, _, err :=
-		s.segPlugin.Merge(sbs, sbsDrops, path, s.closeCh, s)
+	// deploy the workers to merge and flush the batches of segments parallely
+	for i := 0; i < numFlushes; i++ {
+		wg.Add(1)
+		go func(segsBatch []segment.Segment, dropsBatch []*roaring.Bitmap, id int) {
+			defer wg.Done()
+			newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1)
+			filename := zapFileName(newSegmentID)
+			path := s.path + string(os.PathSeparator) + filename
+
+			// the newly merged segment is already flushed out to disk, just needs
+			// to be opened using mmap.
+			newDocIDs, _, err :=
+				s.segPlugin.Merge(segsBatch, dropsBatch, path, s.closeCh, s)
+			if err != nil {
+				em.Lock()
+				errs = append(errs, err)
+				em.Unlock()
+				atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
+				return
+			}
+			newMergedSegmentIDs[id] = newSegmentID
+			newDocIDsSet[id] = newDocIDs
+			newMergedSegments[id], err = s.segPlugin.Open(path)
+			if err != nil {
+				em.Lock()
+				errs = append(errs, err)
+				em.Unlock()
+				atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
+				return
+			}
+			atomic.AddUint64(&newMergedCount, newMergedSegments[id].Count())
+			atomic.AddUint64(&numSegments, uint64(len(segsBatch)))
+		}(flushableObjs[i].segments, flushableObjs[i].drops, i)
+	}
+	wg.Wait()
+
+	if errs != nil {
+		// close the new merged segments
+		_ = closeNewMergedSegments(newMergedSegments)
+		var errf error
+		for _, err := range errs {
+			if err == segment.ErrClosed {
+				// the index snapshot was closed which will be handled gracefully
+				// by retrying the whole merge+flush operation in a later iteration
+				// so its safe to early exit the same error.
+				return nil, nil, err
+			}
+			errf = fmt.Errorf("%w; %v", errf, err)
+		}
+		return nil, nil, errf
+	}
 
 	atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1)
 
@@ -479,39 +567,30 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot,
 		atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime)
 	}
 
-	if err != nil {
-		atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
-		return nil, 0, err
-	}
-
-	seg, err := s.segPlugin.Open(path)
-	if err != nil {
-		atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
-		return nil, 0, err
-	}
-
-	// update persisted stats
-	atomic.AddUint64(&s.stats.TotPersistedItems, seg.Count())
-	atomic.AddUint64(&s.stats.TotPersistedSegments, 1)
-
 	sm := &segmentMerge{
-		id:            newSegmentID,
-		old:           make(map[uint64]*SegmentSnapshot, len(sbsIndexes)),
-		oldNewDocNums: make(map[uint64][]uint64, len(sbsIndexes)),
-		new:           seg,
-		notifyCh:      make(chan *mergeTaskIntroStatus),
+		id:               newMergedSegmentIDs,
+		new:              newMergedSegments,
+		mergedSegHistory: make(map[uint64]*mergedSegmentHistory, numSegments),
+		notifyCh:         make(chan *mergeTaskIntroStatus),
+		newCount:         newMergedCount,
 	}
 
-	for i, idx := range sbsIndexes {
-		ss := snapshot.segment[idx]
-		sm.old[ss.id] = ss
-		sm.oldNewDocNums[ss.id] = newDocNums[i]
+	for i, flushable := range flushableObjs {
+		for j, idx := range flushable.sbIdxs {
+			ss := snapshot.segment[idx]
+			// oldSegmentSnapshot.id -> {workerID, oldSegmentSnapshot, docIDs}
+			sm.mergedSegHistory[ss.id] = &mergedSegmentHistory{
+				workerID:     uint64(i),
+				oldNewDocIDs: newDocIDsSet[i][j],
+				oldSegment:   ss,
+			}
+		}
 	}
 
 	select { // send to introducer
 	case <-s.closeCh:
-		_ = seg.DecRef()
-		return nil, 0, segment.ErrClosed
+		_ = closeNewMergedSegments(newMergedSegments)
+		return nil, nil, segment.ErrClosed
 	case s.merges <- sm:
 	}
 
@@ -520,17 +599,17 @@ func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot,
 	introStatus := <-sm.notifyCh
 	if introStatus != nil && introStatus.indexSnapshot != nil {
 		newSnapshot = introStatus.indexSnapshot
-		atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(len(sbs)))
+		atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(numSegments))
 		atomic.AddUint64(&s.stats.TotMemMergeDone, 1)
 		if introStatus.skipped {
 			// close the segment on skipping introduction.
 			_ = newSnapshot.DecRef()
-			_ = seg.Close()
+			_ = closeNewMergedSegments(newMergedSegments)
 			newSnapshot = nil
 		}
 	}
 
-	return newSnapshot, newSegmentID, nil
+	return newSnapshot, newMergedSegmentIDs, nil
 }
 
 func (s *Scorch) ReportBytesWritten(bytesWritten uint64) {

vendor/github.com/blevesearch/bleve/v2/index/scorch/mergeplan/merge_plan.go

@@ -99,6 +99,10 @@ type MergePlanOptions struct {
 	// of tiny segments from resulting in a long tail in the index.
 	FloorSegmentSize int64
 
+	// Small segments' file size are rounded up to this size to prevent lot
+	// of tiny segments causing a long tail in the index.
+	FloorSegmentFileSize int64
+
 	// Controls how aggressively merges that reclaim more deletions
 	// are favored.  Higher values will more aggressively target
 	// merges that reclaim deletions, but be careful not to go so high
@@ -126,6 +130,13 @@ func (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 {
 	return o.FloorSegmentSize
 }
 
+func (o *MergePlanOptions) RaiseToFloorSegmentFileSize(s int64) int64 {
+	if s > o.FloorSegmentFileSize {
+		return s
+	}
+	return o.FloorSegmentFileSize
+}
+
 // MaxSegmentSizeLimit represents the maximum size of a segment,
 // this limit comes with hit-1 optimisation/max encoding limit uint31.
 const MaxSegmentSizeLimit = 1<<31 - 1
@@ -155,6 +166,7 @@ var SingleSegmentMergePlanOptions = MergePlanOptions{
 	SegmentsPerMergeTask: 10,
 	FloorSegmentSize:     1 << 30,
 	ReclaimDeletesWeight: 2.0,
+	FloorSegmentFileSize: 1 << 40,
 }
 
 // -------------------------------------------
@@ -176,12 +188,18 @@ func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
 
 	var eligibles []Segment
 	var eligiblesLiveSize int64
+	var eligiblesFileSize int64
+	var minFileSize int64 = math.MaxInt64
 
 	for _, segment := range segments {
 		if minLiveSize > segment.LiveSize() {
 			minLiveSize = segment.LiveSize()
 		}
 
+		if minFileSize > segment.FileSize() {
+			minFileSize = segment.FileSize()
+		}
+
 		isEligible := segment.LiveSize() < o.MaxSegmentSize/2
 		// An eligible segment (based on #documents) may be too large
 		// and thus need a stricter check based on the file size.
@@ -195,17 +213,24 @@ func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) {
 		if isEligible {
 			eligibles = append(eligibles, segment)
 			eligiblesLiveSize += segment.LiveSize()
+			eligiblesFileSize += segment.FileSize()
 		}
 	}
 
-	minLiveSize = o.RaiseToFloorSegmentSize(minLiveSize)
-
 	calcBudget := o.CalcBudget
 	if calcBudget == nil {
 		calcBudget = CalcBudget
 	}
 
-	budgetNumSegments := calcBudget(eligiblesLiveSize, minLiveSize, o)
+	var budgetNumSegments int
+	if o.FloorSegmentFileSize > 0 {
+		minFileSize = o.RaiseToFloorSegmentFileSize(minFileSize)
+		budgetNumSegments = calcBudget(eligiblesFileSize, minFileSize, o)
+
+	} else {
+		minLiveSize = o.RaiseToFloorSegmentSize(minLiveSize)
+		budgetNumSegments = calcBudget(eligiblesLiveSize, minLiveSize, o)
+	}
 
 	scoreSegments := o.ScoreSegments
 	if scoreSegments == nil {

vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 )

vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize_knn.go

@@ -23,7 +23,6 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/v2/search"
 	index "github.com/blevesearch/bleve_index_api"
 	segment_api "github.com/blevesearch/scorch_segment_api/v2"
@@ -65,11 +64,6 @@ func (o *OptimizeVR) Finish() error {
 	var errorsM sync.Mutex
 	var errors []error
 
-	var snapshotGlobalDocNums map[int]*roaring.Bitmap
-	if o.requiresFiltering {
-		snapshotGlobalDocNums = o.snapshot.globalDocNums()
-	}
-
 	defer o.invokeSearcherEndCallback()
 
 	wg := sync.WaitGroup{}
@@ -104,27 +98,12 @@ func (o *OptimizeVR) Finish() error {
 						// for each VR, populate postings list and iterators
 						// by passing the obtained vector index and getting similar vectors.
 
-						// Only applies to filtered kNN.
-						if vr.eligibleDocIDs != nil && len(vr.eligibleDocIDs) > 0 {
-							eligibleVectorInternalIDs := vr.getEligibleDocIDs()
-							if snapshotGlobalDocNums != nil {
-								// Only the eligible documents belonging to this segment
-								// will get filtered out.
-								// There is no way to determine which doc belongs to which segment
-								eligibleVectorInternalIDs.And(snapshotGlobalDocNums[index])
-							}
-
-							eligibleLocalDocNums := make([]uint64,
-								eligibleVectorInternalIDs.GetCardinality())
-							// get the (segment-)local document numbers
-							for i, docNum := range eligibleVectorInternalIDs.ToArray() {
-								localDocNum := o.snapshot.localDocNumFromGlobal(index,
-									uint64(docNum))
-								eligibleLocalDocNums[i] = localDocNum
-							}
-
+						// check if the vector reader is configured to use a pre-filter
+						// to filter out ineligible documents before performing
+						// kNN search.
+						if vr.eligibleSelector != nil {
 							pl, err = vecIndex.SearchWithFilter(vr.vector, vr.k,
-								eligibleLocalDocNums, vr.searchParams)
+								vr.eligibleSelector.SegmentEligibleDocs(index), vr.searchParams)
 						} else {
 							pl, err = vecIndex.Search(vr.vector, vr.k, vr.searchParams)
 						}
@@ -178,7 +157,7 @@ func (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context,
 	}
 	o.ctx = ctx
 	if !o.requiresFiltering {
-		o.requiresFiltering = len(s.eligibleDocIDs) > 0
+		o.requiresFiltering = s.eligibleSelector != nil
 	}
 
 	if o.snapshot != s.snapshot {

vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go

@@ -24,13 +24,14 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
@@ -79,6 +80,14 @@ type persisterOptions struct {
 	// for the number of paused application threads. The default value would
 	// be a very high number to always favour the merging of memory segments.
 	MemoryPressurePauseThreshold uint64
+
+	// NumPersisterWorkers decides the number of parallel workers that will
+	// perform the in-memory merge of segments followed by a flush operation.
+	NumPersisterWorkers int
+
+	// MaxSizeInMemoryMerge is the maximum size of data that a single persister
+	// worker is allowed to work on
+	MaxSizeInMemoryMergePerWorker int
 }
 
 type notificationChan chan struct{}
@@ -240,7 +249,8 @@ OUTER:
 }
 
 func notifyMergeWatchers(lastPersistedEpoch uint64,
-	persistWatchers []*epochWatcher) []*epochWatcher {
+	persistWatchers []*epochWatcher,
+) []*epochWatcher {
 	var watchersNext []*epochWatcher
 	for _, w := range persistWatchers {
 		if w.epoch < lastPersistedEpoch {
@@ -254,8 +264,8 @@ func notifyMergeWatchers(lastPersistedEpoch uint64,
 
 func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64,
 	lastMergedEpoch uint64, persistWatchers []*epochWatcher,
-	po *persisterOptions) (uint64, []*epochWatcher) {
-
+	po *persisterOptions,
+) (uint64, []*epochWatcher) {
 	// First, let the watchers proceed if they lag behind
 	persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers)
 
@@ -320,9 +330,11 @@ OUTER:
 
 func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) {
 	po := persisterOptions{
-		PersisterNapTimeMSec:         DefaultPersisterNapTimeMSec,
-		PersisterNapUnderNumFiles:    DefaultPersisterNapUnderNumFiles,
-		MemoryPressurePauseThreshold: DefaultMemoryPressurePauseThreshold,
+		PersisterNapTimeMSec:          DefaultPersisterNapTimeMSec,
+		PersisterNapUnderNumFiles:     DefaultPersisterNapUnderNumFiles,
+		MemoryPressurePauseThreshold:  DefaultMemoryPressurePauseThreshold,
+		NumPersisterWorkers:           DefaultNumPersisterWorkers,
+		MaxSizeInMemoryMergePerWorker: DefaultMaxSizeInMemoryMergePerWorker,
 	}
 	if v, ok := s.config["scorchPersisterOptions"]; ok {
 		b, err := util.MarshalJSON(v)
@@ -339,12 +351,13 @@ func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) {
 }
 
 func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot,
-	po *persisterOptions) error {
+	po *persisterOptions,
+) error {
 	// Perform in-memory segment merging only when the memory pressure is
 	// below the configured threshold, else the persister performs the
 	// direct persistence of segments.
 	if s.NumEventsBlocking() < po.MemoryPressurePauseThreshold {
-		persisted, err := s.persistSnapshotMaybeMerge(snapshot)
+		persisted, err := s.persistSnapshotMaybeMerge(snapshot, po)
 		if err != nil {
 			return err
 		}
@@ -353,7 +366,7 @@ func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot,
 		}
 	}
 
-	return s.persistSnapshotDirect(snapshot)
+	return s.persistSnapshotDirect(snapshot, nil)
 }
 
 // DefaultMinSegmentsForInMemoryMerge represents the default number of
@@ -362,32 +375,118 @@ func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot,
 // those segments
 var DefaultMinSegmentsForInMemoryMerge = 2
 
+type flushable struct {
+	segments []segment.Segment
+	drops    []*roaring.Bitmap
+	sbIdxs   []int
+	totDocs  uint64
+}
+
+// number workers which parallely perform an in-memory merge of the segments
+// followed by a flush operation.
+var DefaultNumPersisterWorkers = 1
+
+// maximum size of data that a single worker is allowed to perform the in-memory
+// merge operation.
+var DefaultMaxSizeInMemoryMergePerWorker = 0
+
+func legacyFlushBehaviour(maxSizeInMemoryMergePerWorker, numPersisterWorkers int) bool {
+	// DefaultMaxSizeInMemoryMergePerWorker = 0 is a special value to preserve the leagcy
+	// one-shot in-memory merge + flush behaviour.
+	return maxSizeInMemoryMergePerWorker == 0 && numPersisterWorkers == 1
+}
+
 // persistSnapshotMaybeMerge examines the snapshot and might merge and
 // persist the in-memory zap segments if there are enough of them
-func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) (
+func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persisterOptions) (
 	bool, error) {
 	// collect the in-memory zap segments (SegmentBase instances)
 	var sbs []segment.Segment
 	var sbsDrops []*roaring.Bitmap
 	var sbsIndexes []int
+	var oldSegIdxs []int
 
-	for i, segmentSnapshot := range snapshot.segment {
-		if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); !ok {
-			sbs = append(sbs, segmentSnapshot.segment)
-			sbsDrops = append(sbsDrops, segmentSnapshot.deleted)
-			sbsIndexes = append(sbsIndexes, i)
+	flushSet := make([]*flushable, 0)
+	var totSize int
+	var numSegsToFlushOut int
+	var totDocs uint64
+
+	// legacy behaviour of merge + flush of all in-memory segments in one-shot
+	if legacyFlushBehaviour(po.MaxSizeInMemoryMergePerWorker, po.NumPersisterWorkers) {
+		val := &flushable{
+			segments: make([]segment.Segment, 0),
+			drops:    make([]*roaring.Bitmap, 0),
+			sbIdxs:   make([]int, 0),
+			totDocs:  totDocs,
+		}
+		for i, snapshot := range snapshot.segment {
+			if _, ok := snapshot.segment.(segment.PersistedSegment); !ok {
+				val.segments = append(val.segments, snapshot.segment)
+				val.drops = append(val.drops, snapshot.deleted)
+				val.sbIdxs = append(val.sbIdxs, i)
+				oldSegIdxs = append(oldSegIdxs, i)
+				val.totDocs += snapshot.segment.Count()
+				numSegsToFlushOut++
+			}
+		}
+
+		flushSet = append(flushSet, val)
+	} else {
+		// constructs a flushSet where each flushable object contains a set of segments
+		// to be merged and flushed out to disk.
+		for i, snapshot := range snapshot.segment {
+			if totSize >= po.MaxSizeInMemoryMergePerWorker &&
+				len(sbs) >= DefaultMinSegmentsForInMemoryMerge {
+				numSegsToFlushOut += len(sbs)
+				val := &flushable{
+					segments: slices.Clone(sbs),
+					drops:    slices.Clone(sbsDrops),
+					sbIdxs:   slices.Clone(sbsIndexes),
+					totDocs:  totDocs,
+				}
+				flushSet = append(flushSet, val)
+				oldSegIdxs = append(oldSegIdxs, sbsIndexes...)
+
+				sbs, sbsDrops, sbsIndexes = sbs[:0], sbsDrops[:0], sbsIndexes[:0]
+				totSize, totDocs = 0, 0
+			}
+
+			if len(flushSet) >= int(po.NumPersisterWorkers) {
+				break
+			}
+
+			if _, ok := snapshot.segment.(segment.PersistedSegment); !ok {
+				sbs = append(sbs, snapshot.segment)
+				sbsDrops = append(sbsDrops, snapshot.deleted)
+				sbsIndexes = append(sbsIndexes, i)
+				totDocs += snapshot.segment.Count()
+				totSize += snapshot.segment.Size()
+			}
+		}
+		// if there were too few segments just merge them all as part of a single worker
+		if len(flushSet) < po.NumPersisterWorkers {
+			numSegsToFlushOut += len(sbs)
+			val := &flushable{
+				segments: slices.Clone(sbs),
+				drops:    slices.Clone(sbsDrops),
+				sbIdxs:   slices.Clone(sbsIndexes),
+				totDocs:  totDocs,
+			}
+			flushSet = append(flushSet, val)
+			oldSegIdxs = append(oldSegIdxs, sbsIndexes...)
 		}
 	}
 
-	if len(sbs) < DefaultMinSegmentsForInMemoryMerge {
+	if numSegsToFlushOut < DefaultMinSegmentsForInMemoryMerge {
 		return false, nil
 	}
 
-	newSnapshot, newSegmentID, err := s.mergeSegmentBases(
-		snapshot, sbs, sbsDrops, sbsIndexes)
+	// drains out (after merging in memory) the segments in the flushSet parallely
+	newSnapshot, newSegmentIDs, err := s.mergeSegmentBasesParallel(snapshot, flushSet)
 	if err != nil {
 		return false, err
 	}
+
 	if newSnapshot == nil {
 		return false, nil
 	}
@@ -397,10 +496,15 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) (
 	}()
 
 	mergedSegmentIDs := map[uint64]struct{}{}
-	for _, idx := range sbsIndexes {
+	for _, idx := range oldSegIdxs {
 		mergedSegmentIDs[snapshot.segment[idx].id] = struct{}{}
 	}
 
+	newMergedSegmentIDs := make(map[uint64]struct{}, len(newSegmentIDs))
+	for _, id := range newSegmentIDs {
+		newMergedSegmentIDs[id] = struct{}{}
+	}
+
 	// construct a snapshot that's logically equivalent to the input
 	// snapshot, but with merged segments replaced by the new segment
 	equiv := &IndexSnapshot{
@@ -411,18 +515,25 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) (
 		creator:  "persistSnapshotMaybeMerge",
 	}
 
+	// to track which segments haven't participated in the in-memory merge
+	// they won't be flushed out to the disk yet, but in the next cycle will be
+	// merged in-memory and then flushed out - this is to keep the number of
+	// on-disk files in limit.
+	exclude := make(map[uint64]struct{})
+
 	// copy to the equiv the segments that weren't replaced
 	for _, segment := range snapshot.segment {
 		if _, wasMerged := mergedSegmentIDs[segment.id]; !wasMerged {
 			equiv.segment = append(equiv.segment, segment)
+			exclude[segment.id] = struct{}{}
 		}
 	}
 
 	// append to the equiv the new segment
 	for _, segment := range newSnapshot.segment {
-		if segment.id == newSegmentID {
+		if _, ok := newMergedSegmentIDs[segment.id]; ok {
 			equiv.segment = append(equiv.segment, &SegmentSnapshot{
-				id:      newSegmentID,
+				id:      segment.id,
 				segment: segment.segment,
 				deleted: nil, // nil since merging handled deletions
 				stats:   nil,
@@ -431,7 +542,7 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) (
 		}
 	}
 
-	err = s.persistSnapshotDirect(equiv)
+	err = s.persistSnapshotDirect(equiv, exclude)
 	if err != nil {
 		return false, err
 	}
@@ -468,7 +579,8 @@ func copyToDirectory(srcPath string, d index.Directory) (int64, error) {
 }
 
 func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory,
-	path string) error {
+	path string,
+) error {
 	if d == nil {
 		return seg.Persist(path)
 	}
@@ -490,7 +602,7 @@ func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory,
 }
 
 func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
-	segPlugin SegmentPlugin, d index.Directory) (
+	segPlugin SegmentPlugin, exclude map[uint64]struct{}, d index.Directory) (
 	[]string, map[uint64]string, error) {
 	snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket)
 	if err != nil {
@@ -579,19 +691,22 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
 			}
 			filenames = append(filenames, filename)
 		case segment.UnpersistedSegment:
-			// need to persist this to disk
-			filename := zapFileName(segmentSnapshot.id)
-			path := filepath.Join(path, filename)
-			err := persistToDirectory(seg, d, path)
-			if err != nil {
-				return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err)
+			// need to persist this to disk if its not part of exclude list (which
+			// restricts which in-memory segment to be persisted to disk)
+			if _, ok := exclude[segmentSnapshot.id]; !ok {
+				filename := zapFileName(segmentSnapshot.id)
+				path := filepath.Join(path, filename)
+				err := persistToDirectory(seg, d, path)
+				if err != nil {
+					return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err)
+				}
+				newSegmentPaths[segmentSnapshot.id] = path
+				err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename))
+				if err != nil {
+					return nil, nil, err
+				}
+				filenames = append(filenames, filename)
 			}
-			newSegmentPaths[segmentSnapshot.id] = path
-			err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename))
-			if err != nil {
-				return nil, nil, err
-			}
-			filenames = append(filenames, filename)
 		default:
 			return nil, nil, fmt.Errorf("unknown segment type: %T", seg)
 		}
@@ -624,7 +739,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string,
 	return filenames, newSegmentPaths, nil
 }
 
-func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) {
+func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot, exclude map[uint64]struct{}) (err error) {
 	// start a write transaction
 	tx, err := s.rootBolt.Begin(true)
 	if err != nil {
@@ -637,7 +752,7 @@ func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) {
 		}
 	}()
 
-	filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil)
+	filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, exclude, nil)
 	if err != nil {
 		return err
 	}
@@ -713,16 +828,18 @@ func zapFileName(epoch uint64) string {
 
 // bolt snapshot code
 
-var boltSnapshotsBucket = []byte{'s'}
-var boltPathKey = []byte{'p'}
-var boltDeletedKey = []byte{'d'}
-var boltInternalKey = []byte{'i'}
-var boltMetaDataKey = []byte{'m'}
-var boltMetaDataSegmentTypeKey = []byte("type")
-var boltMetaDataSegmentVersionKey = []byte("version")
-var boltMetaDataTimeStamp = []byte("timeStamp")
-var boltStatsKey = []byte("stats")
-var TotBytesWrittenKey = []byte("TotBytesWritten")
+var (
+	boltSnapshotsBucket           = []byte{'s'}
+	boltPathKey                   = []byte{'p'}
+	boltDeletedKey                = []byte{'d'}
+	boltInternalKey               = []byte{'i'}
+	boltMetaDataKey               = []byte{'m'}
+	boltMetaDataSegmentTypeKey    = []byte("type")
+	boltMetaDataSegmentVersionKey = []byte("version")
+	boltMetaDataTimeStamp         = []byte("timeStamp")
+	boltStatsKey                  = []byte("stats")
+	TotBytesWrittenKey            = []byte("TotBytesWritten")
+)
 
 func (s *Scorch) loadFromBolt() error {
 	return s.rootBolt.View(func(tx *bolt.Tx) error {
@@ -800,7 +917,6 @@ func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) {
 }
 
 func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) {
-
 	rv := &IndexSnapshot{
 		parent:   s,
 		internal: make(map[string][]byte),
@@ -947,7 +1063,8 @@ var RollbackSamplingInterval = 0 * time.Minute
 var RollbackRetentionFactor = float64(0.5)
 
 func getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration,
-	snapshots []*snapshotMetaData) (int, map[uint64]time.Time) {
+	snapshots []*snapshotMetaData,
+) (int, map[uint64]time.Time) {
 	if interval == 0 {
 		return len(snapshots), map[uint64]time.Time{}
 	}
@@ -994,8 +1111,8 @@ func getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration,
 // by a time duration of RollbackSamplingInterval.
 func getProtectedSnapshots(rollbackSamplingInterval time.Duration,
 	numSnapshotsToKeep int,
-	persistedSnapshots []*snapshotMetaData) map[uint64]time.Time {
-
+	persistedSnapshots []*snapshotMetaData,
+) map[uint64]time.Time {
 	lastPoint, protectedEpochs := getTimeSeriesSnapshots(numSnapshotsToKeep,
 		rollbackSamplingInterval, persistedSnapshots)
 	if len(protectedEpochs) < numSnapshotsToKeep {
@@ -1162,7 +1279,8 @@ func (s *Scorch) removeOldZapFiles() error {
 // Hence we try to retain atleast retentionFactor portion worth of old snapshots
 // in such a scenario using the following function
 func getBoundaryCheckPoint(retentionFactor float64,
-	checkPoints []*snapshotMetaData, timeStamp time.Time) time.Time {
+	checkPoints []*snapshotMetaData, timeStamp time.Time,
+) time.Time {
 	if checkPoints != nil {
 		boundary := checkPoints[int(math.Floor(float64(len(checkPoints))*
 			retentionFactor))]

vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go

@@ -23,7 +23,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/registry"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
@@ -110,7 +110,8 @@ type internalStats struct {
 
 func NewScorch(storeName string,
 	config map[string]interface{},
-	analysisQueue *index.AnalysisQueue) (index.Index, error) {
+	analysisQueue *index.AnalysisQueue,
+) (index.Index, error) {
 	rv := &Scorch{
 		version:              Version,
 		config:               config,
@@ -137,7 +138,9 @@ func NewScorch(storeName string,
 
 	typ, ok := config["spatialPlugin"].(string)
 	if ok {
-		rv.loadSpatialAnalyzerPlugin(typ)
+		if err := rv.loadSpatialAnalyzerPlugin(typ); err != nil {
+			return nil, err
+		}
 	}
 
 	rv.root = &IndexSnapshot{parent: rv, refs: 1, creator: "NewScorch"}
@@ -157,6 +160,18 @@ func NewScorch(storeName string,
 	if ok {
 		rv.onAsyncError = RegistryAsyncErrorCallbacks[aecbName]
 	}
+	// validate any custom persistor options to
+	// prevent an async error in the persistor routine
+	_, err = rv.parsePersisterOptions()
+	if err != nil {
+		return nil, err
+	}
+	// validate any custom merge planner options to
+	// prevent an async error in the merger routine
+	_, err = rv.parseMergePlannerOptions()
+	if err != nil {
+		return nil, err
+	}
 
 	return rv, nil
 }
@@ -230,7 +245,7 @@ func (s *Scorch) openBolt() error {
 		s.unsafeBatch = true
 	}
 
-	var rootBoltOpt = *bolt.DefaultOptions
+	rootBoltOpt := *bolt.DefaultOptions
 	if s.readOnly {
 		rootBoltOpt.ReadOnly = true
 		rootBoltOpt.OpenFile = func(path string, flag int, mode os.FileMode) (*os.File, error) {
@@ -244,7 +259,7 @@ func (s *Scorch) openBolt() error {
 		}
 	} else {
 		if s.path != "" {
-			err := os.MkdirAll(s.path, 0700)
+			err := os.MkdirAll(s.path, 0o700)
 			if err != nil {
 				return err
 			}
@@ -263,7 +278,7 @@ func (s *Scorch) openBolt() error {
 	rootBoltPath := s.path + string(os.PathSeparator) + "root.bolt"
 	var err error
 	if s.path != "" {
-		s.rootBolt, err = bolt.Open(rootBoltPath, 0600, &rootBoltOpt)
+		s.rootBolt, err = bolt.Open(rootBoltPath, 0o600, &rootBoltOpt)
 		if err != nil {
 			return err
 		}
@@ -325,7 +340,9 @@ func (s *Scorch) openBolt() error {
 
 	typ, ok := s.config["spatialPlugin"].(string)
 	if ok {
-		s.loadSpatialAnalyzerPlugin(typ)
+		if err := s.loadSpatialAnalyzerPlugin(typ); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -424,6 +441,10 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) {
 	for itemsDeQueued < numUpdates {
 		result := <-resultChan
 		resultSize := result.Size()
+		// check if the document is searchable by the index
+		if result.Indexed() {
+			atomic.AddUint64(&s.stats.TotMutationsFiltered, 1)
+		}
 		atomic.AddUint64(&s.iStats.analysisBytesAdded, uint64(resultSize))
 		totalAnalysisSize += resultSize
 		analysisResults[itemsDeQueued] = result
@@ -477,8 +498,8 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) {
 }
 
 func (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string,
-	internalOps map[string][]byte, persistedCallback index.BatchCallback, stats *fieldStats) error {
-
+	internalOps map[string][]byte, persistedCallback index.BatchCallback, stats *fieldStats,
+) error {
 	// new introduction
 	introduction := &segmentIntroduction{
 		id:                atomic.AddUint64(&s.nextSegmentID, 1),
@@ -572,7 +593,8 @@ func (s *Scorch) BytesReadQueryTime() uint64 {
 }
 
 func (s *Scorch) diskFileStats(rootSegmentPaths map[string]struct{}) (uint64,
-	uint64, uint64) {
+	uint64, uint64,
+) {
 	var numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot uint64
 	if s.path != "" {
 		files, err := os.ReadDir(s.path)
@@ -635,6 +657,8 @@ func (s *Scorch) StatsMap() map[string]interface{} {
 	m["term_searchers_started"] = m["TotTermSearchersStarted"]
 	m["term_searchers_finished"] = m["TotTermSearchersFinished"]
 	m["knn_searches"] = m["TotKNNSearches"]
+	m["synonym_searches"] = m["TotSynonymSearches"]
+	m["total_mutations_filtered"] = m["TotMutationsFiltered"]
 
 	m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"]
 	m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"]
@@ -657,6 +681,17 @@ func (s *Scorch) StatsMap() map[string]interface{} {
 	m["num_persister_nap_merger_break"] = m["TotPersisterMergerNapBreak"]
 	m["total_compaction_written_bytes"] = m["TotFileMergeWrittenBytes"]
 
+	// the bool stat `index_bgthreads_active` indicates whether the background routines
+	// (which are responsible for the index to attain a steady state) are still
+	// doing some work.
+	if rootEpoch, ok := m["CurRootEpoch"].(uint64); ok {
+		if lastMergedEpoch, ok := m["LastMergedEpoch"].(uint64); ok {
+			if lastPersistedEpoch, ok := m["LastPersistedEpoch"].(uint64); ok {
+				m["index_bgthreads_active"] = !(lastMergedEpoch == rootEpoch && lastPersistedEpoch == rootEpoch)
+			}
+		}
+	}
+
 	// calculate the aggregate of all the segment's field stats
 	aggFieldStats := newFieldStats()
 	for _, segmentSnapshot := range indexSnapshot.Segments() {
@@ -705,6 +740,27 @@ func analyze(d index.Document, fn customAnalyzerPluginInitFunc) {
 				d.VisitComposite(func(cf index.CompositeField) {
 					cf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies())
 				})
+				// Since the encoded geoShape is only necessary within the doc values
+				// of the geoShapeField, it has been removed from the field's term dictionary.
+				// However, '_all' field uses its term dictionary as its docValues, so it
+				// becomes necessary to add the geoShape into the '_all' field's term dictionary
+				if f, ok := field.(index.GeoShapeField); ok {
+					d.VisitComposite(func(cf index.CompositeField) {
+						geoshape := f.EncodedShape()
+						cf.Compose(field.Name(), 1, index.TokenFrequencies{
+							string(geoshape): &index.TokenFreq{
+								Term: geoshape,
+								Locations: []*index.TokenLocation{
+									{
+										Start:    0,
+										End:      len(geoshape),
+										Position: 1,
+									},
+								},
+							},
+						})
+					})
+				}
 			}
 		}
 	})
@@ -771,7 +827,10 @@ func (s *Scorch) unmarkIneligibleForRemoval(filename string) {
 }
 
 func init() {
-	registry.RegisterIndexType(Name, NewScorch)
+	err := registry.RegisterIndexType(Name, NewScorch)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func parseToTimeDuration(i interface{}) (time.Duration, error) {
@@ -812,7 +871,6 @@ func (fs *fieldStats) Store(statName, fieldName string, value uint64) {
 
 // Combine the given stats map with the existing map
 func (fs *fieldStats) Aggregate(stats segment.FieldStats) {
-
 	statMap := stats.Fetch()
 	if statMap == nil {
 		return

vendor/github.com/blevesearch/bleve/v2/index/scorch/segment_plugin.go

@@ -17,10 +17,9 @@ package scorch
 import (
 	"fmt"
 
-	"github.com/RoaringBitmap/roaring"
-	index "github.com/blevesearch/bleve_index_api"
-
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/geo"
+	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 
 	zapv11 "github.com/blevesearch/zapx/v11"

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go

@@ -26,7 +26,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/document"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
@@ -42,10 +42,11 @@ type asynchSegmentResult struct {
 	dict    segment.TermDictionary
 	dictItr segment.DictionaryIterator
 
-	index int
-	docs  *roaring.Bitmap
+	cardinality int
+	index       int
+	docs        *roaring.Bitmap
 
-	postings segment.PostingsList
+	thesItr segment.ThesaurusIterator
 
 	err error
 }
@@ -133,10 +134,11 @@ func (i *IndexSnapshot) updateSize() {
 
 func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 	makeItr func(i segment.TermDictionary) segment.DictionaryIterator,
-	randomLookup bool) (*IndexSnapshotFieldDict, error) {
-
+	randomLookup bool,
+) (*IndexSnapshotFieldDict, error) {
 	results := make(chan *asynchSegmentResult)
 	var totalBytesRead uint64
+	var fieldCardinality int64
 	for _, s := range is.segment {
 		go func(s *SegmentSnapshot) {
 			dict, err := s.segment.Dictionary(field)
@@ -146,6 +148,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 				if dictStats, ok := dict.(segment.DiskStatsReporter); ok {
 					atomic.AddUint64(&totalBytesRead, dictStats.BytesRead())
 				}
+				atomic.AddInt64(&fieldCardinality, int64(dict.Cardinality()))
 				if randomLookup {
 					results <- &asynchSegmentResult{dict: dict}
 				} else {
@@ -160,6 +163,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 		snapshot: is,
 		cursors:  make([]*segmentDictCursor, 0, len(is.segment)),
 	}
+
 	for count := 0; count < len(is.segment); count++ {
 		asr := <-results
 		if asr.err != nil && err == nil {
@@ -183,6 +187,7 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 			}
 		}
 	}
+	rv.cardinality = int(fieldCardinality)
 	rv.bytesRead = totalBytesRead
 	// after ensuring we've read all items on channel
 	if err != nil {
@@ -225,7 +230,8 @@ func calculateExclusiveEndFromInclusiveEnd(inclusiveEnd []byte) []byte {
 }
 
 func (is *IndexSnapshot) FieldDictRange(field string, startTerm []byte,
-	endTerm []byte) (index.FieldDict, error) {
+	endTerm []byte,
+) (index.FieldDict, error) {
 	return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
 		endTermExclusive := calculateExclusiveEndFromInclusiveEnd(endTerm)
 		return is.AutomatonIterator(nil, startTerm, endTermExclusive)
@@ -251,7 +257,8 @@ func calculateExclusiveEndFromPrefix(in []byte) []byte {
 }
 
 func (is *IndexSnapshot) FieldDictPrefix(field string,
-	termPrefix []byte) (index.FieldDict, error) {
+	termPrefix []byte,
+) (index.FieldDict, error) {
 	termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix)
 	return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
 		return is.AutomatonIterator(nil, termPrefix, termPrefixEnd)
@@ -259,22 +266,41 @@ func (is *IndexSnapshot) FieldDictPrefix(field string,
 }
 
 func (is *IndexSnapshot) FieldDictRegexp(field string,
-	termRegex string) (index.FieldDict, error) {
+	termRegex string,
+) (index.FieldDict, error) {
+	fd, _, err := is.FieldDictRegexpAutomaton(field, termRegex)
+	return fd, err
+}
+
+func (is *IndexSnapshot) FieldDictRegexpAutomaton(field string,
+	termRegex string,
+) (index.FieldDict, index.RegexAutomaton, error) {
+	return is.fieldDictRegexp(field, termRegex)
+}
+
+func (is *IndexSnapshot) fieldDictRegexp(field string,
+	termRegex string,
+) (index.FieldDict, index.RegexAutomaton, error) {
 	// TODO: potential optimization where the literal prefix represents the,
 	//       entire regexp, allowing us to use PrefixIterator(prefixTerm)?
 
 	a, prefixBeg, prefixEnd, err := parseRegexp(termRegex)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
+	fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
 		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
 	}, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	return fd, a, nil
 }
 
 func (is *IndexSnapshot) getLevAutomaton(term string,
-	fuzziness uint8) (vellum.Automaton, error) {
+	fuzziness uint8,
+) (vellum.Automaton, error) {
 	if fuzziness == 1 {
 		return lb1.BuildDfa(term, fuzziness)
 	} else if fuzziness == 2 {
@@ -284,21 +310,41 @@ func (is *IndexSnapshot) getLevAutomaton(term string,
 }
 
 func (is *IndexSnapshot) FieldDictFuzzy(field string,
-	term string, fuzziness int, prefix string) (index.FieldDict, error) {
+	term string, fuzziness int, prefix string,
+) (index.FieldDict, error) {
+	fd, _, err := is.FieldDictFuzzyAutomaton(field, term, fuzziness, prefix)
+	return fd, err
+}
+
+func (is *IndexSnapshot) FieldDictFuzzyAutomaton(field string,
+	term string, fuzziness int, prefix string,
+) (index.FieldDict, index.FuzzyAutomaton, error) {
+	return is.fieldDictFuzzy(field, term, fuzziness, prefix)
+}
+
+func (is *IndexSnapshot) fieldDictFuzzy(field string,
+	term string, fuzziness int, prefix string,
+) (index.FieldDict, index.FuzzyAutomaton, error) {
 	a, err := is.getLevAutomaton(term, uint8(fuzziness))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+	var fa index.FuzzyAutomaton
+	if vfa, ok := a.(vellum.FuzzyAutomaton); ok {
+		fa = vfa
 	}
-
 	var prefixBeg, prefixEnd []byte
 	if prefix != "" {
 		prefixBeg = []byte(prefix)
 		prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg)
 	}
-
-	return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
+	fd, err := is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
 		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
 	}, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	return fd, fa, nil
 }
 
 func (is *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) {
@@ -394,7 +440,7 @@ func (is *IndexSnapshot) DocCount() (uint64, error) {
 
 func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 	// FIXME could be done more efficiently directly, but reusing for simplicity
-	tfr, err := is.TermFieldReader(nil, []byte(id), "_id", false, false, false)
+	tfr, err := is.TermFieldReader(context.TODO(), []byte(id), "_id", false, false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -473,31 +519,7 @@ func (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (in
 			return is.offsets[x] > docNum
 		}) - 1
 
-	localDocNum := is.localDocNumFromGlobal(segmentIndex, docNum)
-	return int(segmentIndex), localDocNum
-}
-
-// This function returns the local docnum, given the segment index and global docnum
-func (is *IndexSnapshot) localDocNumFromGlobal(segmentIndex int, docNum uint64) uint64 {
-	return docNum - is.offsets[segmentIndex]
-}
-
-// Function to return a mapping of the segment index to the live global	doc nums
-// in the segment of the specified index snapshot.
-func (is *IndexSnapshot) globalDocNums() map[int]*roaring.Bitmap {
-	if len(is.segment) == 0 {
-		return nil
-	}
-
-	segmentIndexGlobalDocNums := make(map[int]*roaring.Bitmap)
-
-	for i := range is.segment {
-		segmentIndexGlobalDocNums[i] = roaring.NewBitmap()
-		for _, localDocNum := range is.segment[i].DocNumbersLive().ToArray() {
-			segmentIndexGlobalDocNums[i].Add(localDocNum + uint32(is.offsets[i]))
-		}
-	}
-	return segmentIndexGlobalDocNums
+	return int(segmentIndex), docNum - is.offsets[segmentIndex]
 }
 
 func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
@@ -518,9 +540,18 @@ func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
 	return string(v), nil
 }
 
+func (is *IndexSnapshot) segmentIndexAndLocalDocNum(id index.IndexInternalID) (int, uint64, error) {
+	docNum, err := docInternalToNumber(id)
+	if err != nil {
+		return 0, 0, err
+	}
+	segIdx, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum)
+	return segIdx, localDocNum, nil
+}
+
 func (is *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) {
 	// FIXME could be done more efficiently directly, but reusing for simplicity
-	tfr, err := is.TermFieldReader(nil, []byte(id), "_id", false, false, false)
+	tfr, err := is.TermFieldReader(context.TODO(), []byte(id), "_id", false, false, false)
 	if err != nil {
 		return nil, err
 	}
@@ -539,7 +570,8 @@ func (is *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err er
 }
 
 func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq,
-	includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
+	includeNorm, includeTermVectors bool,
+) (index.TermFieldReader, error) {
 	rv := is.allocTermFieldReaderDicts(field)
 
 	rv.ctx = ctx
@@ -706,7 +738,8 @@ func docInternalToNumber(in index.IndexInternalID) (uint64, error) {
 func (is *IndexSnapshot) documentVisitFieldTermsOnSegment(
 	segmentIndex int, localDocNum uint64, fields []string, cFields []string,
 	visitor index.DocValueVisitor, dvs segment.DocVisitState) (
-	cFieldsOut []string, dvsOut segment.DocVisitState, err error) {
+	cFieldsOut []string, dvsOut segment.DocVisitState, err error,
+) {
 	ss := is.segment[segmentIndex]
 
 	var vFields []string // fields that are visitable via the segment
@@ -763,7 +796,8 @@ func (is *IndexSnapshot) documentVisitFieldTermsOnSegment(
 }
 
 func (is *IndexSnapshot) DocValueReader(fields []string) (
-	index.DocValueReader, error) {
+	index.DocValueReader, error,
+) {
 	return &DocValueReader{i: is, fields: fields, currSegmentIndex: -1}, nil
 }
 
@@ -784,7 +818,8 @@ func (dvr *DocValueReader) BytesRead() uint64 {
 }
 
 func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,
-	visitor index.DocValueVisitor) (err error) {
+	visitor index.DocValueVisitor,
+) (err error) {
 	docNum, err := docInternalToNumber(id)
 	if err != nil {
 		return err
@@ -892,7 +927,7 @@ func (is *IndexSnapshot) CopyTo(d index.Directory) error {
 		return fmt.Errorf("invalid root.bolt file found")
 	}
 
-	copyBolt, err := bolt.Open(rootFile.Name(), 0600, nil)
+	copyBolt, err := bolt.Open(rootFile.Name(), 0o600, nil)
 	if err != nil {
 		return err
 	}
@@ -909,7 +944,7 @@ func (is *IndexSnapshot) CopyTo(d index.Directory) error {
 		return err
 	}
 
-	_, _, err = prepareBoltSnapshot(is, tx, "", is.parent.segPlugin, d)
+	_, _, err = prepareBoltSnapshot(is, tx, "", is.parent.segPlugin, nil, d)
 	if err != nil {
 		_ = tx.Rollback()
 		return fmt.Errorf("error backing up index snapshot: %v", err)
@@ -929,7 +964,8 @@ func (is *IndexSnapshot) UpdateIOStats(val uint64) {
 }
 
 func (is *IndexSnapshot) GetSpatialAnalyzerPlugin(typ string) (
-	index.SpatialAnalyzerPlugin, error) {
+	index.SpatialAnalyzerPlugin, error,
+) {
 	var rv index.SpatialAnalyzerPlugin
 	is.m.Lock()
 	rv = is.parent.spatialPlugin
@@ -963,3 +999,144 @@ func (is *IndexSnapshot) CloseCopyReader() error {
 	// close the index snapshot normally
 	return is.Close()
 }
+
+func (is *IndexSnapshot) ThesaurusTermReader(ctx context.Context, thesaurusName string, term []byte) (index.ThesaurusTermReader, error) {
+	rv := &IndexSnapshotThesaurusTermReader{}
+	rv.name = thesaurusName
+	rv.snapshot = is
+	if rv.postings == nil {
+		rv.postings = make([]segment.SynonymsList, len(is.segment))
+	}
+	if rv.iterators == nil {
+		rv.iterators = make([]segment.SynonymsIterator, len(is.segment))
+	}
+	rv.segmentOffset = 0
+
+	if rv.thesauri == nil {
+		rv.thesauri = make([]segment.Thesaurus, len(is.segment))
+		for i, s := range is.segment {
+			if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {
+				thes, err := synSeg.Thesaurus(thesaurusName)
+				if err != nil {
+					return nil, err
+				}
+				rv.thesauri[i] = thes
+			}
+		}
+	}
+
+	for i, s := range is.segment {
+		if _, ok := s.segment.(segment.ThesaurusSegment); ok {
+			pl, err := rv.thesauri[i].SynonymsList(term, s.deleted, rv.postings[i])
+			if err != nil {
+				return nil, err
+			}
+			rv.postings[i] = pl
+
+			rv.iterators[i] = pl.Iterator(rv.iterators[i])
+		}
+	}
+	return rv, nil
+}
+
+func (is *IndexSnapshot) newIndexSnapshotThesaurusKeys(name string,
+	makeItr func(i segment.Thesaurus) segment.ThesaurusIterator,
+) (*IndexSnapshotThesaurusKeys, error) {
+	results := make(chan *asynchSegmentResult, len(is.segment))
+	var wg sync.WaitGroup
+	wg.Add(len(is.segment))
+	for _, s := range is.segment {
+		go func(s *SegmentSnapshot) {
+			defer wg.Done()
+			if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {
+				thes, err := synSeg.Thesaurus(name)
+				if err != nil {
+					results <- &asynchSegmentResult{err: err}
+				} else {
+					results <- &asynchSegmentResult{thesItr: makeItr(thes)}
+				}
+			}
+		}(s)
+	}
+	// Close the channel after all goroutines complete
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	var err error
+	rv := &IndexSnapshotThesaurusKeys{
+		snapshot: is,
+		cursors:  make([]*segmentThesCursor, 0, len(is.segment)),
+	}
+	for asr := range results {
+		if asr.err != nil && err == nil {
+			err = asr.err
+		} else {
+			next, err2 := asr.thesItr.Next()
+			if err2 != nil && err == nil {
+				err = err2
+			}
+			if next != nil {
+				rv.cursors = append(rv.cursors, &segmentThesCursor{
+					itr:  asr.thesItr,
+					curr: *next,
+				})
+			}
+		}
+	}
+	// after ensuring we've read all items on channel
+	if err != nil {
+		return nil, err
+	}
+
+	return rv, nil
+}
+
+func (is *IndexSnapshot) ThesaurusKeys(name string) (index.ThesaurusKeys, error) {
+	return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {
+		return is.AutomatonIterator(nil, nil, nil)
+	})
+}
+
+func (is *IndexSnapshot) ThesaurusKeysFuzzy(name string,
+	term string, fuzziness int, prefix string,
+) (index.ThesaurusKeys, error) {
+	a, err := is.getLevAutomaton(term, uint8(fuzziness))
+	if err != nil {
+		return nil, err
+	}
+	var prefixBeg, prefixEnd []byte
+	if prefix != "" {
+		prefixBeg = []byte(prefix)
+		prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg)
+	}
+	return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {
+		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
+	})
+}
+
+func (is *IndexSnapshot) ThesaurusKeysPrefix(name string,
+	termPrefix []byte,
+) (index.ThesaurusKeys, error) {
+	termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix)
+	return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {
+		return is.AutomatonIterator(nil, termPrefix, termPrefixEnd)
+	})
+}
+
+func (is *IndexSnapshot) ThesaurusKeysRegexp(name string,
+	termRegex string,
+) (index.ThesaurusKeys, error) {
+	a, prefixBeg, prefixEnd, err := parseRegexp(termRegex)
+	if err != nil {
+		return nil, err
+	}
+	return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator {
+		return is.AutomatonIterator(a, prefixBeg, prefixEnd)
+	})
+}
+
+func (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) {
+	atomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta)
+}

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_dict.go

@@ -28,10 +28,12 @@ type segmentDictCursor struct {
 }
 
 type IndexSnapshotFieldDict struct {
-	snapshot  *IndexSnapshot
-	cursors   []*segmentDictCursor
-	entry     index.DictEntry
-	bytesRead uint64
+	cardinality int
+	bytesRead   uint64
+
+	snapshot *IndexSnapshot
+	cursors  []*segmentDictCursor
+	entry    index.DictEntry
 }
 
 func (i *IndexSnapshotFieldDict) BytesRead() uint64 {
@@ -94,6 +96,10 @@ func (i *IndexSnapshotFieldDict) Next() (*index.DictEntry, error) {
 	return &i.entry, nil
 }
 
+func (i *IndexSnapshotFieldDict) Cardinality() int {
+	return i.cardinality
+}
+
 func (i *IndexSnapshotFieldDict) Close() error {
 	return nil
 }

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_doc.go

@@ -18,7 +18,7 @@ import (
 	"bytes"
 	"reflect"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/size"
 	index "github.com/blevesearch/bleve_index_api"
 )

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_str.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/size"
+	segment "github.com/blevesearch/scorch_segment_api/v2"
+)
+
+var reflectStaticSizeIndexSnapshotThesaurusTermReader int
+
+func init() {
+	var istr IndexSnapshotThesaurusTermReader
+	reflectStaticSizeIndexSnapshotThesaurusTermReader = int(reflect.TypeOf(istr).Size())
+}
+
+type IndexSnapshotThesaurusTermReader struct {
+	name          string
+	snapshot      *IndexSnapshot
+	thesauri      []segment.Thesaurus
+	postings      []segment.SynonymsList
+	iterators     []segment.SynonymsIterator
+	segmentOffset int
+}
+
+func (i *IndexSnapshotThesaurusTermReader) Size() int {
+	sizeInBytes := reflectStaticSizeIndexSnapshotThesaurusTermReader + size.SizeOfPtr +
+		len(i.name) + size.SizeOfString
+
+	for _, postings := range i.postings {
+		sizeInBytes += postings.Size()
+	}
+
+	for _, iterator := range i.iterators {
+		sizeInBytes += iterator.Size()
+	}
+
+	return sizeInBytes
+}
+
+func (i *IndexSnapshotThesaurusTermReader) Next() (string, error) {
+	// find the next hit
+	for i.segmentOffset < len(i.iterators) {
+		if i.iterators[i.segmentOffset] != nil {
+			next, err := i.iterators[i.segmentOffset].Next()
+			if err != nil {
+				return "", err
+			}
+			if next != nil {
+				synTerm := next.Term()
+				return synTerm, nil
+			}
+			i.segmentOffset++
+		}
+	}
+	return "", nil
+}
+
+func (i *IndexSnapshotThesaurusTermReader) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_tfr.go

@@ -146,7 +146,7 @@ func (i *IndexSnapshotTermFieldReader) Advance(ID index.IndexInternalID, preAllo
 	// FIXME do something better
 	// for now, if we need to seek backwards, then restart from the beginning
 	if i.currPosting != nil && bytes.Compare(i.currID, ID) >= 0 {
-		i2, err := i.snapshot.TermFieldReader(nil, i.term, i.field,
+		i2, err := i.snapshot.TermFieldReader(context.TODO(), i.term, i.field,
 			i.includeFreq, i.includeNorm, i.includeTermVectors)
 		if err != nil {
 			return nil, err

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_thes.go

@@ -0,0 +1,107 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorch
+
+import (
+	"container/heap"
+
+	index "github.com/blevesearch/bleve_index_api"
+	segment "github.com/blevesearch/scorch_segment_api/v2"
+)
+
+type segmentThesCursor struct {
+	thes segment.Thesaurus
+	itr  segment.ThesaurusIterator
+	curr index.ThesaurusEntry
+}
+
+type IndexSnapshotThesaurusKeys struct {
+	snapshot *IndexSnapshot
+	cursors  []*segmentThesCursor
+	entry    index.ThesaurusEntry
+}
+
+func (i *IndexSnapshotThesaurusKeys) Len() int { return len(i.cursors) }
+func (i *IndexSnapshotThesaurusKeys) Less(a, b int) bool {
+	return i.cursors[a].curr.Term < i.cursors[b].curr.Term
+}
+func (i *IndexSnapshotThesaurusKeys) Swap(a, b int) {
+	i.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a]
+}
+
+func (i *IndexSnapshotThesaurusKeys) Push(x interface{}) {
+	i.cursors = append(i.cursors, x.(*segmentThesCursor))
+}
+
+func (i *IndexSnapshotThesaurusKeys) Pop() interface{} {
+	n := len(i.cursors)
+	x := i.cursors[n-1]
+	i.cursors = i.cursors[0 : n-1]
+	return x
+}
+
+func (i *IndexSnapshotThesaurusKeys) Next() (*index.ThesaurusEntry, error) {
+	if len(i.cursors) == 0 {
+		return nil, nil
+	}
+	i.entry = i.cursors[0].curr
+	next, err := i.cursors[0].itr.Next()
+	if err != nil {
+		return nil, err
+	}
+	if next == nil {
+		// at end of this cursor, remove it
+		heap.Pop(i)
+	} else {
+		// modified heap, fix it
+		i.cursors[0].curr = *next
+		heap.Fix(i, 0)
+	}
+	// look for any other entries with the exact same term
+	for len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term {
+		next, err := i.cursors[0].itr.Next()
+		if err != nil {
+			return nil, err
+		}
+		if next == nil {
+			// at end of this cursor, remove it
+			heap.Pop(i)
+		} else {
+			// modified heap, fix it
+			i.cursors[0].curr = *next
+			heap.Fix(i, 0)
+		}
+	}
+
+	return &i.entry, nil
+}
+
+func (i *IndexSnapshotThesaurusKeys) Close() error {
+	return nil
+}
+
+func (i *IndexSnapshotThesaurusKeys) Contains(key []byte) (bool, error) {
+	if len(i.cursors) == 0 {
+		return false, nil
+	}
+
+	for _, cursor := range i.cursors {
+		if found, _ := cursor.thes.Contains(key); found {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_vr.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/v2/size"
 	index "github.com/blevesearch/bleve_index_api"
 	segment_api "github.com/blevesearch/scorch_segment_api/v2"
@@ -51,32 +50,8 @@ type IndexSnapshotVectorReader struct {
 	currID        index.IndexInternalID
 	ctx           context.Context
 
-	searchParams json.RawMessage
-
-	// The following fields are only applicable for vector readers which will
-	// process pre-filtered kNN queries.
-	eligibleDocIDs []index.IndexInternalID
-}
-
-// Function to convert the internal IDs of the eligible documents to a type suitable
-// for addition to a bitmap.
-// Useful to have the eligible doc IDs in a bitmap to leverage the fast intersection
-// (AND) operations. Eg. finding the eligible doc IDs present in a segment.
-func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap {
-	res := roaring.NewBitmap()
-	if len(i.eligibleDocIDs) > 0 {
-		internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs))
-		// converts the doc IDs to uint32 and returns
-		for _, eligibleDocInternalID := range i.eligibleDocIDs {
-			internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID))
-			if err != nil {
-				continue
-			}
-			internalDocIDs = append(internalDocIDs, uint32(internalDocID))
-		}
-		res.AddMany(internalDocIDs)
-	}
-	return res
+	searchParams     json.RawMessage
+	eligibleSelector index.EligibleDocumentSelector
 }
 
 func (i *IndexSnapshotVectorReader) Size() int {
@@ -134,17 +109,8 @@ func (i *IndexSnapshotVectorReader) Advance(ID index.IndexInternalID,
 	preAlloced *index.VectorDoc) (*index.VectorDoc, error) {
 
 	if i.currPosting != nil && bytes.Compare(i.currID, ID) >= 0 {
-		var i2 index.VectorReader
-		var err error
-
-		if len(i.eligibleDocIDs) > 0 {
-			i2, err = i.snapshot.VectorReaderWithFilter(i.ctx, i.vector, i.field,
-				i.k, i.searchParams, i.eligibleDocIDs)
-		} else {
-			i2, err = i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k,
-				i.searchParams)
-		}
-
+		i2, err := i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k,
+			i.searchParams, i.eligibleSelector)
 		if err != nil {
 			return nil, err
 		}

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_segment.go

@@ -20,7 +20,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/size"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_vector_index.go

@@ -20,47 +20,23 @@ package scorch
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	index "github.com/blevesearch/bleve_index_api"
 	segment_api "github.com/blevesearch/scorch_segment_api/v2"
 )
 
 func (is *IndexSnapshot) VectorReader(ctx context.Context, vector []float32,
-	field string, k int64, searchParams json.RawMessage) (
-	index.VectorReader, error) {
-
-	rv := &IndexSnapshotVectorReader{
-		vector:       vector,
-		field:        field,
-		k:            k,
-		snapshot:     is,
-		searchParams: searchParams,
-	}
-
-	if rv.postings == nil {
-		rv.postings = make([]segment_api.VecPostingsList, len(is.segment))
-	}
-	if rv.iterators == nil {
-		rv.iterators = make([]segment_api.VecPostingsIterator, len(is.segment))
-	}
-
-	// initialize postings and iterators within the OptimizeVR's Finish()
-
-	return rv, nil
-}
-
-func (is *IndexSnapshot) VectorReaderWithFilter(ctx context.Context, vector []float32,
 	field string, k int64, searchParams json.RawMessage,
-	filterIDs []index.IndexInternalID) (
+	eligibleSelector index.EligibleDocumentSelector) (
 	index.VectorReader, error) {
-
 	rv := &IndexSnapshotVectorReader{
-		vector:         vector,
-		field:          field,
-		k:              k,
-		snapshot:       is,
-		searchParams:   searchParams,
-		eligibleDocIDs: filterIDs,
+		vector:           vector,
+		field:            field,
+		k:                k,
+		snapshot:         is,
+		searchParams:     searchParams,
+		eligibleSelector: eligibleSelector,
 	}
 
 	if rv.postings == nil {
@@ -69,8 +45,41 @@ func (is *IndexSnapshot) VectorReaderWithFilter(ctx context.Context, vector []fl
 	if rv.iterators == nil {
 		rv.iterators = make([]segment_api.VecPostingsIterator, len(is.segment))
 	}
-
 	// initialize postings and iterators within the OptimizeVR's Finish()
-
 	return rv, nil
 }
+
+// eligibleDocumentSelector is used to filter out documents that are eligible for
+// the KNN search from a pre-filter query.
+type eligibleDocumentSelector struct {
+	// segment ID -> segment local doc nums
+	eligibleDocNums map[int][]uint64
+	is              *IndexSnapshot
+}
+
+// SegmentEligibleDocs returns the list of eligible local doc numbers for the given segment.
+func (eds *eligibleDocumentSelector) SegmentEligibleDocs(segmentID int) []uint64 {
+	return eds.eligibleDocNums[segmentID]
+}
+
+// AddEligibleDocumentMatch adds a document match to the list of eligible documents.
+func (eds *eligibleDocumentSelector) AddEligibleDocumentMatch(id index.IndexInternalID) error {
+	if eds.is == nil {
+		return fmt.Errorf("eligibleDocumentSelector is not initialized with IndexSnapshot")
+	}
+	// Get the segment number and the local doc number for this document.
+	segIdx, docNum, err := eds.is.segmentIndexAndLocalDocNum(id)
+	if err != nil {
+		return err
+	}
+	// Add the local doc number to the list of eligible doc numbers for this segment.
+	eds.eligibleDocNums[segIdx] = append(eds.eligibleDocNums[segIdx], docNum)
+	return nil
+}
+
+func (is *IndexSnapshot) NewEligibleDocumentSelector() index.EligibleDocumentSelector {
+	return &eligibleDocumentSelector{
+		eligibleDocNums: map[int][]uint64{},
+		is:              is,
+	}
+}

vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go

@@ -51,7 +51,8 @@ type Stats struct {
 	TotTermSearchersStarted  uint64
 	TotTermSearchersFinished uint64
 
-	TotKNNSearches uint64
+	TotKNNSearches     uint64
+	TotSynonymSearches uint64
 
 	TotEventTriggerStarted   uint64
 	TotEventTriggerCompleted uint64
@@ -80,6 +81,7 @@ type Stats struct {
 	TotPersistedItems    uint64
 	TotItemsToPersist    uint64
 	TotPersistedSegments uint64
+	TotMutationsFiltered uint64
 
 	TotPersisterSlowMergerPause  uint64
 	TotPersisterSlowMergerResume uint64

vendor/github.com/blevesearch/bleve/v2/index/scorch/unadorned.go

@@ -18,7 +18,7 @@ import (
 	"math"
 	"reflect"
 
-	"github.com/RoaringBitmap/roaring"
+	"github.com/RoaringBitmap/roaring/v2"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 )
 

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/field_dict.go

@@ -77,6 +77,10 @@ func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) {
 
 }
 
+func (r *UpsideDownCouchFieldDict) Cardinality() int {
+	return 0
+}
+
 func (r *UpsideDownCouchFieldDict) Close() error {
 	return r.iterator.Close()
 }

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go

@@ -26,8 +26,10 @@ import (
 	"github.com/golang/protobuf/proto"
 )
 
-var reflectStaticSizeTermFrequencyRow int
-var reflectStaticSizeTermVector int
+var (
+	reflectStaticSizeTermFrequencyRow int
+	reflectStaticSizeTermVector       int
+)
 
 func init() {
 	var tfr TermFrequencyRow
@@ -322,7 +324,6 @@ func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) {
 		return nil, err
 	}
 	return rv, nil
-
 }
 
 func NewDictionaryRowK(key []byte) (*DictionaryRow, error) {
@@ -642,7 +643,7 @@ func (tfr *TermFrequencyRow) parseV(value []byte, includeTermVectors bool) error
 		}
 		currOffset += bytesRead
 
-		var arrayPositionsLen uint64 = 0
+		var arrayPositionsLen uint64
 		arrayPositionsLen, bytesRead = binary.Uvarint(value[currOffset:])
 		if bytesRead <= 0 {
 			return fmt.Errorf("invalid term frequency value, vector contains no arrayPositionLen")
@@ -682,7 +683,6 @@ func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) {
 		return nil, err
 	}
 	return rv, nil
-
 }
 
 type BackIndexRow struct {
@@ -1029,7 +1029,7 @@ func visitBackIndexRow(data []byte, callback backIndexFieldTermVisitor) error {
 				return io.ErrUnexpectedEOF
 			}
 			// don't track unrecognized data
-			//m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
+			// m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}
@@ -1109,7 +1109,7 @@ func visitBackIndexRowFieldTerms(data []byte, callback backIndexFieldTermVisitor
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			//m.Terms = append(m.Terms, string(data[iNdEx:postIndex]))
+			// m.Terms = append(m.Terms, string(data[iNdEx:postIndex]))
 			callback(theField, data[iNdEx:postIndex])
 			iNdEx = postIndex
 		default:
@@ -1132,7 +1132,7 @@ func visitBackIndexRowFieldTerms(data []byte, callback backIndexFieldTermVisitor
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
-			//m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
+			// m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
 			iNdEx += skippy
 		}
 	}

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/store.go

@@ -177,5 +177,8 @@ func (bs *Store) Compact() error {
 }
 
 func init() {
-	registry.RegisterKVStore(Name, New)
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap/store.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/blevesearch/bleve/v2/registry"
 	"github.com/blevesearch/gtreap"
-	"github.com/blevesearch/upsidedown_store_api"
+	store "github.com/blevesearch/upsidedown_store_api"
 )
 
 const Name = "gtreap"
@@ -78,5 +78,8 @@ func (s *Store) Writer() (store.KVWriter, error) {
 }
 
 func init() {
-	registry.RegisterKVStore(Name, New)
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go

@@ -1042,7 +1042,10 @@ func (udc *UpsideDownCouch) fieldIndexOrNewRow(name string) (uint16, *FieldRow)
 }
 
 func init() {
-	registry.RegisterIndexType(Name, NewUpsideDownCouch)
+	err := registry.RegisterIndexType(Name, NewUpsideDownCouch)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) {

vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go

@@ -16,6 +16,7 @@ package bleve
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -31,6 +32,10 @@ type indexAliasImpl struct {
 	indexes []Index
 	mutex   sync.RWMutex
 	open    bool
+	// if all the indexes in tha alias have the same mapping
+	// then the user can set the mapping here to avoid
+	// checking the mapping of each index in the alias
+	mapping mapping.IndexMapping
 }
 
 // NewIndexAlias creates a new IndexAlias over the provided
@@ -78,6 +83,25 @@ func (i *indexAliasImpl) Index(id string, data interface{}) error {
 	return i.indexes[0].Index(id, data)
 }
 
+func (i *indexAliasImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {
+	i.mutex.RLock()
+	defer i.mutex.RUnlock()
+
+	if !i.open {
+		return ErrorIndexClosed
+	}
+
+	err := i.isAliasToSingleIndex()
+	if err != nil {
+		return err
+	}
+
+	if si, ok := i.indexes[0].(SynonymIndex); ok {
+		return si.IndexSynonym(id, collection, definition)
+	}
+	return ErrorSynonymSearchNotSupported
+}
+
 func (i *indexAliasImpl) Delete(id string) error {
 	i.mutex.RLock()
 	defer i.mutex.RUnlock()
@@ -168,7 +192,13 @@ func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest
 		// indicates that this index alias is set as an Index
 		// in another alias, so we need to do a preSearch search
 		// and NOT a real search
-		return preSearchDataSearch(ctx, req, i.indexes...)
+		bm25PreSearch := isBM25Enabled(i.mapping)
+		flags := &preSearchFlags{
+			knn:      requestHasKNN(req),
+			synonyms: !isMatchNoneQuery(req.Query),
+			bm25:     bm25PreSearch,
+		}
+		return preSearchDataSearch(ctx, req, flags, i.indexes...)
 	}
 
 	// at this point we know we are doing a real search
@@ -182,12 +212,10 @@ func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest
 	// if necessary
 	var preSearchData map[string]map[string]interface{}
 	if req.PreSearchData != nil {
-		if requestHasKNN(req) {
-			var err error
-			preSearchData, err = redistributeKNNPreSearchData(req, i.indexes)
-			if err != nil {
-				return nil, err
-			}
+		var err error
+		preSearchData, err = redistributePreSearchData(req, i.indexes)
+		if err != nil {
+			return nil, err
 		}
 	}
 
@@ -208,12 +236,17 @@ func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest
 	//  - the request requires preSearch
 	var preSearchDuration time.Duration
 	var sr *SearchResult
-	if req.PreSearchData == nil && preSearchRequired(req) {
+	flags, err := preSearchRequired(ctx, req, i.mapping)
+	if err != nil {
+		return nil, err
+	}
+	if req.PreSearchData == nil && flags != nil {
 		searchStart := time.Now()
-		preSearchResult, err := preSearch(ctx, req, i.indexes...)
+		preSearchResult, err := preSearch(ctx, req, flags, i.indexes...)
 		if err != nil {
 			return nil, err
 		}
+
 		// check if the preSearch result has any errors and if so
 		// return the search result as is without executing the query
 		// so that the errors are not lost
@@ -221,17 +254,17 @@ func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest
 			return preSearchResult, nil
 		}
 		// finalize the preSearch result now
-		finalizePreSearchResult(req, preSearchResult)
+		finalizePreSearchResult(req, flags, preSearchResult)
 
 		// if there are no errors, then merge the data in the preSearch result
 		// and construct the preSearchData to be used in the actual search
 		// if the request is satisfied by the preSearch result, then we can
 		// directly return the preSearch result as the final result
-		if requestSatisfiedByPreSearch(req) {
+		if requestSatisfiedByPreSearch(req, flags) {
 			sr = finalizeSearchResult(req, preSearchResult)
 			// no need to run the 2nd phase MultiSearch(..)
 		} else {
-			preSearchData, err = constructPreSearchData(req, preSearchResult, i.indexes)
+			preSearchData, err = constructPreSearchData(req, flags, preSearchResult, i.indexes)
 			if err != nil {
 				return nil, err
 			}
@@ -352,6 +385,20 @@ func (i *indexAliasImpl) Close() error {
 	return nil
 }
 
+// SetIndexMapping sets the mapping for the alias and must be used
+// ONLY when all the indexes in the alias have the same mapping.
+// This is to avoid checking the mapping of each index in the alias
+// when executing a search request.
+func (i *indexAliasImpl) SetIndexMapping(m mapping.IndexMapping) error {
+	i.mutex.Lock()
+	defer i.mutex.Unlock()
+	if !i.open {
+		return ErrorIndexClosed
+	}
+	i.mapping = m
+	return nil
+}
+
 func (i *indexAliasImpl) Mapping() mapping.IndexMapping {
 	i.mutex.RLock()
 	defer i.mutex.RUnlock()
@@ -360,6 +407,11 @@ func (i *indexAliasImpl) Mapping() mapping.IndexMapping {
 		return nil
 	}
 
+	// if the mapping is already set, return it
+	if i.mapping != nil {
+		return i.mapping
+	}
+
 	err := i.isAliasToSingleIndex()
 	if err != nil {
 		return nil
@@ -520,21 +572,82 @@ type asyncSearchResult struct {
 	Err    error
 }
 
-func preSearchRequired(req *SearchRequest) bool {
-	return requestHasKNN(req)
+// preSearchFlags is a struct to hold flags indicating why preSearch is required
+type preSearchFlags struct {
+	knn      bool
+	synonyms bool
+	bm25     bool // needs presearch for this too
 }
 
-func preSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*SearchResult, error) {
+func isBM25Enabled(m mapping.IndexMapping) bool {
+	var rv bool
+	if m, ok := m.(*mapping.IndexMappingImpl); ok {
+		rv = m.ScoringModel == index.BM25Scoring
+	}
+	return rv
+}
+
+// preSearchRequired checks if preSearch is required and returns the presearch flags struct
+// indicating which preSearch is required
+func preSearchRequired(ctx context.Context, req *SearchRequest, m mapping.IndexMapping) (*preSearchFlags, error) {
+	// Check for KNN query
+	knn := requestHasKNN(req)
+	var synonyms bool
+	if !isMatchNoneQuery(req.Query) {
+		// Check if synonyms are defined in the mapping
+		if sm, ok := m.(mapping.SynonymMapping); ok && sm.SynonymCount() > 0 {
+			// check if any of the fields queried have a synonym source
+			// in the index mapping, to prevent unnecessary preSearch
+			fs, err := query.ExtractFields(req.Query, m, nil)
+			if err != nil {
+				return nil, err
+			}
+			for field := range fs {
+				if sm.SynonymSourceForPath(field) != "" {
+					synonyms = true
+					break
+				}
+			}
+		}
+	}
+	var bm25 bool
+	if !isMatchNoneQuery(req.Query) {
+		if ctx != nil {
+			if searchType := ctx.Value(search.SearchTypeKey); searchType != nil {
+				if searchType.(string) == search.GlobalScoring {
+					bm25 = isBM25Enabled(m)
+				}
+			}
+		}
+	}
+
+	if knn || synonyms || bm25 {
+		return &preSearchFlags{
+			knn:      knn,
+			synonyms: synonyms,
+			bm25:     bm25,
+		}, nil
+	}
+	return nil, nil
+}
+
+func preSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) {
 	// create a dummy request with a match none query
 	// since we only care about the preSearchData in PreSearch
+	var dummyQuery = req.Query
+	if !flags.bm25 && !flags.synonyms {
+		// create a dummy request with a match none query
+		// since we only care about the preSearchData in PreSearch
+		dummyQuery = query.NewMatchNoneQuery()
+	}
 	dummyRequest := &SearchRequest{
-		Query: query.NewMatchNoneQuery(),
+		Query: dummyQuery,
 	}
 	newCtx := context.WithValue(ctx, search.PreSearchKey, true)
-	if requestHasKNN(req) {
+	if flags.knn {
 		addKnnToDummyRequest(dummyRequest, req)
 	}
-	return preSearchDataSearch(newCtx, dummyRequest, indexes...)
+	return preSearchDataSearch(newCtx, dummyRequest, flags, indexes...)
 }
 
 // if the request is satisfied by just the preSearch result,
@@ -585,29 +698,67 @@ func finalizeSearchResult(req *SearchRequest, preSearchResult *SearchResult) *Se
 	return preSearchResult
 }
 
-func requestSatisfiedByPreSearch(req *SearchRequest) bool {
-	if requestHasKNN(req) && isKNNrequestSatisfiedByPreSearch(req) {
+func requestSatisfiedByPreSearch(req *SearchRequest, flags *preSearchFlags) bool {
+	if flags == nil {
+		return false
+	}
+	// if the synonyms presearch flag is set the request can never be satisfied by
+	// the preSearch result as synonyms are not part of the preSearch result
+	if flags.synonyms {
+		return false
+	}
+	if flags.knn && isKNNrequestSatisfiedByPreSearch(req) {
 		return true
 	}
 	return false
 }
 
-func constructPreSearchData(req *SearchRequest, preSearchResult *SearchResult, indexes []Index) (map[string]map[string]interface{}, error) {
+func constructSynonymPreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} {
+	for _, index := range indexes {
+		rv[index.Name()][search.SynonymPreSearchDataKey] = sr.SynonymResult
+	}
+	return rv
+}
+
+func constructBM25PreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} {
+	bmStats := sr.BM25Stats
+	if bmStats != nil {
+		for _, index := range indexes {
+			rv[index.Name()][search.BM25PreSearchDataKey] = &search.BM25Stats{
+				DocCount:         bmStats.DocCount,
+				FieldCardinality: bmStats.FieldCardinality,
+			}
+		}
+	}
+	return rv
+}
+
+func constructPreSearchData(req *SearchRequest, flags *preSearchFlags,
+	preSearchResult *SearchResult, indexes []Index) (map[string]map[string]interface{}, error) {
+	if flags == nil || preSearchResult == nil {
+		return nil, fmt.Errorf("invalid input, flags: %v, preSearchResult: %v", flags, preSearchResult)
+	}
 	mergedOut := make(map[string]map[string]interface{}, len(indexes))
 	for _, index := range indexes {
 		mergedOut[index.Name()] = make(map[string]interface{})
 	}
 	var err error
-	if requestHasKNN(req) {
+	if flags.knn {
 		mergedOut, err = constructKnnPreSearchData(mergedOut, preSearchResult, indexes)
 		if err != nil {
 			return nil, err
 		}
 	}
+	if flags.synonyms {
+		mergedOut = constructSynonymPreSearchData(mergedOut, preSearchResult, indexes)
+	}
+	if flags.bm25 {
+		mergedOut = constructBM25PreSearchData(mergedOut, preSearchResult, indexes)
+	}
 	return mergedOut, nil
 }
 
-func preSearchDataSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*SearchResult, error) {
+func preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) {
 	asyncResults := make(chan *asyncSearchResult, len(indexes))
 	// run search on each index in separate go routine
 	var waitGroup sync.WaitGroup
@@ -638,7 +789,7 @@ func preSearchDataSearch(ctx context.Context, req *SearchRequest, indexes ...Ind
 			if prp == nil {
 				// first valid preSearch result
 				// create a new preSearch result processor
-				prp = createPreSearchResultProcessor(req)
+				prp = createPreSearchResultProcessor(req, flags)
 			}
 			prp.add(asr.Result, asr.Name)
 			if sr == nil {
@@ -684,6 +835,56 @@ func preSearchDataSearch(ctx context.Context, req *SearchRequest, indexes ...Ind
 	return sr, nil
 }
 
+// redistributePreSearchData redistributes the preSearchData sent in the search request to an index alias
+// which would happen in the case of an alias tree and depending on the level of the tree, the preSearchData
+// needs to be redistributed to the indexes at that level
+func redistributePreSearchData(req *SearchRequest, indexes []Index) (map[string]map[string]interface{}, error) {
+	rv := make(map[string]map[string]interface{})
+	for _, index := range indexes {
+		rv[index.Name()] = make(map[string]interface{})
+	}
+	if knnHits, ok := req.PreSearchData[search.KnnPreSearchDataKey].([]*search.DocumentMatch); ok {
+		// the preSearchData for KNN is a list of DocumentMatch objects
+		// that need to be redistributed to the right index.
+		// This is used only in the case of an alias tree, where the indexes
+		// are at the leaves of the tree, and the master alias is at the root.
+		// At each level of the tree, the preSearchData needs to be redistributed
+		// to the indexes/aliases at that level. Because the preSearchData is
+		// specific to each final index at the leaf.
+		segregatedKnnHits, err := validateAndDistributeKNNHits(knnHits, indexes)
+		if err != nil {
+			return nil, err
+		}
+		for _, index := range indexes {
+			rv[index.Name()][search.KnnPreSearchDataKey] = segregatedKnnHits[index.Name()]
+		}
+	}
+	if fts, ok := req.PreSearchData[search.SynonymPreSearchDataKey].(search.FieldTermSynonymMap); ok {
+		for _, index := range indexes {
+			rv[index.Name()][search.SynonymPreSearchDataKey] = fts
+		}
+	}
+
+	if bm25Data, ok := req.PreSearchData[search.BM25PreSearchDataKey].(*search.BM25Stats); ok {
+		for _, index := range indexes {
+			rv[index.Name()][search.BM25PreSearchDataKey] = bm25Data
+		}
+	}
+	return rv, nil
+}
+
+// finalizePreSearchResult finalizes the preSearch result by applying the finalization steps
+// specific to the preSearch flags
+func finalizePreSearchResult(req *SearchRequest, flags *preSearchFlags, preSearchResult *SearchResult) {
+	// if flags is nil then return
+	if flags == nil {
+		return
+	}
+	if flags.knn {
+		preSearchResult.Hits = finalizeKNNResults(req, preSearchResult.Hits)
+	}
+}
+
 // hitsInCurrentPage returns the hits in the current page
 // using the From and Size parameters in the request
 func hitsInCurrentPage(req *SearchRequest, hits []*search.DocumentMatch) []*search.DocumentMatch {
@@ -856,3 +1057,7 @@ func (f *indexAliasImplFieldDict) Close() error {
 	defer f.index.mutex.RUnlock()
 	return f.fieldDict.Close()
 }
+
+func (f *indexAliasImplFieldDict) Cardinality() int {
+	return f.fieldDict.Cardinality()
+}

vendor/github.com/blevesearch/bleve/v2/index_impl.go

@@ -38,6 +38,7 @@ import (
 	"github.com/blevesearch/bleve/v2/search/collector"
 	"github.com/blevesearch/bleve/v2/search/facet"
 	"github.com/blevesearch/bleve/v2/search/highlight"
+	"github.com/blevesearch/bleve/v2/search/query"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
 	"github.com/blevesearch/geo/s2"
@@ -267,6 +268,40 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) {
 	return
 }
 
+// IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection.
+// Synonym definition defines term relationships for query expansion in searches.
+func (i *indexImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error {
+	if id == "" {
+		return ErrorEmptyID
+	}
+
+	i.mutex.RLock()
+	defer i.mutex.RUnlock()
+
+	if !i.open {
+		return ErrorIndexClosed
+	}
+
+	i.FireIndexEvent()
+
+	synMap, ok := i.m.(mapping.SynonymMapping)
+	if !ok {
+		return ErrorSynonymSearchNotSupported
+	}
+
+	if err := definition.Validate(); err != nil {
+		return err
+	}
+
+	doc := document.NewSynonymDocument(id)
+	err := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms)
+	if err != nil {
+		return err
+	}
+	err = i.i.Update(doc)
+	return err
+}
+
 // IndexAdvanced takes a document.Document object
 // skips the mapping and indexes it.
 func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) {
@@ -449,12 +484,51 @@ func (i *indexImpl) preSearch(ctx context.Context, req *SearchRequest, reader in
 		}
 	}
 
+	var fts search.FieldTermSynonymMap
+	var count uint64
+	var fieldCardinality map[string]int
+	if !isMatchNoneQuery(req.Query) {
+		if synMap, ok := i.m.(mapping.SynonymMapping); ok {
+			if synReader, ok := reader.(index.ThesaurusReader); ok {
+				fts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+		if ok := isBM25Enabled(i.m); ok {
+			fieldCardinality = make(map[string]int)
+			count, err = reader.DocCount()
+			if err != nil {
+				return nil, err
+			}
+
+			fs := make(query.FieldSet)
+			fs, err := query.ExtractFields(req.Query, i.m, fs)
+			if err != nil {
+				return nil, err
+			}
+			for field := range fs {
+				dict, err := reader.FieldDict(field)
+				if err != nil {
+					return nil, err
+				}
+				fieldCardinality[field] = dict.Cardinality()
+			}
+		}
+	}
+
 	return &SearchResult{
 		Status: &SearchStatus{
 			Total:      1,
 			Successful: 1,
 		},
-		Hits: knnHits,
+		Hits:          knnHits,
+		SynonymResult: fts,
+		BM25Stats: &search.BM25Stats{
+			DocCount:         float64(count),
+			FieldCardinality: fieldCardinality,
+		},
 	}, nil
 }
 
@@ -505,8 +579,13 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	}
 
 	var knnHits []*search.DocumentMatch
+	var skipKNNCollector bool
+
+	var fts search.FieldTermSynonymMap
+	var skipSynonymCollector bool
+
+	var bm25Data *search.BM25Stats
 	var ok bool
-	var skipKnnCollector bool
 	if req.PreSearchData != nil {
 		for k, v := range req.PreSearchData {
 			switch k {
@@ -516,20 +595,68 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 					if !ok {
 						return nil, fmt.Errorf("knn preSearchData must be of type []*search.DocumentMatch")
 					}
+					skipKNNCollector = true
+				}
+			case search.SynonymPreSearchDataKey:
+				if v != nil {
+					fts, ok = v.(search.FieldTermSynonymMap)
+					if !ok {
+						return nil, fmt.Errorf("synonym preSearchData must be of type search.FieldTermSynonymMap")
+					}
+					skipSynonymCollector = true
+				}
+			case search.BM25PreSearchDataKey:
+				if v != nil {
+					bm25Data, ok = v.(*search.BM25Stats)
+					if !ok {
+						return nil, fmt.Errorf("bm25 preSearchData must be of type map[string]interface{}")
+					}
 				}
-				skipKnnCollector = true
 			}
 		}
 	}
-	if !skipKnnCollector && requestHasKNN(req) {
+	if !skipKNNCollector && requestHasKNN(req) {
 		knnHits, err = i.runKnnCollector(ctx, req, indexReader, false)
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	if !skipSynonymCollector {
+		if synMap, ok := i.m.(mapping.SynonymMapping); ok && synMap.SynonymCount() > 0 {
+			if synReader, ok := indexReader.(index.ThesaurusReader); ok {
+				fts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+
 	setKnnHitsInCollector(knnHits, req, coll)
 
+	if fts != nil {
+		if is, ok := indexReader.(*scorch.IndexSnapshot); ok {
+			is.UpdateSynonymSearchCount(1)
+		}
+		ctx = context.WithValue(ctx, search.FieldTermSynonymMapKey, fts)
+	}
+
+	scoringModelCallback := func() string {
+		if isBM25Enabled(i.m) {
+			return index.BM25Scoring
+		}
+		return index.DefaultScoringModel
+	}
+	ctx = context.WithValue(ctx, search.GetScoringModelCallbackKey,
+		search.GetScoringModelCallbackFn(scoringModelCallback))
+
+	// set the bm25 presearch data (stats important for consistent scoring) in
+	// the context object
+	if bm25Data != nil {
+		ctx = context.WithValue(ctx, search.BM25PreSearchDataKey, bm25Data)
+	}
+
 	// This callback and variable handles the tracking of bytes read
 	//  1. as part of creation of tfr and its Next() calls which is
 	//     accounted by invoking this callback when the TFR is closed.
@@ -786,6 +913,11 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
 								if err == nil {
 									value = v
 								}
+							case index.IPField:
+								ip, err := docF.IP()
+								if err == nil {
+									value = ip.String()
+								}
 							}
 
 							if value != nil {
@@ -1032,6 +1164,10 @@ func (f *indexImplFieldDict) Close() error {
 	return f.indexReader.Close()
 }
 
+func (f *indexImplFieldDict) Cardinality() int {
+	return f.fieldDict.Cardinality()
+}
+
 // helper function to remove duplicate entries from slice of strings
 func deDuplicate(fields []string) []string {
 	entries := make(map[string]struct{})

vendor/github.com/blevesearch/bleve/v2/mapping/analysis.go

@@ -21,6 +21,7 @@ type customAnalysis struct {
 	TokenFilters    map[string]map[string]interface{} `json:"token_filters,omitempty"`
 	Analyzers       map[string]map[string]interface{} `json:"analyzers,omitempty"`
 	DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"`
+	SynonymSources  map[string]map[string]interface{} `json:"synonym_sources,omitempty"`
 }
 
 func (c *customAnalysis) registerAll(i *IndexMappingImpl) error {
@@ -83,6 +84,12 @@ func (c *customAnalysis) registerAll(i *IndexMappingImpl) error {
 			return err
 		}
 	}
+	for name, config := range c.SynonymSources {
+		_, err := i.cache.DefineSynonymSource(name, config)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -94,6 +101,7 @@ func newCustomAnalysis() *customAnalysis {
 		TokenFilters:    make(map[string]map[string]interface{}),
 		Analyzers:       make(map[string]map[string]interface{}),
 		DateTimeParsers: make(map[string]map[string]interface{}),
+		SynonymSources:  make(map[string]map[string]interface{}),
 	}
 	return &rv
 }

vendor/github.com/blevesearch/bleve/v2/mapping/document.go

@@ -40,11 +40,12 @@ import (
 // are used.  To disable this automatic handling, set
 // Dynamic to false.
 type DocumentMapping struct {
-	Enabled         bool                        `json:"enabled"`
-	Dynamic         bool                        `json:"dynamic"`
-	Properties      map[string]*DocumentMapping `json:"properties,omitempty"`
-	Fields          []*FieldMapping             `json:"fields,omitempty"`
-	DefaultAnalyzer string                      `json:"default_analyzer,omitempty"`
+	Enabled              bool                        `json:"enabled"`
+	Dynamic              bool                        `json:"dynamic"`
+	Properties           map[string]*DocumentMapping `json:"properties,omitempty"`
+	Fields               []*FieldMapping             `json:"fields,omitempty"`
+	DefaultAnalyzer      string                      `json:"default_analyzer,omitempty"`
+	DefaultSynonymSource string                      `json:"default_synonym_source,omitempty"`
 
 	// StructTagKey overrides "json" when looking for field names in struct tags
 	StructTagKey string `json:"struct_tag_key,omitempty"`
@@ -59,6 +60,12 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache,
 			return err
 		}
 	}
+	if dm.DefaultSynonymSource != "" {
+		_, err := cache.SynonymSourceNamed(dm.DefaultSynonymSource)
+		if err != nil {
+			return err
+		}
+	}
 	for propertyName, property := range dm.Properties {
 		newParent := propertyName
 		if parentName != "" {
@@ -82,7 +89,12 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache,
 				return err
 			}
 		}
-
+		if field.SynonymSource != "" {
+			_, err = cache.SynonymSourceNamed(field.SynonymSource)
+			if err != nil {
+				return err
+			}
+		}
 		err := validateFieldMapping(field, parentName, fieldAliasCtx)
 		if err != nil {
 			return err
@@ -112,6 +124,17 @@ func (dm *DocumentMapping) analyzerNameForPath(path string) string {
 	return ""
 }
 
+// synonymSourceForPath attempts to first find the field
+// described by this path, then returns the analyzer
+// configured for that field
+func (dm *DocumentMapping) synonymSourceForPath(path string) string {
+	field := dm.fieldDescribedByPath(path)
+	if field != nil {
+		return field.SynonymSource
+	}
+	return ""
+}
+
 func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping {
 	pathElements := decodePath(path)
 	if len(pathElements) > 1 {
@@ -295,6 +318,11 @@ func (dm *DocumentMapping) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "default_synonym_source":
+			err := util.UnmarshalJSON(v, &dm.DefaultSynonymSource)
+			if err != nil {
+				return err
+			}
 		case "properties":
 			err := util.UnmarshalJSON(v, &dm.Properties)
 			if err != nil {
@@ -338,6 +366,22 @@ func (dm *DocumentMapping) defaultAnalyzerName(path []string) string {
 	return rv
 }
 
+func (dm *DocumentMapping) defaultSynonymSource(path []string) string {
+	current := dm
+	rv := current.DefaultSynonymSource
+	for _, pathElement := range path {
+		var ok bool
+		current, ok = current.Properties[pathElement]
+		if !ok {
+			break
+		}
+		if current.DefaultSynonymSource != "" {
+			rv = current.DefaultSynonymSource
+		}
+	}
+	return rv
+}
+
 func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) {
 	// allow default "json" tag to be overridden
 	structTagKey := dm.StructTagKey

vendor/github.com/blevesearch/bleve/v2/mapping/field.go

@@ -74,12 +74,14 @@ type FieldMapping struct {
 	Dims int `json:"dims,omitempty"`
 
 	// Similarity is the similarity algorithm used for scoring
-	// vector fields.
-	// See: index.DefaultSimilarityMetric & index.SupportedSimilarityMetrics
+	// field's content while performing search on it.
+	// See: index.SimilarityModels
 	Similarity string `json:"similarity,omitempty"`
 
 	// Applicable to vector fields only - optimization string
 	VectorIndexOptimizedFor string `json:"vector_index_optimized_for,omitempty"`
+
+	SynonymSource string `json:"synonym_source,omitempty"`
 }
 
 // NewTextFieldMapping returns a default field mapping for text
@@ -460,17 +462,22 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error {
 				return err
 			}
 		case "dims":
-			err := json.Unmarshal(v, &fm.Dims)
+			err := util.UnmarshalJSON(v, &fm.Dims)
 			if err != nil {
 				return err
 			}
 		case "similarity":
-			err := json.Unmarshal(v, &fm.Similarity)
+			err := util.UnmarshalJSON(v, &fm.Similarity)
 			if err != nil {
 				return err
 			}
 		case "vector_index_optimized_for":
-			err := json.Unmarshal(v, &fm.VectorIndexOptimizedFor)
+			err := util.UnmarshalJSON(v, &fm.VectorIndexOptimizedFor)
+			if err != nil {
+				return err
+			}
+		case "synonym_source":
+			err := util.UnmarshalJSON(v, &fm.SynonymSource)
 			if err != nil {
 				return err
 			}

vendor/github.com/blevesearch/bleve/v2/mapping/index.go

@@ -49,6 +49,8 @@ type IndexMappingImpl struct {
 	DefaultType           string                      `json:"default_type"`
 	DefaultAnalyzer       string                      `json:"default_analyzer"`
 	DefaultDateTimeParser string                      `json:"default_datetime_parser"`
+	DefaultSynonymSource  string                      `json:"default_synonym_source,omitempty"`
+	ScoringModel          string                      `json:"scoring_model,omitempty"`
 	DefaultField          string                      `json:"default_field"`
 	StoreDynamic          bool                        `json:"store_dynamic"`
 	IndexDynamic          bool                        `json:"index_dynamic"`
@@ -145,6 +147,15 @@ func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[stri
 	return nil
 }
 
+func (im *IndexMappingImpl) AddSynonymSource(name string, config map[string]interface{}) error {
+	_, err := im.cache.DefineSynonymSource(name, config)
+	if err != nil {
+		return err
+	}
+	im.CustomAnalysis.SynonymSources[name] = config
+	return nil
+}
+
 // NewIndexMapping creates a new IndexMapping that will use all the default indexing rules
 func NewIndexMapping() *IndexMappingImpl {
 	return &IndexMappingImpl{
@@ -174,7 +185,12 @@ func (im *IndexMappingImpl) Validate() error {
 	if err != nil {
 		return err
 	}
-
+	if im.DefaultSynonymSource != "" {
+		_, err = im.cache.SynonymSourceNamed(im.DefaultSynonymSource)
+		if err != nil {
+			return err
+		}
+	}
 	fieldAliasCtx := make(map[string]*FieldMapping)
 	err = im.DefaultMapping.Validate(im.cache, "", fieldAliasCtx)
 	if err != nil {
@@ -186,6 +202,11 @@ func (im *IndexMappingImpl) Validate() error {
 			return err
 		}
 	}
+
+	if _, ok := index.SupportedScoringModels[im.ScoringModel]; !ok && im.ScoringModel != "" {
+		return fmt.Errorf("unsupported scoring model: %s", im.ScoringModel)
+	}
+
 	return nil
 }
 
@@ -253,6 +274,11 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "default_synonym_source":
+			err := util.UnmarshalJSON(v, &im.DefaultSynonymSource)
+			if err != nil {
+				return err
+			}
 		case "default_field":
 			err := util.UnmarshalJSON(v, &im.DefaultField)
 			if err != nil {
@@ -283,6 +309,12 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "scoring_model":
+			err := util.UnmarshalJSON(v, &im.ScoringModel)
+			if err != nil {
+				return err
+			}
+
 		default:
 			invalidKeys = append(invalidKeys, k)
 		}
@@ -334,11 +366,30 @@ func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}
 			field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors)
 			doc.AddField(field)
 		}
+		doc.SetIndexed()
 	}
 
 	return nil
 }
 
+func (im *IndexMappingImpl) MapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error {
+	// determine all the synonym sources with the given collection
+	// and create a synonym field for each
+	err := im.SynonymSourceVisitor(func(name string, item analysis.SynonymSource) error {
+		if item.Collection() == collection {
+			// create a new field with the name of the synonym source
+			analyzer := im.AnalyzerNamed(item.Analyzer())
+			if analyzer == nil {
+				return fmt.Errorf("unknown analyzer named: %s", item.Analyzer())
+			}
+			field := document.NewSynonymField(name, analyzer, input, synonyms)
+			doc.AddField(field)
+		}
+		return nil
+	})
+	return err
+}
+
 type walkContext struct {
 	doc             *document.Document
 	im              *IndexMappingImpl
@@ -457,3 +508,66 @@ func (im *IndexMappingImpl) FieldMappingForPath(path string) FieldMapping {
 func (im *IndexMappingImpl) DefaultSearchField() string {
 	return im.DefaultField
 }
+
+func (im *IndexMappingImpl) SynonymSourceNamed(name string) analysis.SynonymSource {
+	syn, err := im.cache.SynonymSourceNamed(name)
+	if err != nil {
+		logger.Printf("error using synonym source named: %s", name)
+		return nil
+	}
+	return syn
+}
+
+func (im *IndexMappingImpl) SynonymSourceForPath(path string) string {
+	// first we look for explicit mapping on the field
+	for _, docMapping := range im.TypeMapping {
+		synonymSource := docMapping.synonymSourceForPath(path)
+		if synonymSource != "" {
+			return synonymSource
+		}
+	}
+
+	// now try the default mapping
+	pathMapping, _ := im.DefaultMapping.documentMappingForPath(path)
+	if pathMapping != nil {
+		if len(pathMapping.Fields) > 0 {
+			if pathMapping.Fields[0].SynonymSource != "" {
+				return pathMapping.Fields[0].SynonymSource
+			}
+		}
+	}
+
+	// next we will try default synonym sources for the path
+	pathDecoded := decodePath(path)
+	for _, docMapping := range im.TypeMapping {
+		if docMapping.Enabled {
+			rv := docMapping.defaultSynonymSource(pathDecoded)
+			if rv != "" {
+				return rv
+			}
+		}
+	}
+	// now the default analyzer for the default mapping
+	if im.DefaultMapping.Enabled {
+		rv := im.DefaultMapping.defaultSynonymSource(pathDecoded)
+		if rv != "" {
+			return rv
+		}
+	}
+
+	return im.DefaultSynonymSource
+}
+
+// SynonymCount() returns the number of synonym sources defined in the mapping
+func (im *IndexMappingImpl) SynonymCount() int {
+	return len(im.CustomAnalysis.SynonymSources)
+}
+
+// SynonymSourceVisitor() allows a visitor to iterate over all synonym sources
+func (im *IndexMappingImpl) SynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error {
+	err := im.cache.SynonymSources.VisitSynonymSources(visitor)
+	if err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go

@@ -58,3 +58,19 @@ type IndexMapping interface {
 
 	FieldMappingForPath(path string) FieldMapping
 }
+
+// A SynonymMapping extends the IndexMapping interface to provide
+// additional methods for working with synonyms.
+type SynonymMapping interface {
+	IndexMapping
+
+	MapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error
+
+	SynonymSourceForPath(path string) string
+
+	SynonymSourceNamed(name string) analysis.SynonymSource
+
+	SynonymCount() int
+
+	SynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error
+}

vendor/github.com/blevesearch/bleve/v2/mapping/mapping_vectors.go

@@ -204,7 +204,7 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,
 	}
 
 	if field.Similarity == "" {
-		field.Similarity = index.DefaultSimilarityMetric
+		field.Similarity = index.DefaultVectorSimilarityMetric
 	}
 
 	if field.VectorIndexOptimizedFor == "" {
@@ -249,10 +249,10 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,
 			MinVectorDims, MaxVectorDims)
 	}
 
-	if _, ok := index.SupportedSimilarityMetrics[field.Similarity]; !ok {
+	if _, ok := index.SupportedVectorSimilarityMetrics[field.Similarity]; !ok {
 		return fmt.Errorf("field: '%s', invalid similarity "+
 			"metric: '%s', valid metrics are: %+v", field.Name, field.Similarity,
-			reflect.ValueOf(index.SupportedSimilarityMetrics).MapKeys())
+			reflect.ValueOf(index.SupportedVectorSimilarityMetrics).MapKeys())
 	}
 
 	if fieldAliasCtx != nil { // writing to a nil map is unsafe

vendor/github.com/blevesearch/bleve/v2/mapping/synonym.go

@@ -0,0 +1,71 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mapping
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+type SynonymSource struct {
+	CollectionName string `json:"collection"`
+	AnalyzerName   string `json:"analyzer"`
+}
+
+func NewSynonymSource(collection, analyzer string) *SynonymSource {
+	return &SynonymSource{
+		CollectionName: collection,
+		AnalyzerName:   analyzer,
+	}
+}
+
+func (s *SynonymSource) Collection() string {
+	return s.CollectionName
+}
+
+func (s *SynonymSource) Analyzer() string {
+	return s.AnalyzerName
+}
+
+func (s *SynonymSource) SetCollection(c string) {
+	s.CollectionName = c
+}
+
+func (s *SynonymSource) SetAnalyzer(a string) {
+	s.AnalyzerName = a
+}
+func SynonymSourceConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.SynonymSource, error) {
+	collection, ok := config["collection"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify collection")
+	}
+	analyzer, ok := config["analyzer"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify analyzer")
+	}
+	if _, err := cache.AnalyzerNamed(analyzer); err != nil {
+		return nil, fmt.Errorf("analyzer named '%s' not found", analyzer)
+	}
+	return NewSynonymSource(collection, analyzer), nil
+}
+
+func init() {
+	err := registry.RegisterSynonymSource(analysis.SynonymSourceType, SynonymSourceConstructor)
+	if err != nil {
+		panic(err)
+	}
+}

vendor/github.com/blevesearch/bleve/v2/pre_search.go

@@ -14,6 +14,10 @@
 
 package bleve
 
+import (
+	"github.com/blevesearch/bleve/v2/search"
+)
+
 // A preSearchResultProcessor processes the data in
 // the preSearch result from multiple
 // indexes in an alias and merges them together to
@@ -26,6 +30,8 @@ type preSearchResultProcessor interface {
 	finalize(*SearchResult)
 }
 
+// -----------------------------------------------------------------------------
+// KNN preSearchResultProcessor for handling KNN presearch results
 type knnPreSearchResultProcessor struct {
 	addFn      func(sr *SearchResult, indexName string)
 	finalizeFn func(sr *SearchResult)
@@ -44,16 +50,121 @@ func (k *knnPreSearchResultProcessor) finalize(sr *SearchResult) {
 }
 
 // -----------------------------------------------------------------------------
+// Synonym preSearchResultProcessor for handling Synonym presearch results
+type synonymPreSearchResultProcessor struct {
+	finalizedFts search.FieldTermSynonymMap
+}
 
-func finalizePreSearchResult(req *SearchRequest, preSearchResult *SearchResult) {
-	if requestHasKNN(req) {
-		preSearchResult.Hits = finalizeKNNResults(req, preSearchResult.Hits)
+func newSynonymPreSearchResultProcessor() *synonymPreSearchResultProcessor {
+	return &synonymPreSearchResultProcessor{}
+}
+
+func (s *synonymPreSearchResultProcessor) add(sr *SearchResult, indexName string) {
+	// Check if SynonymResult or the synonym data key is nil
+	if sr.SynonymResult == nil {
+		return
+	}
+
+	// Attempt to cast PreSearchResults to FieldTermSynonymMap
+
+	// Merge with finalizedFts or initialize it if nil
+	if s.finalizedFts == nil {
+		s.finalizedFts = sr.SynonymResult
+	} else {
+		s.finalizedFts.MergeWith(sr.SynonymResult)
 	}
 }
 
-func createPreSearchResultProcessor(req *SearchRequest) preSearchResultProcessor {
-	if requestHasKNN(req) {
-		return newKnnPreSearchResultProcessor(req)
+func (s *synonymPreSearchResultProcessor) finalize(sr *SearchResult) {
+	// Set the finalized synonym data to the PreSearchResults
+	if s.finalizedFts != nil {
+		sr.SynonymResult = s.finalizedFts
+	}
+}
+
+type bm25PreSearchResultProcessor struct {
+	docCount         float64 // bm25 specific stats
+	fieldCardinality map[string]int
+}
+
+func newBM25PreSearchResultProcessor() *bm25PreSearchResultProcessor {
+	return &bm25PreSearchResultProcessor{
+		fieldCardinality: make(map[string]int),
+	}
+}
+
+// TODO How will this work for queries other than term queries?
+func (b *bm25PreSearchResultProcessor) add(sr *SearchResult, indexName string) {
+	if sr.BM25Stats != nil {
+		b.docCount += sr.BM25Stats.DocCount
+		for field, cardinality := range sr.BM25Stats.FieldCardinality {
+			b.fieldCardinality[field] += cardinality
+		}
+	}
+}
+
+func (b *bm25PreSearchResultProcessor) finalize(sr *SearchResult) {
+	sr.BM25Stats = &search.BM25Stats{
+		DocCount:         b.docCount,
+		FieldCardinality: b.fieldCardinality,
+	}
+}
+
+// -----------------------------------------------------------------------------
+// Master struct that can hold any number of presearch result processors
+type compositePreSearchResultProcessor struct {
+	presearchResultProcessors []preSearchResultProcessor
+}
+
+// Implements the add method, which forwards to all the internal processors
+func (m *compositePreSearchResultProcessor) add(sr *SearchResult, indexName string) {
+	for _, p := range m.presearchResultProcessors {
+		p.add(sr, indexName)
+	}
+}
+
+// Implements the finalize method, which forwards to all the internal processors
+func (m *compositePreSearchResultProcessor) finalize(sr *SearchResult) {
+	for _, p := range m.presearchResultProcessors {
+		p.finalize(sr)
+	}
+}
+
+// -----------------------------------------------------------------------------
+// Function to create the appropriate preSearchResultProcessor(s)
+func createPreSearchResultProcessor(req *SearchRequest, flags *preSearchFlags) preSearchResultProcessor {
+	// return nil for invalid input
+	if flags == nil || req == nil {
+		return nil
+	}
+	var processors []preSearchResultProcessor
+	// Add KNN processor if the request has KNN
+	if flags.knn {
+		if knnProcessor := newKnnPreSearchResultProcessor(req); knnProcessor != nil {
+			processors = append(processors, knnProcessor)
+		}
+	}
+	// Add Synonym processor if the request has Synonym
+	if flags.synonyms {
+		if synonymProcessor := newSynonymPreSearchResultProcessor(); synonymProcessor != nil {
+			processors = append(processors, synonymProcessor)
+		}
+	}
+	if flags.bm25 {
+		if bm25Processtor := newBM25PreSearchResultProcessor(); bm25Processtor != nil {
+			processors = append(processors, bm25Processtor)
+		}
+	}
+	// Return based on the number of processors, optimizing for the common case of 1 processor
+	// If there are no processors, return nil
+	switch len(processors) {
+	case 0:
+		return nil
+	case 1:
+		return processors[0]
+	default:
+		return &compositePreSearchResultProcessor{
+			presearchResultProcessors: processors,
+		}
 	}
-	return &knnPreSearchResultProcessor{} // equivalent to nil
 }

vendor/github.com/blevesearch/bleve/v2/registry/analyzer.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterAnalyzer(name string, constructor AnalyzerConstructor) {
+func RegisterAnalyzer(name string, constructor AnalyzerConstructor) error {
 	_, exists := analyzers[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate analyzer named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate analyzer named '%s'", name)
 	}
 	analyzers[name] = constructor
+	return nil
 }
 
 type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Analyzer, error)

vendor/github.com/blevesearch/bleve/v2/registry/char_filter.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterCharFilter(name string, constructor CharFilterConstructor) {
+func RegisterCharFilter(name string, constructor CharFilterConstructor) error {
 	_, exists := charFilters[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate char filter named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate char filter named '%s'", name)
 	}
 	charFilters[name] = constructor
+	return nil
 }
 
 type CharFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.CharFilter, error)

vendor/github.com/blevesearch/bleve/v2/registry/datetime_parser.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) {
+func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) error {
 	_, exists := dateTimeParsers[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate date time parser named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate date time parser named '%s'", name)
 	}
 	dateTimeParsers[name] = constructor
+	return nil
 }
 
 type DateTimeParserConstructor func(config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error)

vendor/github.com/blevesearch/bleve/v2/registry/fragment_formatter.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/search/highlight"
 )
 
-func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) {
+func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) error {
 	_, exists := fragmentFormatters[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate fragment formatter named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate fragment formatter named '%s'", name)
 	}
 	fragmentFormatters[name] = constructor
+	return nil
 }
 
 type FragmentFormatterConstructor func(config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error)

vendor/github.com/blevesearch/bleve/v2/registry/fragmenter.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/search/highlight"
 )
 
-func RegisterFragmenter(name string, constructor FragmenterConstructor) {
+func RegisterFragmenter(name string, constructor FragmenterConstructor) error {
 	_, exists := fragmenters[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate fragmenter named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate fragmenter named '%s'", name)
 	}
 	fragmenters[name] = constructor
+	return nil
 }
 
 type FragmenterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error)

vendor/github.com/blevesearch/bleve/v2/registry/highlighter.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/search/highlight"
 )
 
-func RegisterHighlighter(name string, constructor HighlighterConstructor) {
+func RegisterHighlighter(name string, constructor HighlighterConstructor) error {
 	_, exists := highlighters[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate highlighter named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate highlighter named '%s'", name)
 	}
 	highlighters[name] = constructor
+	return nil
 }
 
 type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error)

vendor/github.com/blevesearch/bleve/v2/registry/index_type.go

@@ -20,12 +20,13 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
-func RegisterIndexType(name string, constructor IndexTypeConstructor) {
+func RegisterIndexType(name string, constructor IndexTypeConstructor) error {
 	_, exists := indexTypes[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate index encoding named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate index encoding named '%s'", name)
 	}
 	indexTypes[name] = constructor
+	return nil
 }
 
 type IndexTypeConstructor func(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error)
@@ -38,7 +39,7 @@ func IndexTypeConstructorByName(name string) IndexTypeConstructor {
 func IndexTypesAndInstances() ([]string, []string) {
 	var types []string
 	var instances []string
-	for name := range stores {
+	for name := range indexTypes {
 		types = append(types, name)
 	}
 	return types, instances

vendor/github.com/blevesearch/bleve/v2/registry/registry.go

@@ -36,6 +36,7 @@ var tokenMaps = make(TokenMapRegistry, 0)
 var tokenFilters = make(TokenFilterRegistry, 0)
 var analyzers = make(AnalyzerRegistry, 0)
 var dateTimeParsers = make(DateTimeParserRegistry, 0)
+var synonymSources = make(SynonymSourceRegistry, 0)
 
 type Cache struct {
 	CharFilters        *CharFilterCache
@@ -47,6 +48,7 @@ type Cache struct {
 	FragmentFormatters *FragmentFormatterCache
 	Fragmenters        *FragmenterCache
 	Highlighters       *HighlighterCache
+	SynonymSources     *SynonymSourceCache
 }
 
 func NewCache() *Cache {
@@ -60,6 +62,7 @@ func NewCache() *Cache {
 		FragmentFormatters: NewFragmentFormatterCache(),
 		Fragmenters:        NewFragmenterCache(),
 		Highlighters:       NewHighlighterCache(),
+		SynonymSources:     NewSynonymSourceCache(),
 	}
 }
 
@@ -147,6 +150,14 @@ func (c *Cache) DefineDateTimeParser(name string, config map[string]interface{})
 	return c.DateTimeParsers.DefineDateTimeParser(name, typ, config, c)
 }
 
+func (c *Cache) SynonymSourceNamed(name string) (analysis.SynonymSource, error) {
+	return c.SynonymSources.SynonymSourceNamed(name, c)
+}
+
+func (c *Cache) DefineSynonymSource(name string, config map[string]interface{}) (analysis.SynonymSource, error) {
+	return c.SynonymSources.DefineSynonymSource(name, analysis.SynonymSourceType, config, c)
+}
+
 func (c *Cache) FragmentFormatterNamed(name string) (highlight.FragmentFormatter, error) {
 	return c.FragmentFormatters.FragmentFormatterNamed(name, c)
 }

vendor/github.com/blevesearch/bleve/v2/registry/store.go

@@ -17,15 +17,16 @@ package registry
 import (
 	"fmt"
 
-	"github.com/blevesearch/upsidedown_store_api"
+	store "github.com/blevesearch/upsidedown_store_api"
 )
 
-func RegisterKVStore(name string, constructor KVStoreConstructor) {
+func RegisterKVStore(name string, constructor KVStoreConstructor) error {
 	_, exists := stores[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate store named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate store named '%s'", name)
 	}
 	stores[name] = constructor
+	return nil
 }
 
 // KVStoreConstructor is used to build a KVStore of a specific type when

vendor/github.com/blevesearch/bleve/v2/registry/synonym_source.go

@@ -0,0 +1,86 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package registry
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+)
+
+func RegisterSynonymSource(typ string, constructor SynonymSourceConstructor) error {
+	_, exists := synonymSources[typ]
+	if exists {
+		return fmt.Errorf("attempted to register duplicate synonym source with type '%s'", typ)
+	}
+	synonymSources[typ] = constructor
+	return nil
+}
+
+type SynonymSourceCache struct {
+	*ConcurrentCache
+}
+
+func NewSynonymSourceCache() *SynonymSourceCache {
+	return &SynonymSourceCache{
+		NewConcurrentCache(),
+	}
+}
+
+type SynonymSourceConstructor func(config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error)
+type SynonymSourceRegistry map[string]SynonymSourceConstructor
+
+func SynonymSourceBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
+	cons, registered := synonymSources[name]
+	if !registered {
+		return nil, fmt.Errorf("no synonym source with name '%s' registered", name)
+	}
+	synonymSource, err := cons(config, cache)
+	if err != nil {
+		return nil, fmt.Errorf("error building synonym source: %v", err)
+	}
+	return synonymSource, nil
+}
+
+func (c *SynonymSourceCache) SynonymSourceNamed(name string, cache *Cache) (analysis.SynonymSource, error) {
+	item, err := c.ItemNamed(name, cache, SynonymSourceBuild)
+	if err != nil {
+		return nil, err
+	}
+	return item.(analysis.SynonymSource), nil
+}
+
+func (c *SynonymSourceCache) DefineSynonymSource(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error) {
+	item, err := c.DefineItem(name, typ, config, cache, SynonymSourceBuild)
+	if err != nil {
+		if err == ErrAlreadyDefined {
+			return nil, fmt.Errorf("synonym source named '%s' already defined", name)
+		}
+		return nil, err
+	}
+	return item.(analysis.SynonymSource), nil
+}
+
+func (c *SynonymSourceCache) VisitSynonymSources(visitor analysis.SynonymSourceVisitor) error {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+	for k, v := range c.data {
+		err := visitor(k, v.(analysis.SynonymSource))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/blevesearch/bleve/v2/registry/token_filter.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterTokenFilter(name string, constructor TokenFilterConstructor) {
+func RegisterTokenFilter(name string, constructor TokenFilterConstructor) error {
 	_, exists := tokenFilters[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate token filter named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate token filter named '%s'", name)
 	}
 	tokenFilters[name] = constructor
+	return nil
 }
 
 type TokenFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error)

vendor/github.com/blevesearch/bleve/v2/registry/token_maps.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterTokenMap(name string, constructor TokenMapConstructor) {
+func RegisterTokenMap(name string, constructor TokenMapConstructor) error {
 	_, exists := tokenMaps[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate token map named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate token map named '%s'", name)
 	}
 	tokenMaps[name] = constructor
+	return nil
 }
 
 type TokenMapConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenMap, error)

vendor/github.com/blevesearch/bleve/v2/registry/tokenizer.go

@@ -20,12 +20,13 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis"
 )
 
-func RegisterTokenizer(name string, constructor TokenizerConstructor) {
+func RegisterTokenizer(name string, constructor TokenizerConstructor) error {
 	_, exists := tokenizers[name]
 	if exists {
-		panic(fmt.Errorf("attempted to register duplicate tokenizer named '%s'", name))
+		return fmt.Errorf("attempted to register duplicate tokenizer named '%s'", name)
 	}
 	tokenizers[name] = constructor
+	return nil
 }
 
 type TokenizerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error)

vendor/github.com/blevesearch/bleve/v2/search.go

@@ -31,8 +31,10 @@ import (
 	"github.com/blevesearch/bleve/v2/util"
 )
 
-var reflectStaticSizeSearchResult int
-var reflectStaticSizeSearchStatus int
+var (
+	reflectStaticSizeSearchResult int
+	reflectStaticSizeSearchStatus int
+)
 
 func init() {
 	var sr SearchResult
@@ -444,6 +446,12 @@ type SearchResult struct {
 	MaxScore float64                        `json:"max_score"`
 	Took     time.Duration                  `json:"took"`
 	Facets   search.FacetResults            `json:"facets"`
+	// special fields that are applicable only for search
+	// results that are obtained from a presearch
+	SynonymResult search.FieldTermSynonymMap `json:"synonym_result,omitempty"`
+
+	// The following fields are applicable to BM25 preSearch
+	BM25Stats *search.BM25Stats `json:"bm25_stats,omitempty"`
 }
 
 func (sr *SearchResult) Size() int {
@@ -491,7 +499,7 @@ func (sr *SearchResult) String() string {
 		rv = "No matches"
 	}
 	if len(sr.Facets) > 0 {
-		rv += fmt.Sprintf("Facets:\n")
+		rv += "Facets:\n"
 		for fn, f := range sr.Facets {
 			rv += fmt.Sprintf("%s(%d)\n", fn, f.Total)
 			for _, t := range f.Terms.Terms() {
@@ -589,3 +597,13 @@ func (r *SearchRequest) SortFunc() func(data sort.Interface) {
 
 	return sort.Sort
 }
+
+func isMatchNoneQuery(q query.Query) bool {
+	_, ok := q.(*query.MatchNoneQuery)
+	return ok
+}
+
+func isMatchAllQuery(q query.Query) bool {
+	_, ok := q.(*query.MatchAllQuery)
+	return ok
+}

vendor/github.com/blevesearch/bleve/v2/search/collector/eligible.go

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build vectors
+// +build vectors
+
 package collector
 
 import (
@@ -24,12 +27,10 @@ import (
 )
 
 type EligibleCollector struct {
-	size    int
-	total   uint64
-	took    time.Duration
-	results search.DocumentMatchCollection
-
-	ids []index.IndexInternalID
+	size             int
+	total            uint64
+	took             time.Duration
+	eligibleSelector index.EligibleDocumentSelector
 }
 
 func NewEligibleCollector(size int) *EligibleCollector {
@@ -38,28 +39,33 @@ func NewEligibleCollector(size int) *EligibleCollector {
 
 func newEligibleCollector(size int) *EligibleCollector {
 	// No sort order & skip always 0 since this is only to filter eligible docs.
-	ec := &EligibleCollector{size: size,
-		ids: make([]index.IndexInternalID, 0, size),
+	ec := &EligibleCollector{
+		size: size,
 	}
 	return ec
 }
 
-func makeEligibleDocumentMatchHandler(ctx *search.SearchContext) (search.DocumentMatchHandler, error) {
+func makeEligibleDocumentMatchHandler(ctx *search.SearchContext, reader index.IndexReader) (search.DocumentMatchHandler, error) {
 	if ec, ok := ctx.Collector.(*EligibleCollector); ok {
-		return func(d *search.DocumentMatch) error {
-			if d == nil {
+		if vr, ok := reader.(index.VectorIndexReader); ok {
+			// create a new eligible document selector to add eligible document matches
+			ec.eligibleSelector = vr.NewEligibleDocumentSelector()
+			// return a document match handler that adds eligible document matches
+			// to the eligible document selector
+			return func(d *search.DocumentMatch) error {
+				if d == nil {
+					return nil
+				}
+				err := ec.eligibleSelector.AddEligibleDocumentMatch(d.IndexInternalID)
+				if err != nil {
+					return err
+				}
+				// recycle the DocumentMatch
+				ctx.DocumentMatchPool.Put(d)
 				return nil
-			}
-
-			copyOfID := make([]byte, len(d.IndexInternalID))
-			copy(copyOfID, d.IndexInternalID)
-			ec.ids = append(ec.ids, copyOfID)
-
-			// recycle the DocumentMatch
-			ctx.DocumentMatchPool.Put(d)
-
-			return nil
-		}, nil
+			}, nil
+		}
+		return nil, fmt.Errorf("reader is not a VectorIndexReader")
 	}
 
 	return nil, fmt.Errorf("eligiblity collector not available")
@@ -80,7 +86,7 @@ func (ec *EligibleCollector) Collect(ctx context.Context, searcher search.Search
 		IndexReader:       reader,
 	}
 
-	dmHandler, err := makeEligibleDocumentMatchHandler(searchContext)
+	dmHandler, err := makeEligibleDocumentMatchHandler(searchContext, reader)
 	if err != nil {
 		return err
 	}
@@ -126,12 +132,21 @@ func (ec *EligibleCollector) Collect(ctx context.Context, searcher search.Search
 	return nil
 }
 
+// The eligible collector does not return any document matches and hence
+// this method is a dummy method returning nil, to conform to the
+// search.Collector interface.
 func (ec *EligibleCollector) Results() search.DocumentMatchCollection {
 	return nil
 }
 
-func (ec *EligibleCollector) IDs() []index.IndexInternalID {
-	return ec.ids
+// EligibleSelector returns the eligible document selector, which can be used
+// to retrieve the list of eligible documents from this collector.
+// If the collector has no results, it returns nil.
+func (ec *EligibleCollector) EligibleSelector() index.EligibleDocumentSelector {
+	if ec.total == 0 {
+		return nil
+	}
+	return ec.eligibleSelector
 }
 
 func (ec *EligibleCollector) Total() uint64 {

vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/blevesearch/bleve/v2/numeric"
 	"github.com/blevesearch/bleve/v2/search"
 	"github.com/blevesearch/bleve/v2/size"
 	index "github.com/blevesearch/bleve_index_api"
@@ -500,7 +501,23 @@ func (hc *TopNCollector) finalizeResults(r index.IndexReader) error {
 		doc.Complete(nil)
 		return nil
 	})
+	if err != nil {
+		return err
+	}
 
+	// Decode geo sort keys back to its distance values
+	for i, so := range hc.sort {
+		if _, ok := so.(*search.SortGeoDistance); ok {
+			for _, dm := range hc.results {
+				// The string is a int64 bit representation of a float64 distance
+				distInt, err := numeric.PrefixCoded(dm.Sort[i]).Int64()
+				if err != nil {
+					return err
+				}
+				dm.Sort[i] = strconv.FormatFloat(numeric.Int64ToFloat64(distInt), 'f', -1, 64)
+			}
+		}
+	}
 	return err
 }
 

vendor/github.com/blevesearch/bleve/v2/search/explanation.go

@@ -30,9 +30,10 @@ func init() {
 }
 
 type Explanation struct {
-	Value    float64        `json:"value"`
-	Message  string         `json:"message"`
-	Children []*Explanation `json:"children,omitempty"`
+	Value        float64        `json:"value"`
+	Message      string         `json:"message"`
+	PartialMatch bool           `json:"partial_match,omitempty"`
+	Children     []*Explanation `json:"children,omitempty"`
 }
 
 func (expl *Explanation) String() string {

vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go

@@ -24,8 +24,10 @@ import (
 	"github.com/blevesearch/bleve/v2/size"
 )
 
-var reflectStaticSizeDateTimeFacetBuilder int
-var reflectStaticSizedateTimeRange int
+var (
+	reflectStaticSizeDateTimeFacetBuilder int
+	reflectStaticSizedateTimeRange        int
+)
 
 func init() {
 	var dtfb DateTimeFacetBuilder
@@ -62,12 +64,12 @@ func (fb *DateTimeFacetBuilder) Size() int {
 	sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr +
 		len(fb.field)
 
-	for k, _ := range fb.termsCount {
+	for k := range fb.termsCount {
 		sizeInBytes += size.SizeOfString + len(k) +
 			size.SizeOfInt
 	}
 
-	for k, _ := range fb.ranges {
+	for k := range fb.ranges {
 		sizeInBytes += size.SizeOfString + len(k) +
 			size.SizeOfPtr + reflectStaticSizedateTimeRange
 	}

vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go

@@ -23,8 +23,10 @@ import (
 	"github.com/blevesearch/bleve/v2/size"
 )
 
-var reflectStaticSizeNumericFacetBuilder int
-var reflectStaticSizenumericRange int
+var (
+	reflectStaticSizeNumericFacetBuilder int
+	reflectStaticSizenumericRange        int
+)
 
 func init() {
 	var nfb NumericFacetBuilder
@@ -61,12 +63,12 @@ func (fb *NumericFacetBuilder) Size() int {
 	sizeInBytes := reflectStaticSizeNumericFacetBuilder + size.SizeOfPtr +
 		len(fb.field)
 
-	for k, _ := range fb.termsCount {
+	for k := range fb.termsCount {
 		sizeInBytes += size.SizeOfString + len(k) +
 			size.SizeOfInt
 	}
 
-	for k, _ := range fb.ranges {
+	for k := range fb.ranges {
 		sizeInBytes += size.SizeOfString + len(k) +
 			size.SizeOfPtr + reflectStaticSizenumericRange
 	}

vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go

@@ -50,7 +50,7 @@ func (fb *TermsFacetBuilder) Size() int {
 	sizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr +
 		len(fb.field)
 
-	for k, _ := range fb.termsCount {
+	for k := range fb.termsCount {
 		sizeInBytes += size.SizeOfString + len(k) +
 			size.SizeOfInt
 	}

vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go

@@ -87,5 +87,8 @@ func Constructor(config map[string]interface{}, cache *registry.Cache) (highligh
 }
 
 func init() {
-	registry.RegisterFragmentFormatter(Name, Constructor)
+	err := registry.RegisterFragmentFormatter(Name, Constructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go

@@ -149,5 +149,8 @@ func Constructor(config map[string]interface{}, cache *registry.Cache) (highligh
 }
 
 func init() {
-	registry.RegisterFragmenter(Name, Constructor)
+	err := registry.RegisterFragmenter(Name, Constructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go

@@ -46,5 +46,8 @@ func Constructor(config map[string]interface{}, cache *registry.Cache) (highligh
 }
 
 func init() {
-	registry.RegisterHighlighter(Name, Constructor)
+	err := registry.RegisterHighlighter(Name, Constructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go

@@ -17,6 +17,7 @@ package simple
 import (
 	"container/heap"
 	"fmt"
+
 	index "github.com/blevesearch/bleve_index_api"
 
 	"github.com/blevesearch/bleve/v2/registry"
@@ -217,5 +218,8 @@ func Constructor(config map[string]interface{}, cache *registry.Cache) (highligh
 }
 
 func init() {
-	registry.RegisterHighlighter(Name, Constructor)
+	err := registry.RegisterHighlighter(Name, Constructor)
+	if err != nil {
+		panic(err)
+	}
 }

vendor/github.com/blevesearch/bleve/v2/search/levenshtein.go

@@ -68,6 +68,10 @@ func LevenshteinDistanceMaxReuseSlice(a, b string, max int, d []int) (int, bool,
 	ld := int(math.Abs(float64(la - lb)))
 	if ld > max {
 		return max, true, d
+	} else if la == 0 || lb == 0 {
+		// if one string of the two strings is empty, then ld is
+		// the length of the other string and as such is <= max
+		return ld, false, d
 	}
 
 	if cap(d) < la+1 {

vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go

@@ -49,9 +49,7 @@ func (q *ConjunctionQuery) Boost() float64 {
 }
 
 func (q *ConjunctionQuery) AddQuery(aq ...Query) {
-	for _, aaq := range aq {
-		q.Conjuncts = append(q.Conjuncts, aaq)
-	}
+	q.Conjuncts = append(q.Conjuncts, aq...)
 }
 
 func (q *ConjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {

vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go

@@ -56,9 +56,7 @@ func (q *DisjunctionQuery) Boost() float64 {
 }
 
 func (q *DisjunctionQuery) AddQuery(aq ...Query) {
-	for _, aaq := range aq {
-		q.Disjuncts = append(q.Disjuncts, aaq)
-	}
+	q.Disjuncts = append(q.Disjuncts, aq...)
 }
 
 func (q *DisjunctionQuery) SetMin(m float64) {
@@ -66,7 +64,8 @@ func (q *DisjunctionQuery) SetMin(m float64) {
 }
 
 func (q *DisjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,
-	options search.SearcherOptions) (search.Searcher, error) {
+	options search.SearcherOptions,
+) (search.Searcher, error) {
 	ss := make([]search.Searcher, 0, len(q.Disjuncts))
 	for _, disjunct := range q.Disjuncts {
 		sr, err := disjunct.Searcher(ctx, i, m, options)

vendor/github.com/blevesearch/bleve/v2/search/query/fuzzy.go

@@ -20,6 +20,7 @@ import (
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search"
 	"github.com/blevesearch/bleve/v2/search/searcher"
+	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
 )
 
@@ -29,6 +30,7 @@ type FuzzyQuery struct {
 	Fuzziness int    `json:"fuzziness"`
 	FieldVal  string `json:"field,omitempty"`
 	BoostVal  *Boost `json:"boost,omitempty"`
+	autoFuzzy bool
 }
 
 // NewFuzzyQuery creates a new Query which finds
@@ -66,6 +68,10 @@ func (q *FuzzyQuery) SetFuzziness(f int) {
 	q.Fuzziness = f
 }
 
+func (q *FuzzyQuery) SetAutoFuzziness(a bool) {
+	q.autoFuzzy = a
+}
+
 func (q *FuzzyQuery) SetPrefix(p int) {
 	q.Prefix = p
 }
@@ -75,5 +81,54 @@ func (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mappin
 	if q.FieldVal == "" {
 		field = m.DefaultSearchField()
 	}
+	if q.autoFuzzy {
+		return searcher.NewAutoFuzzySearcher(ctx, i, q.Term, q.Prefix, field, q.BoostVal.Value(), options)
+	}
 	return searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options)
 }
+
+func (q *FuzzyQuery) UnmarshalJSON(data []byte) error {
+	type Alias FuzzyQuery
+	aux := &struct {
+		Fuzziness interface{} `json:"fuzziness"`
+		*Alias
+	}{
+		Alias: (*Alias)(q),
+	}
+	if err := util.UnmarshalJSON(data, &aux); err != nil {
+		return err
+	}
+	switch v := aux.Fuzziness.(type) {
+	case float64:
+		q.Fuzziness = int(v)
+	case string:
+		if v == "auto" {
+			q.autoFuzzy = true
+		}
+	}
+	return nil
+}
+
+func (f *FuzzyQuery) MarshalJSON() ([]byte, error) {
+	var fuzzyValue interface{}
+	if f.autoFuzzy {
+		fuzzyValue = "auto"
+	} else {
+		fuzzyValue = f.Fuzziness
+	}
+	type fuzzyQuery struct {
+		Term      string      `json:"term"`
+		Prefix    int         `json:"prefix_length"`
+		Fuzziness interface{} `json:"fuzziness"`
+		FieldVal  string      `json:"field,omitempty"`
+		BoostVal  *Boost      `json:"boost,omitempty"`
+	}
+	aux := fuzzyQuery{
+		Term:      f.Term,
+		Prefix:    f.Prefix,
+		Fuzziness: fuzzyValue,
+		FieldVal:  f.FieldVal,
+		BoostVal:  f.BoostVal,
+	}
+	return util.MarshalJSON(aux)
+}

vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go

@@ -85,6 +85,9 @@ func (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader,
 }
 
 func (q *GeoBoundingBoxQuery) Validate() error {
+	if q.TopLeft[1] < q.BottomRight[1] {
+		return fmt.Errorf("geo bounding box top left should be higher than bottom right")
+	}
 	return nil
 }