build(deps): bump github.com/blevesearch/bleve/v2 from 2.5.0 to 2.5.1

Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/blevesearch/bleve/releases)
- [Commits](https://github.com/blevesearch/bleve/compare/v2.5.0...v2.5.1)

---
updated-dependencies:
- dependency-name: github.com/blevesearch/bleve/v2
  dependency-version: 2.5.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

vendor/github.com/blevesearch/bleve/v2/document/field_geoshape.go

@@ -143,7 +143,27 @@ func NewGeoShapeFieldFromBytes(name string, arrayPositions []uint64,
 func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
 	coordinates [][][][]float64, typ string,
 	options index.FieldIndexingOptions) *GeoShapeField {
-	shape, encodedValue, err := geo.NewGeoJsonShape(coordinates, typ)
+	shape := &geojson.GeoShape{
+		Coordinates: coordinates,
+		Type:        typ,
+	}
+
+	return NewGeoShapeFieldFromShapeWithIndexingOptions(name,
+		arrayPositions, shape, options)
+}
+
+func NewGeoShapeFieldFromShapeWithIndexingOptions(name string, arrayPositions []uint64,
+	geoShape *geojson.GeoShape, options index.FieldIndexingOptions) *GeoShapeField {
+
+	var shape index.GeoJSON
+	var encodedValue []byte
+	var err error
+
+	if geoShape.Type == geo.CircleType {
+		shape, encodedValue, err = geo.NewGeoCircleShape(geoShape.Center, geoShape.Radius)
+	} else {
+		shape, encodedValue, err = geo.NewGeoJsonShape(geoShape.Coordinates, geoShape.Type)
+	}
 	if err != nil {
 		return nil
 	}
@@ -158,7 +178,9 @@ func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
 		return nil
 	}
 
-	options = options | DefaultGeoShapeIndexingOptions
+	// docvalues are always enabled for geoshape fields, even if the
+	// indexing options are set to not include docvalues.
+	options = options | index.DocValues
 
 	return &GeoShapeField{
 		shape:             shape,
@@ -174,7 +196,26 @@ func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
 func NewGeometryCollectionFieldWithIndexingOptions(name string,
 	arrayPositions []uint64, coordinates [][][][][]float64, types []string,
 	options index.FieldIndexingOptions) *GeoShapeField {
-	shape, encodedValue, err := geo.NewGeometryCollection(coordinates, types)
+	if len(coordinates) != len(types) {
+		return nil
+	}
+
+	shapes := make([]*geojson.GeoShape, len(types))
+	for i := range coordinates {
+		shapes[i] = &geojson.GeoShape{
+			Coordinates: coordinates[i],
+			Type:        types[i],
+		}
+	}
+
+	return NewGeometryCollectionFieldFromShapesWithIndexingOptions(name,
+		arrayPositions, shapes, options)
+}
+
+func NewGeometryCollectionFieldFromShapesWithIndexingOptions(name string,
+	arrayPositions []uint64, geoShapes []*geojson.GeoShape,
+	options index.FieldIndexingOptions) *GeoShapeField {
+	shape, encodedValue, err := geo.NewGeometryCollectionFromShapes(geoShapes)
 	if err != nil {
 		return nil
 	}
@@ -189,7 +230,9 @@ func NewGeometryCollectionFieldWithIndexingOptions(name string,
 		return nil
 	}
 
-	options = options | DefaultGeoShapeIndexingOptions
+	// docvalues are always enabled for geoshape fields, even if the
+	// indexing options are set to not include docvalues.
+	options = options | index.DocValues
 
 	return &GeoShapeField{
 		shape:             shape,
@@ -205,32 +248,15 @@ func NewGeometryCollectionFieldWithIndexingOptions(name string,
 func NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64,
 	centerPoint []float64, radius string,
 	options index.FieldIndexingOptions) *GeoShapeField {
-	shape, encodedValue, err := geo.NewGeoCircleShape(centerPoint, radius)
-	if err != nil {
-		return nil
+
+	shape := &geojson.GeoShape{
+		Center: centerPoint,
+		Radius: radius,
+		Type:   geo.CircleType,
 	}
 
-	// extra glue bytes to work around the term splitting logic from interfering
-	// the custom encoding of the geoshape coordinates inside the docvalues.
-	encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)
-
-	// get the byte value for the circle.
-	value, err := shape.Value()
-	if err != nil {
-		return nil
-	}
-
-	options = options | DefaultGeoShapeIndexingOptions
-
-	return &GeoShapeField{
-		shape:             shape,
-		name:              name,
-		arrayPositions:    arrayPositions,
-		options:           options,
-		encodedValue:      encodedValue,
-		value:             value,
-		numPlainTextBytes: uint64(len(value)),
-	}
+	return NewGeoShapeFieldFromShapeWithIndexingOptions(name,
+		arrayPositions, shape, options)
 }
 
 // GeoShape is an implementation of the index.GeoShapeField interface.

vendor/github.com/blevesearch/bleve/v2/document/field_vector.go

@@ -109,7 +109,6 @@ func NewVectorField(name string, arrayPositions []uint64,
 func NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64,
 	vector []float32, dims int, similarity, vectorIndexOptimizedFor string,
 	options index.FieldIndexingOptions) *VectorField {
-	options = options | DefaultVectorIndexingOptions
 
 	return &VectorField{
 		name:                    name,

vendor/github.com/blevesearch/bleve/v2/geo/README.md

@@ -274,4 +274,10 @@ First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo
 
 - All of the APIs will use float64 for lon/lat values.
 - When describing a point in function arguments or return values, we always use the order lon, lat.
-- High level APIs will use TopLeft and BottomRight to describe bounding boxes.  This may not map cleanly to min/max lon/lat when crossing the dateline.  The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.
+- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly.
+- Points and MultiPoints may only contain Points and MultiPoints.
+- LineStrings and MultiLineStrings may only contain Points and MultiPoints.
+- Polygons or MultiPolygons intersecting Polygons and MultiPolygons may return arbitrary results when the overlap is only an edge or a vertex.
+- Circles containing polygon will return a false positive result if all of the vertices of the polygon are within the circle, but the orientation of those points are clock-wise.
+- The edges of an Envelope follows the latitude and logitude lines instead of the shortest path on a globe.
+- Envelope intersecting queries with LineStrings, MultiLineStrings, Polygons and MultiPolygons implicitly converts the Envelope into a Polygon which changes the curvature of the edges causing inaccurate results for few edge cases.

vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go

@@ -396,8 +396,21 @@ func (pd *pointDistance) QueryTokens(s *S2SpatialAnalyzerPlugin) []string {
 // can be used later while filering the doc values.
 func NewGeometryCollection(coordinates [][][][][]float64,
 	typs []string) (index.GeoJSON, []byte, error) {
+	shapes := make([]*geojson.GeoShape, len(coordinates))
+	for i := range coordinates {
+		shapes[i] = &geojson.GeoShape{
+			Coordinates: coordinates[i],
+			Type:        typs[i],
+		}
+	}
 
-	return geojson.NewGeometryCollection(coordinates, typs)
+	return geojson.NewGeometryCollection(shapes)
+}
+
+func NewGeometryCollectionFromShapes(shapes []*geojson.GeoShape) (
+	index.GeoJSON, []byte, error) {
+
+	return geojson.NewGeometryCollection(shapes)
 }
 
 // NewGeoCircleShape instantiate a circle shape and

vendor/github.com/blevesearch/bleve/v2/geo/parse.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/blevesearch/bleve/v2/util"
+	"github.com/blevesearch/geo/geojson"
 )
 
 // ExtractGeoPoint takes an arbitrary interface{} and tries it's best to
@@ -298,10 +299,15 @@ func ParseGeoShapeField(thing interface{}) (interface{}, string, error) {
 	return coordValue, strings.ToLower(shape), nil
 }
 
-func extractGeoShape(thing interface{}) ([][][][]float64, string, bool) {
+func extractGeoShape(thing interface{}) (*geojson.GeoShape, bool) {
+
 	coordValue, typ, err := ParseGeoShapeField(thing)
 	if err != nil {
-		return nil, "", false
+		return nil, false
+	}
+
+	if typ == CircleType {
+		return ExtractCircle(thing)
 	}
 
 	return ExtractGeoShapeCoordinates(coordValue, typ)
@@ -309,13 +315,12 @@ func extractGeoShape(thing interface{}) ([][][][]float64, string, bool) {
 
 // ExtractGeometryCollection takes an interface{} and tries it's best to
 // interpret all the member geojson shapes within it.
-func ExtractGeometryCollection(thing interface{}) ([][][][][]float64, []string, bool) {
+func ExtractGeometryCollection(thing interface{}) ([]*geojson.GeoShape, bool) {
 	thingVal := reflect.ValueOf(thing)
 	if !thingVal.IsValid() {
-		return nil, nil, false
+		return nil, false
 	}
-	var rv [][][][][]float64
-	var types []string
+	var rv []*geojson.GeoShape
 	var f bool
 
 	if thingVal.Kind() == reflect.Map {
@@ -331,70 +336,74 @@ func ExtractGeometryCollection(thing interface{}) ([][][][][]float64, []string,
 				items := reflect.ValueOf(collection)
 
 				for j := 0; j < items.Len(); j++ {
-					coords, shape, found := extractGeoShape(items.Index(j).Interface())
+					shape, found := extractGeoShape(items.Index(j).Interface())
 					if found {
 						f = found
-						rv = append(rv, coords)
-						types = append(types, shape)
+						rv = append(rv, shape)
 					}
 				}
 			}
 		}
 	}
 
-	return rv, types, f
+	return rv, f
 }
 
 // ExtractCircle takes an interface{} and tries it's best to
 // interpret the center point coordinates and the radius for a
 // given circle shape.
-func ExtractCircle(thing interface{}) ([]float64, string, bool) {
+func ExtractCircle(thing interface{}) (*geojson.GeoShape, bool) {
 	thingVal := reflect.ValueOf(thing)
 	if !thingVal.IsValid() {
-		return nil, "", false
+		return nil, false
+	}
+
+	rv := &geojson.GeoShape{
+		Type:   CircleType,
+		Center: make([]float64, 0, 2),
 	}
-	var rv []float64
-	var radiusStr string
 
 	if thingVal.Kind() == reflect.Map {
 		iter := thingVal.MapRange()
 		for iter.Next() {
 
 			if iter.Key().String() == "radius" {
-				radiusStr = iter.Value().Interface().(string)
+				rv.Radius = iter.Value().Interface().(string)
 				continue
 			}
 
 			if iter.Key().String() == "coordinates" {
 				lng, lat, found := ExtractGeoPoint(iter.Value().Interface())
 				if !found {
-					return nil, radiusStr, false
+					return nil, false
 				}
-				rv = append(rv, lng)
-				rv = append(rv, lat)
+				rv.Center = append(rv.Center, lng, lat)
 			}
 		}
 	}
 
-	return rv, radiusStr, true
+	return rv, true
 }
 
 // ExtractGeoShapeCoordinates takes an interface{} and tries it's best to
 // interpret the coordinates for any of the given geoshape typ like
 // a point, multipoint, linestring, multilinestring, polygon, multipolygon,
 func ExtractGeoShapeCoordinates(coordValue interface{},
-	typ string) ([][][][]float64, string, bool) {
-	var rv [][][][]float64
+	typ string) (*geojson.GeoShape, bool) {
+	rv := &geojson.GeoShape{
+		Type: typ,
+	}
+
 	if typ == PointType {
 		point := extractCoordinates(coordValue)
 
 		// ignore the contents with invalid entry.
 		if len(point) < 2 {
-			return nil, typ, false
+			return nil, false
 		}
 
-		rv = [][][][]float64{{{point}}}
-		return rv, typ, true
+		rv.Coordinates = [][][][]float64{{{point}}}
+		return rv, true
 	}
 
 	if typ == MultiPointType || typ == LineStringType ||
@@ -403,19 +412,19 @@ func ExtractGeoShapeCoordinates(coordValue interface{},
 
 		// ignore the contents with invalid entry.
 		if len(coords) == 0 {
-			return nil, typ, false
+			return nil, false
 		}
 
 		if typ == EnvelopeType && len(coords) != 2 {
-			return nil, typ, false
+			return nil, false
 		}
 
 		if typ == LineStringType && len(coords) < 2 {
-			return nil, typ, false
+			return nil, false
 		}
 
-		rv = [][][][]float64{{coords}}
-		return rv, typ, true
+		rv.Coordinates = [][][][]float64{{coords}}
+		return rv, true
 	}
 
 	if typ == PolygonType || typ == MultiLineStringType {
@@ -423,33 +432,34 @@ func ExtractGeoShapeCoordinates(coordValue interface{},
 
 		// ignore the contents with invalid entry.
 		if len(coords) == 0 {
-			return nil, typ, false
+			return nil, false
 		}
 
 		if typ == PolygonType && len(coords[0]) < 3 ||
 			typ == MultiLineStringType && len(coords[0]) < 2 {
-			return nil, typ, false
+			return nil, false
 		}
 
-		rv = [][][][]float64{coords}
-		return rv, typ, true
+		rv.Coordinates = [][][][]float64{coords}
+		return rv, true
 	}
 
 	if typ == MultiPolygonType {
-		rv = extract4DCoordinates(coordValue)
+		coords := extract4DCoordinates(coordValue)
 
 		// ignore the contents with invalid entry.
-		if len(rv) == 0 || len(rv[0]) == 0 {
-			return nil, typ, false
+		if len(coords) == 0 || len(coords[0]) == 0 {
+			return nil, false
 
 		}
 
-		if len(rv[0][0]) < 3 {
-			return nil, typ, false
+		if len(coords[0][0]) < 3 {
+			return nil, false
 		}
 
-		return rv, typ, true
+		rv.Coordinates = coords
+		return rv, true
 	}
 
-	return rv, typ, false
+	return rv, false
 }

vendor/github.com/blevesearch/bleve/v2/index/scorch/event.go

@@ -71,3 +71,7 @@ var EventKindPreMergeCheck = EventKind(9)
 // EventKindIndexStart is fired when Index() is invoked which
 // creates a new Document object from an interface using the index mapping.
 var EventKindIndexStart = EventKind(10)
+
+// EventKindPurgerCheck is fired before the purge code is invoked and decides
+// whether to execute or not. For unit test purposes
+var EventKindPurgerCheck = EventKind(11)

vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go

@@ -81,6 +81,10 @@ OUTER:
 					// Retry instead of blocking/waiting here since a long wait
 					// can result in more segments introduced i.e. s.root will
 					// be updated.
+
+					// decrement the ref count since its no longer needed in this
+					// iteration
+					_ = ourSnapshot.DecRef()
 					continue OUTER
 				}
 
@@ -488,7 +492,11 @@ func closeNewMergedSegments(segs []segment.Segment) error {
 	return nil
 }
 
-func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObjs []*flushable) (*IndexSnapshot, []uint64, error) {
+// mergeAndPersistInMemorySegments takes an IndexSnapshot and a list of in-memory segments,
+// which are merged and persisted to disk concurrently. These are then introduced as
+// the new root snapshot in one-shot.
+func (s *Scorch) mergeAndPersistInMemorySegments(snapshot *IndexSnapshot,
+	flushableObjs []*flushable) (*IndexSnapshot, []uint64, error) {
 	atomic.AddUint64(&s.stats.TotMemMergeBeg, 1)
 
 	memMergeZapStartTime := time.Now()
@@ -507,7 +515,8 @@ func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObj
 	var em sync.Mutex
 	var errs []error
 
-	// deploy the workers to merge and flush the batches of segments parallely
+	// deploy the workers to merge and flush the batches of segments concurrently
+	// and create a new file segment
 	for i := 0; i < numFlushes; i++ {
 		wg.Add(1)
 		go func(segsBatch []segment.Segment, dropsBatch []*roaring.Bitmap, id int) {
@@ -527,6 +536,11 @@ func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObj
 				atomic.AddUint64(&s.stats.TotMemMergeErr, 1)
 				return
 			}
+			// to prevent accidental cleanup of this newly created file, mark it
+			// as ineligible for removal. this will be flipped back when the bolt
+			// is updated - which is valid, since the snapshot updated in bolt is
+			// cleaned up only if its zero ref'd (MB-66163 for more details)
+			s.markIneligibleForRemoval(filename)
 			newMergedSegmentIDs[id] = newSegmentID
 			newDocIDsSet[id] = newDocIDs
 			newMergedSegments[id], err = s.segPlugin.Open(path)
@@ -567,6 +581,8 @@ func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObj
 		atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime)
 	}
 
+	// update the segmentMerge task with the newly merged + flushed segments which
+	// are to be introduced atomically.
 	sm := &segmentMerge{
 		id:               newMergedSegmentIDs,
 		new:              newMergedSegments,
@@ -575,6 +591,10 @@ func (s *Scorch) mergeSegmentBasesParallel(snapshot *IndexSnapshot, flushableObj
 		newCount:         newMergedCount,
 	}
 
+	// create a history map which maps the old in-memory segments with the specific
+	// persister worker (also the specific file segment its going to be part of)
+	// which flushed it out. This map will be used on the introducer side to out-ref
+	// the in-memory segments and also track the new tombstones if present.
 	for i, flushable := range flushableObjs {
 		for j, idx := range flushable.sbIdxs {
 			ss := snapshot.segment[idx]

vendor/github.com/blevesearch/bleve/v2/index/scorch/optimize_knn.go

@@ -137,8 +137,8 @@ func (o *OptimizeVR) Finish() error {
 }
 
 func (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context,
-	octx index.VectorOptimizableContext) (index.VectorOptimizableContext, error) {
-
+	octx index.VectorOptimizableContext,
+) (index.VectorOptimizableContext, error) {
 	if s.snapshot.parent.segPlugin.Version() < VectorSearchSupportedSegmentVersion {
 		return nil, fmt.Errorf("vector search not supported for this index, "+
 			"index's segment version %v, supported segment version for vector search %v",
@@ -146,8 +146,9 @@ func (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context,
 	}
 
 	if octx == nil {
-		octx = &OptimizeVR{snapshot: s.snapshot,
-			vrs: make(map[string][]*IndexSnapshotVectorReader),
+		octx = &OptimizeVR{
+			snapshot: s.snapshot,
+			vrs:      make(map[string][]*IndexSnapshotVectorReader),
 		}
 	}
 

vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go

@@ -228,7 +228,9 @@ OUTER:
 		case s.introducerNotifier <- w:
 		}
 
-		s.removeOldData() // might as well cleanup while waiting
+		if ok := s.fireEvent(EventKindPurgerCheck, 0); ok {
+			s.removeOldData() // might as well cleanup while waiting
+		}
 
 		atomic.AddUint64(&s.stats.TotPersistLoopWait, 1)
 
@@ -296,7 +298,9 @@ func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64,
 	// 1. Too many older snapshots awaiting the clean up.
 	// 2. The merger could be lagging behind on merging the disk files.
 	if numFilesOnDisk > uint64(po.PersisterNapUnderNumFiles) {
-		s.removeOldData()
+		if ok := s.fireEvent(EventKindPurgerCheck, 0); ok {
+			s.removeOldData()
+		}
 		numFilesOnDisk, _, _ = s.diskFileStats(nil)
 	}
 
@@ -481,8 +485,9 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persiste
 		return false, nil
 	}
 
-	// drains out (after merging in memory) the segments in the flushSet parallely
-	newSnapshot, newSegmentIDs, err := s.mergeSegmentBasesParallel(snapshot, flushSet)
+	// the newSnapshot at this point would contain the newly created file segments
+	// and updated with the root.
+	newSnapshot, newSegmentIDs, err := s.mergeAndPersistInMemorySegments(snapshot, flushSet)
 	if err != nil {
 		return false, err
 	}
@@ -529,7 +534,7 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persiste
 		}
 	}
 
-	// append to the equiv the new segment
+	// append to the equiv the newly merged segments
 	for _, segment := range newSnapshot.segment {
 		if _, ok := newMergedSegmentIDs[segment.id]; ok {
 			equiv.segment = append(equiv.segment, &SegmentSnapshot{
@@ -538,7 +543,6 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persiste
 				deleted: nil, // nil since merging handled deletions
 				stats:   nil,
 			})
-			break
 		}
 	}
 
@@ -842,7 +846,7 @@ var (
 )
 
 func (s *Scorch) loadFromBolt() error {
-	return s.rootBolt.View(func(tx *bolt.Tx) error {
+	err := s.rootBolt.View(func(tx *bolt.Tx) error {
 		snapshots := tx.Bucket(boltSnapshotsBucket)
 		if snapshots == nil {
 			return nil
@@ -892,6 +896,16 @@ func (s *Scorch) loadFromBolt() error {
 		}
 		return nil
 	})
+	if err != nil {
+		return err
+	}
+
+	persistedSnapshots, err := s.rootBoltSnapshotMetaData()
+	if err != nil {
+		return err
+	}
+	s.checkPoints = persistedSnapshots
+	return nil
 }
 
 // LoadSnapshot loads the segment with the specified epoch
@@ -1113,7 +1127,10 @@ func getProtectedSnapshots(rollbackSamplingInterval time.Duration,
 	numSnapshotsToKeep int,
 	persistedSnapshots []*snapshotMetaData,
 ) map[uint64]time.Time {
-	lastPoint, protectedEpochs := getTimeSeriesSnapshots(numSnapshotsToKeep,
+	// keep numSnapshotsToKeep - 1 worth of time series snapshots, because we always
+	// must preserve the very latest snapshot in bolt as well to avoid accidental
+	// deletes of bolt entries and cleanups by the purger code.
+	lastPoint, protectedEpochs := getTimeSeriesSnapshots(numSnapshotsToKeep-1,
 		rollbackSamplingInterval, persistedSnapshots)
 	if len(protectedEpochs) < numSnapshotsToKeep {
 		numSnapshotsNeeded := numSnapshotsToKeep - len(protectedEpochs)
@@ -1276,7 +1293,7 @@ func (s *Scorch) removeOldZapFiles() error {
 // duration. This results in all of them being purged from the boltDB
 // and the next iteration of the removeOldData() would end up protecting
 // latest contiguous snapshot which is a poor pattern in the rollback checkpoints.
-// Hence we try to retain atleast retentionFactor portion worth of old snapshots
+// Hence we try to retain atmost retentionFactor portion worth of old snapshots
 // in such a scenario using the following function
 func getBoundaryCheckPoint(retentionFactor float64,
 	checkPoints []*snapshotMetaData, timeStamp time.Time,
@@ -1284,11 +1301,13 @@ func getBoundaryCheckPoint(retentionFactor float64,
 	if checkPoints != nil {
 		boundary := checkPoints[int(math.Floor(float64(len(checkPoints))*
 			retentionFactor))]
-		if timeStamp.Sub(boundary.timeStamp) < 0 {
-			// too less checkPoints would be left.
+		if timeStamp.Sub(boundary.timeStamp) > 0 {
+			// return the extended boundary which will dictate the older snapshots
+			// to be retained
 			return boundary.timeStamp
 		}
 	}
+
 	return timeStamp
 }
 
@@ -1300,7 +1319,10 @@ type snapshotMetaData struct {
 func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) {
 	var rv []*snapshotMetaData
 	currTime := time.Now()
-	expirationDuration := time.Duration(s.numSnapshotsToKeep) * s.rollbackSamplingInterval
+	// including the very latest snapshot there should be n snapshots, so the
+	// very last one would be tc - (n-1) * d
+	// for eg for n = 3 the checkpoints preserved should be tc, tc - d, tc - 2d
+	expirationDuration := time.Duration(s.numSnapshotsToKeep-1) * s.rollbackSamplingInterval
 
 	err := s.rootBolt.View(func(tx *bolt.Tx) error {
 		snapshots := tx.Bucket(boltSnapshotsBucket)
@@ -1309,6 +1331,7 @@ func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) {
 		}
 		sc := snapshots.Cursor()
 		var found bool
+		// traversal order - latest -> oldest epoch
 		for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() {
 			_, snapshotEpoch, err := decodeUvarintAscending(sk)
 			if err != nil {
@@ -1358,7 +1381,6 @@ func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) {
 					err = nil
 				}
 			}
-
 		}
 		return nil
 	})

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go

@@ -81,6 +81,9 @@ type IndexSnapshot struct {
 
 	m2        sync.Mutex                                 // Protects the fields that follow.
 	fieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's
+
+	m3               sync.RWMutex // bm25 metrics specific - not to interfere with TFR creation
+	fieldCardinality map[string]int
 }
 
 func (i *IndexSnapshot) Segments() []*SegmentSnapshot {
@@ -202,6 +205,33 @@ func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string,
 	return rv, nil
 }
 
+func (is *IndexSnapshot) FieldCardinality(field string) (rv int, err error) {
+	is.m3.RLock()
+	rv, ok := is.fieldCardinality[field]
+	is.m3.RUnlock()
+	if ok {
+		return rv, nil
+	}
+
+	is.m3.Lock()
+	defer is.m3.Unlock()
+	if is.fieldCardinality == nil {
+		is.fieldCardinality = make(map[string]int)
+	}
+	// check again to avoid redundant fieldDict creation
+	if rv, ok := is.fieldCardinality[field]; ok {
+		return rv, nil
+	}
+
+	fd, err := is.FieldDict(field)
+	if err != nil {
+		return rv, err
+	}
+	rv = fd.Cardinality()
+	is.fieldCardinality[field] = rv
+	return rv, nil
+}
+
 func (is *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) {
 	return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator {
 		return is.AutomatonIterator(nil, nil, nil)
@@ -301,9 +331,10 @@ func (is *IndexSnapshot) fieldDictRegexp(field string,
 func (is *IndexSnapshot) getLevAutomaton(term string,
 	fuzziness uint8,
 ) (vellum.Automaton, error) {
-	if fuzziness == 1 {
+	switch fuzziness {
+	case 1:
 		return lb1.BuildDfa(term, fuzziness)
-	} else if fuzziness == 2 {
+	case 2:
 		return lb2.BuildDfa(term, fuzziness)
 	}
 	return nil, fmt.Errorf("fuzziness exceeds the max limit")
@@ -1001,32 +1032,22 @@ func (is *IndexSnapshot) CloseCopyReader() error {
 }
 
 func (is *IndexSnapshot) ThesaurusTermReader(ctx context.Context, thesaurusName string, term []byte) (index.ThesaurusTermReader, error) {
-	rv := &IndexSnapshotThesaurusTermReader{}
-	rv.name = thesaurusName
-	rv.snapshot = is
-	if rv.postings == nil {
-		rv.postings = make([]segment.SynonymsList, len(is.segment))
-	}
-	if rv.iterators == nil {
-		rv.iterators = make([]segment.SynonymsIterator, len(is.segment))
-	}
-	rv.segmentOffset = 0
-
-	if rv.thesauri == nil {
-		rv.thesauri = make([]segment.Thesaurus, len(is.segment))
-		for i, s := range is.segment {
-			if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {
-				thes, err := synSeg.Thesaurus(thesaurusName)
-				if err != nil {
-					return nil, err
-				}
-				rv.thesauri[i] = thes
-			}
-		}
+	rv := &IndexSnapshotThesaurusTermReader{
+		name:          thesaurusName,
+		snapshot:      is,
+		postings:      make([]segment.SynonymsList, len(is.segment)),
+		iterators:     make([]segment.SynonymsIterator, len(is.segment)),
+		thesauri:      make([]segment.Thesaurus, len(is.segment)),
+		segmentOffset: 0,
 	}
 
 	for i, s := range is.segment {
-		if _, ok := s.segment.(segment.ThesaurusSegment); ok {
+		if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok {
+			thes, err := synSeg.Thesaurus(thesaurusName)
+			if err != nil {
+				return nil, err
+			}
+			rv.thesauri[i] = thes
 			pl, err := rv.thesauri[i].SynonymsList(term, s.deleted, rv.postings[i])
 			if err != nil {
 				return nil, err

vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index_str.go

@@ -42,11 +42,15 @@ func (i *IndexSnapshotThesaurusTermReader) Size() int {
 		len(i.name) + size.SizeOfString
 
 	for _, postings := range i.postings {
-		sizeInBytes += postings.Size()
+		if postings != nil {
+			sizeInBytes += postings.Size()
+		}
 	}
 
 	for _, iterator := range i.iterators {
-		sizeInBytes += iterator.Size()
+		if iterator != nil {
+			sizeInBytes += iterator.Size()
+		}
 	}
 
 	return sizeInBytes
@@ -64,8 +68,8 @@ func (i *IndexSnapshotThesaurusTermReader) Next() (string, error) {
 				synTerm := next.Term()
 				return synTerm, nil
 			}
-			i.segmentOffset++
 		}
+		i.segmentOffset++
 	}
 	return "", nil
 }

vendor/github.com/blevesearch/bleve/v2/index/upsidedown/index_reader.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/blevesearch/bleve/v2/document"
 	index "github.com/blevesearch/bleve_index_api"
-	"github.com/blevesearch/upsidedown_store_api"
+	store "github.com/blevesearch/upsidedown_store_api"
 )
 
 var reflectStaticSizeIndexReader int

vendor/github.com/blevesearch/bleve/v2/index_alias_impl.go

@@ -634,7 +634,7 @@ func preSearchRequired(ctx context.Context, req *SearchRequest, m mapping.IndexM
 func preSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) {
 	// create a dummy request with a match none query
 	// since we only care about the preSearchData in PreSearch
-	var dummyQuery = req.Query
+	dummyQuery := req.Query
 	if !flags.bm25 && !flags.synonyms {
 		// create a dummy request with a match none query
 		// since we only care about the preSearchData in PreSearch
@@ -734,7 +734,8 @@ func constructBM25PreSearchData(rv map[string]map[string]interface{}, sr *Search
 }
 
 func constructPreSearchData(req *SearchRequest, flags *preSearchFlags,
-	preSearchResult *SearchResult, indexes []Index) (map[string]map[string]interface{}, error) {
+	preSearchResult *SearchResult, indexes []Index,
+) (map[string]map[string]interface{}, error) {
 	if flags == nil || preSearchResult == nil {
 		return nil, fmt.Errorf("invalid input, flags: %v, preSearchResult: %v", flags, preSearchResult)
 	}
@@ -762,7 +763,7 @@ func preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSear
 	asyncResults := make(chan *asyncSearchResult, len(indexes))
 	// run search on each index in separate go routine
 	var waitGroup sync.WaitGroup
-	var searchChildIndex = func(in Index, childReq *SearchRequest) {
+	searchChildIndex := func(in Index, childReq *SearchRequest) {
 		rv := asyncSearchResult{Name: in.Name()}
 		rv.Result, rv.Err = in.SearchInContext(ctx, childReq)
 		asyncResults <- &rv
@@ -827,8 +828,12 @@ func preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSear
 		for indexName, indexErr := range indexErrors {
 			sr.Status.Errors[indexName] = indexErr
 			sr.Status.Total++
-			sr.Status.Failed++
 		}
+		// At this point, all errors have been recorded—either from the preSearch phase
+		// (via status.Merge) or from individual index search failures (indexErrors).
+		// Since partial results are not allowed, mark the entire request as failed.
+		sr.Status.Successful = 0
+		sr.Status.Failed = sr.Status.Total
 	} else {
 		prp.finalize(sr)
 	}
@@ -910,7 +915,6 @@ func hitsInCurrentPage(req *SearchRequest, hits []*search.DocumentMatch) []*sear
 // MultiSearch executes a SearchRequest across multiple Index objects,
 // then merges the results.  The indexes must honor any ctx deadline.
 func MultiSearch(ctx context.Context, req *SearchRequest, preSearchData map[string]map[string]interface{}, indexes ...Index) (*SearchResult, error) {
-
 	searchStart := time.Now()
 	asyncResults := make(chan *asyncSearchResult, len(indexes))
 
@@ -925,7 +929,7 @@ func MultiSearch(ctx context.Context, req *SearchRequest, preSearchData map[stri
 	// run search on each index in separate go routine
 	var waitGroup sync.WaitGroup
 
-	var searchChildIndex = func(in Index, childReq *SearchRequest) {
+	searchChildIndex := func(in Index, childReq *SearchRequest) {
 		rv := asyncSearchResult{Name: in.Name()}
 		rv.Result, rv.Err = in.SearchInContext(ctx, childReq)
 		asyncResults <- &rv

vendor/github.com/blevesearch/bleve/v2/index_impl.go

@@ -59,11 +59,15 @@ const storePath = "store"
 
 var mappingInternalKey = []byte("_mapping")
 
-const SearchQueryStartCallbackKey = "_search_query_start_callback_key"
-const SearchQueryEndCallbackKey = "_search_query_end_callback_key"
+const (
+	SearchQueryStartCallbackKey search.ContextKey = "_search_query_start_callback_key"
+	SearchQueryEndCallbackKey   search.ContextKey = "_search_query_end_callback_key"
+)
 
-type SearchQueryStartCallbackFn func(size uint64) error
-type SearchQueryEndCallbackFn func(size uint64) error
+type (
+	SearchQueryStartCallbackFn func(size uint64) error
+	SearchQueryEndCallbackFn   func(size uint64) error
+)
 
 func indexStorePath(path string) string {
 	return path + string(os.PathSeparator) + storePath
@@ -412,10 +416,12 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) {
 	return i.SearchInContext(context.Background(), req)
 }
 
-var documentMatchEmptySize int
-var searchContextEmptySize int
-var facetResultEmptySize int
-var documentEmptySize int
+var (
+	documentMatchEmptySize int
+	searchContextEmptySize int
+	facetResultEmptySize   int
+	documentEmptySize      int
+)
 
 func init() {
 	var dm search.DocumentMatch
@@ -435,8 +441,8 @@ func init() {
 // needed to execute a search request.
 func memNeededForSearch(req *SearchRequest,
 	searcher search.Searcher,
-	topnCollector *collector.TopNCollector) uint64 {
-
+	topnCollector *collector.TopNCollector,
+) uint64 {
 	backingSize := req.Size + req.From + 1
 	if req.Size+req.From > collector.PreAllocSizeSkipCap {
 		backingSize = collector.PreAllocSizeSkipCap + 1
@@ -509,11 +515,12 @@ func (i *indexImpl) preSearch(ctx context.Context, req *SearchRequest, reader in
 				return nil, err
 			}
 			for field := range fs {
-				dict, err := reader.FieldDict(field)
-				if err != nil {
-					return nil, err
+				if bm25Reader, ok := reader.(index.BM25Reader); ok {
+					fieldCardinality[field], err = bm25Reader.FieldCardinality(field)
+					if err != nil {
+						return nil, err
+					}
 				}
-				fieldCardinality[field] = dict.Cardinality()
 			}
 		}
 	}
@@ -560,6 +567,16 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 		if err != nil {
 			return nil, err
 		}
+		// increment the search count here itself,
+		// since the presearch may already satisfy
+		// the search request
+		atomic.AddUint64(&i.stats.searches, 1)
+		// increment the search time stat here as well,
+		// since presearch is part of the overall search
+		// operation and should be included in the search
+		// time stat
+		searchDuration := time.Since(searchStart)
+		atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))
 		return preSearchResult, nil
 	}
 
@@ -584,7 +601,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	var fts search.FieldTermSynonymMap
 	var skipSynonymCollector bool
 
-	var bm25Data *search.BM25Stats
+	var bm25Stats *search.BM25Stats
 	var ok bool
 	if req.PreSearchData != nil {
 		for k, v := range req.PreSearchData {
@@ -607,9 +624,9 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 				}
 			case search.BM25PreSearchDataKey:
 				if v != nil {
-					bm25Data, ok = v.(*search.BM25Stats)
+					bm25Stats, ok = v.(*search.BM25Stats)
 					if !ok {
-						return nil, fmt.Errorf("bm25 preSearchData must be of type map[string]interface{}")
+						return nil, fmt.Errorf("bm25 preSearchData must be of type *search.BM25Stats")
 					}
 				}
 			}
@@ -651,10 +668,10 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	ctx = context.WithValue(ctx, search.GetScoringModelCallbackKey,
 		search.GetScoringModelCallbackFn(scoringModelCallback))
 
-	// set the bm25 presearch data (stats important for consistent scoring) in
+	// set the bm25Stats (stats important for consistent scoring) in
 	// the context object
-	if bm25Data != nil {
-		ctx = context.WithValue(ctx, search.BM25PreSearchDataKey, bm25Data)
+	if bm25Stats != nil {
+		ctx = context.WithValue(ctx, search.BM25StatsKey, bm25Stats)
 	}
 
 	// This callback and variable handles the tracking of bytes read
@@ -667,8 +684,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 		totalSearchCost += bytesRead
 	}
 
-	ctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey,
-		search.SearchIOStatsCallbackFunc(sendBytesRead))
+	ctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey, search.SearchIOStatsCallbackFunc(sendBytesRead))
 
 	var bufPool *s2.GeoBufferPool
 	getBufferPool := func() *s2.GeoBufferPool {
@@ -679,8 +695,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 		return bufPool
 	}
 
-	ctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey,
-		search.GeoBufferPoolCallbackFunc(getBufferPool))
+	ctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey, search.GeoBufferPoolCallbackFunc(getBufferPool))
 
 	searcher, err := req.Query.Searcher(ctx, indexReader, i.m, search.SearcherOptions{
 		Explain:            req.Explain,
@@ -806,7 +821,13 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 	totalSearchCost += storedFieldsCost
 	search.RecordSearchCost(ctx, search.AddM, storedFieldsCost)
 
-	atomic.AddUint64(&i.stats.searches, 1)
+	if req.PreSearchData == nil {
+		// increment the search count only if this is not a second-phase search
+		// (e.g., for Hybrid Search), since the first-phase search already increments it
+		atomic.AddUint64(&i.stats.searches, 1)
+	}
+	// increment the search time stat, as the first-phase search is part of
+	// the overall operation; adding second-phase time later keeps it accurate
 	searchDuration := time.Since(searchStart)
 	atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))
 
@@ -847,7 +868,8 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
 
 func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
 	indexName string, r index.IndexReader,
-	highlighter highlight.Highlighter) (error, uint64) {
+	highlighter highlight.Highlighter,
+) (error, uint64) {
 	var totalStoredFieldsBytes uint64
 	if len(req.Fields) > 0 || highlighter != nil {
 		doc, err := r.Document(hit.ID)
@@ -1238,7 +1260,8 @@ func (i *indexImpl) CopyTo(d index.Directory) (err error) {
 }
 
 func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser,
-	error) {
+	error,
+) {
 	dir, file := filepath.Split(filePath)
 	if dir != "" {
 		err := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm)
@@ -1248,7 +1271,7 @@ func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser,
 	}
 
 	return os.OpenFile(filepath.Join(string(f), dir, file),
-		os.O_RDWR|os.O_CREATE, 0600)
+		os.O_RDWR|os.O_CREATE, 0o600)
 }
 
 func (i *indexImpl) FireIndexEvent() {

vendor/github.com/blevesearch/bleve/v2/mapping/document.go

@@ -52,7 +52,8 @@ type DocumentMapping struct {
 }
 
 func (dm *DocumentMapping) Validate(cache *registry.Cache,
-	parentName string, fieldAliasCtx map[string]*FieldMapping) error {
+	parentName string, fieldAliasCtx map[string]*FieldMapping,
+) error {
 	var err error
 	if dm.DefaultAnalyzer != "" {
 		_, err := cache.AnalyzerNamed(dm.DefaultAnalyzer)
@@ -183,7 +184,8 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping {
 // document or for an explicitly mapped field; the closest most specific
 // document mapping could be one that matches part of the provided path.
 func (dm *DocumentMapping) documentMappingForPathElements(pathElements []string) (
-	*DocumentMapping, *DocumentMapping) {
+	*DocumentMapping, *DocumentMapping,
+) {
 	var pathElementsCopy []string
 	if len(pathElements) == 0 {
 		pathElementsCopy = []string{""}
@@ -217,7 +219,8 @@ OUTER:
 // document or for an explicitly mapped field; the closest most specific
 // document mapping could be one that matches part of the provided path.
 func (dm *DocumentMapping) documentMappingForPath(path string) (
-	*DocumentMapping, *DocumentMapping) {
+	*DocumentMapping, *DocumentMapping,
+) {
 	pathElements := decodePath(path)
 	return dm.documentMappingForPathElements(pathElements)
 }
@@ -457,7 +460,6 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes
 	case reflect.Bool:
 		dm.processProperty(val.Bool(), path, indexes, context)
 	}
-
 }
 
 func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) {
@@ -483,13 +485,14 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 		if subDocMapping != nil {
 			// index by explicit mapping
 			for _, fieldMapping := range subDocMapping.Fields {
-				if fieldMapping.Type == "geoshape" {
+				switch fieldMapping.Type {
+				case "geoshape":
 					fieldMapping.processGeoShape(property, pathString, path, indexes, context)
-				} else if fieldMapping.Type == "geopoint" {
+				case "geopoint":
 					fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
-				} else if fieldMapping.Type == "vector_base64" {
+				case "vector_base64":
 					fieldMapping.processVectorBase64(property, pathString, path, indexes, context)
-				} else {
+				default:
 					fieldMapping.processString(propertyValueString, pathString, path, indexes, context)
 				}
 			}
@@ -568,9 +571,10 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 		default:
 			if subDocMapping != nil {
 				for _, fieldMapping := range subDocMapping.Fields {
-					if fieldMapping.Type == "geopoint" {
+					switch fieldMapping.Type {
+					case "geopoint":
 						fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
-					} else if fieldMapping.Type == "geoshape" {
+					case "geoshape":
 						fieldMapping.processGeoShape(property, pathString, path, indexes, context)
 					}
 				}

vendor/github.com/blevesearch/bleve/v2/mapping/field.go

@@ -26,6 +26,7 @@ import (
 	"github.com/blevesearch/bleve/v2/geo"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
+	"github.com/blevesearch/geo/geojson"
 )
 
 // control the default behavior for dynamic fields (those not explicitly mapped)
@@ -231,7 +232,9 @@ func (fm *FieldMapping) Options() index.FieldIndexingOptions {
 func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) {
 	fieldName := getFieldName(pathString, path, fm)
 	options := fm.Options()
-	if fm.Type == "text" {
+
+	switch fm.Type {
+	case "text":
 		analyzer := fm.analyzerForField(path, context)
 		field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer)
 		context.doc.AddField(field)
@@ -239,7 +242,7 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str
 		if !fm.IncludeInAll {
 			context.excludedFromAll = append(context.excludedFromAll, fieldName)
 		}
-	} else if fm.Type == "datetime" {
+	case "datetime":
 		dateTimeFormat := context.im.DefaultDateTimeParser
 		if fm.DateFormat != "" {
 			dateTimeFormat = fm.DateFormat
@@ -251,7 +254,7 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str
 				fm.processTime(parsedDateTime, layout, pathString, path, indexes, context)
 			}
 		}
-	} else if fm.Type == "IP" {
+	case "IP":
 		ip := net.ParseIP(propertyValueString)
 		if ip != nil {
 			fm.processIP(ip, pathString, path, indexes, context)
@@ -328,32 +331,20 @@ func (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, i
 }
 
 func (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{},
-	pathString string, path []string, indexes []uint64, context *walkContext) {
+	pathString string, path []string, indexes []uint64, context *walkContext,
+) {
 	coordValue, shape, err := geo.ParseGeoShapeField(propertyMightBeGeoShape)
 	if err != nil {
 		return
 	}
 
-	if shape == geo.CircleType {
-		center, radius, found := geo.ExtractCircle(propertyMightBeGeoShape)
+	if shape == geo.GeometryCollectionType {
+		geoShapes, found := geo.ExtractGeometryCollection(propertyMightBeGeoShape)
 		if found {
 			fieldName := getFieldName(pathString, path, fm)
 			options := fm.Options()
-			field := document.NewGeoCircleFieldWithIndexingOptions(fieldName,
-				indexes, center, radius, options)
-			context.doc.AddField(field)
-
-			if !fm.IncludeInAll {
-				context.excludedFromAll = append(context.excludedFromAll, fieldName)
-			}
-		}
-	} else if shape == geo.GeometryCollectionType {
-		coordinates, shapes, found := geo.ExtractGeometryCollection(propertyMightBeGeoShape)
-		if found {
-			fieldName := getFieldName(pathString, path, fm)
-			options := fm.Options()
-			field := document.NewGeometryCollectionFieldWithIndexingOptions(fieldName,
-				indexes, coordinates, shapes, options)
+			field := document.NewGeometryCollectionFieldFromShapesWithIndexingOptions(fieldName,
+				indexes, geoShapes, options)
 			context.doc.AddField(field)
 
 			if !fm.IncludeInAll {
@@ -361,12 +352,20 @@ func (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{},
 			}
 		}
 	} else {
-		coordinates, shape, found := geo.ExtractGeoShapeCoordinates(coordValue, shape)
+		var geoShape *geojson.GeoShape
+		var found bool
+
+		if shape == geo.CircleType {
+			geoShape, found = geo.ExtractCircle(propertyMightBeGeoShape)
+		} else {
+			geoShape, found = geo.ExtractGeoShapeCoordinates(coordValue, shape)
+		}
+
 		if found {
 			fieldName := getFieldName(pathString, path, fm)
 			options := fm.Options()
-			field := document.NewGeoShapeFieldWithIndexingOptions(fieldName,
-				indexes, coordinates, shape, options)
+			field := document.NewGeoShapeFieldFromShapeWithIndexingOptions(fieldName,
+				indexes, geoShape, options)
 			context.doc.AddField(field)
 
 			if !fm.IncludeInAll {
@@ -401,7 +400,6 @@ func getFieldName(pathString string, path []string, fieldMapping *FieldMapping)
 
 // UnmarshalJSON offers custom unmarshaling with optional strict validation
 func (fm *FieldMapping) UnmarshalJSON(data []byte) error {
-
 	var tmp map[string]json.RawMessage
 	err := util.UnmarshalJSON(data, &tmp)
 	if err != nil {

vendor/github.com/blevesearch/bleve/v2/search/query/knn.go

@@ -35,8 +35,7 @@ type KNNQuery struct {
 	BoostVal    *Boost    `json:"boost,omitempty"`
 
 	// see KNNRequest.Params for description
-	Params      json.RawMessage `json:"params"`
-	FilterQuery Query           `json:"filter,omitempty"`
+	Params json.RawMessage `json:"params"`
 	// elegibleSelector is used to filter out documents that are
 	// eligible for the KNN search from a pre-filter query.
 	elegibleSelector index.EligibleDocumentSelector

vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go

@@ -86,17 +86,18 @@ func bm25ScoreMetrics(ctx context.Context, field string,
 	var fieldCardinality int
 	var err error
 
-	bm25Stats, ok := ctx.Value(search.BM25PreSearchDataKey).(*search.BM25Stats)
+	bm25Stats, ok := ctx.Value(search.BM25StatsKey).(*search.BM25Stats)
 	if !ok {
 		count, err = indexReader.DocCount()
 		if err != nil {
 			return 0, 0, err
 		}
-		dict, err := indexReader.FieldDict(field)
-		if err != nil {
-			return 0, 0, err
+		if bm25Reader, ok := indexReader.(index.BM25Reader); ok {
+			fieldCardinality, err = bm25Reader.FieldCardinality(field)
+			if err != nil {
+				return 0, 0, err
+			}
 		}
-		fieldCardinality = dict.Cardinality()
 	} else {
 		count = uint64(bm25Stats.DocCount)
 		fieldCardinality, ok = bm25Stats.FieldCardinality[field]
@@ -121,9 +122,9 @@ func newTermSearcherFromReader(ctx context.Context, indexReader index.IndexReade
 
 	// as a fallback case we track certain stats for tf-idf scoring
 	if ctx != nil {
-		if similaritModelCallback, ok := ctx.Value(search.
+		if similarityModelCallback, ok := ctx.Value(search.
 			GetScoringModelCallbackKey).(search.GetScoringModelCallbackFn); ok {
-			similarityModel = similaritModelCallback()
+			similarityModel = similarityModelCallback()
 		}
 	}
 	switch similarityModel {

vendor/github.com/blevesearch/bleve/v2/search/sort.go

@@ -154,15 +154,18 @@ func ParseSearchSortString(input string) SearchSort {
 	} else if strings.HasPrefix(input, "+") {
 		input = input[1:]
 	}
-	if input == "_id" {
+
+	switch input {
+	case "_id":
 		return &SortDocID{
 			Desc: descending,
 		}
-	} else if input == "_score" {
+	case "_score":
 		return &SortScore{
 			Desc: descending,
 		}
 	}
+
 	return &SortField{
 		Field: input,
 		Desc:  descending,
@@ -426,7 +429,9 @@ func (s *SortField) filterTermsByMode(terms [][]byte) string {
 // prefix coded numbers with shift of 0
 func (s *SortField) filterTermsByType(terms [][]byte) [][]byte {
 	stype := s.Type
-	if stype == SortFieldAuto {
+
+	switch stype {
+	case SortFieldAuto:
 		allTermsPrefixCoded := true
 		termsWithShiftZero := s.tmp[:0]
 		for _, term := range terms {
@@ -442,7 +447,7 @@ func (s *SortField) filterTermsByType(terms [][]byte) [][]byte {
 			terms = termsWithShiftZero
 			s.tmp = termsWithShiftZero[:0]
 		}
-	} else if stype == SortFieldAsNumber || stype == SortFieldAsDate {
+	case SortFieldAsNumber, SortFieldAsDate:
 		termsWithShiftZero := s.tmp[:0]
 		for _, term := range terms {
 			valid, shift := numeric.ValidPrefixCodedTermBytes(term)
@@ -453,6 +458,7 @@ func (s *SortField) filterTermsByType(terms [][]byte) [][]byte {
 		terms = termsWithShiftZero
 		s.tmp = termsWithShiftZero[:0]
 	}
+
 	return terms
 }
 

vendor/github.com/blevesearch/bleve/v2/search/util.go

@@ -74,8 +74,6 @@ func MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch)
 	return dest
 }
 
-const SearchIOStatsCallbackKey = "_search_io_stats_callback_key"
-
 type SearchIOStatsCallbackFunc func(uint64)
 
 // Implementation of SearchIncrementalCostCallbackFn should handle the following messages
@@ -87,8 +85,11 @@ type SearchIOStatsCallbackFunc func(uint64)
 //     handled safely by the implementation.
 type SearchIncrementalCostCallbackFn func(SearchIncrementalCostCallbackMsg,
 	SearchQueryType, uint64)
-type SearchIncrementalCostCallbackMsg uint
-type SearchQueryType uint
+
+type (
+	SearchIncrementalCostCallbackMsg uint
+	SearchQueryType                  uint
+)
 
 const (
 	Term = SearchQueryType(1 << iota)
@@ -103,13 +104,59 @@ const (
 	DoneM
 )
 
-const SearchIncrementalCostKey = "_search_incremental_cost_key"
-const QueryTypeKey = "_query_type_key"
-const FuzzyMatchPhraseKey = "_fuzzy_match_phrase_key"
-const IncludeScoreBreakdownKey = "_include_score_breakdown_key"
+// ContextKey is used to identify the context key in the context.Context
+type ContextKey string
+
+func (c ContextKey) String() string {
+	return string(c)
+}
+
+const (
+	SearchIncrementalCostKey ContextKey = "_search_incremental_cost_key"
+	QueryTypeKey             ContextKey = "_query_type_key"
+	FuzzyMatchPhraseKey      ContextKey = "_fuzzy_match_phrase_key"
+	IncludeScoreBreakdownKey ContextKey = "_include_score_breakdown_key"
+
+	// PreSearchKey indicates whether to perform a preliminary search to gather necessary
+	// information which would be used in the actual search down the line.
+	PreSearchKey ContextKey = "_presearch_key"
+
+	// GetScoringModelCallbackKey is used to help the underlying searcher identify
+	// which scoring mechanism to use based on index mapping.
+	GetScoringModelCallbackKey ContextKey = "_get_scoring_model"
+
+	// SearchIOStatsCallbackKey is used to help the underlying searcher identify
+	SearchIOStatsCallbackKey ContextKey = "_search_io_stats_callback_key"
+
+	// GeoBufferPoolCallbackKey ContextKey is used to help the underlying searcher
+	GeoBufferPoolCallbackKey ContextKey = "_geo_buffer_pool_callback_key"
+
+	// SearchTypeKey is used to identify type of the search being performed.
+	//
+	// for consistent scoring in cases an index is partitioned/sharded (using an
+	// index alias), GlobalScoring helps in aggregating the necessary stats across
+	// all the child bleve indexes (shards/partitions) first before the actual search
+	// is performed, such that the scoring involved using these stats would be at a
+	// global level.
+	SearchTypeKey ContextKey = "_search_type_key"
+
+	// The following keys are used to invoke the callbacks at the start and end stages
+	// of optimizing the disjunction/conjunction searcher creation.
+	SearcherStartCallbackKey ContextKey = "_searcher_start_callback_key"
+	SearcherEndCallbackKey   ContextKey = "_searcher_end_callback_key"
+
+	// FieldTermSynonymMapKey is used to store and transport the synonym definitions data
+	// to the actual search phase which would use the synonyms to perform the search.
+	FieldTermSynonymMapKey ContextKey = "_field_term_synonym_map_key"
+
+	// BM25StatsKey is used to store and transport the BM25 Data
+	// to the actual search phase which would use it to perform the search.
+	BM25StatsKey ContextKey = "_bm25_stats_key"
+)
 
 func RecordSearchCost(ctx context.Context,
-	msg SearchIncrementalCostCallbackMsg, bytes uint64) {
+	msg SearchIncrementalCostCallbackMsg, bytes uint64,
+) {
 	if ctx != nil {
 		queryType, ok := ctx.Value(QueryTypeKey).(SearchQueryType)
 		if !ok {
@@ -125,52 +172,30 @@ func RecordSearchCost(ctx context.Context,
 	}
 }
 
-const GeoBufferPoolCallbackKey = "_geo_buffer_pool_callback_key"
-
 // Assigning the size of the largest buffer in the pool to 24KB and
 // the smallest buffer to 24 bytes. The pools are used to read a
 // sequence of vertices which are always 24 bytes each.
-const MaxGeoBufPoolSize = 24 * 1024
-const MinGeoBufPoolSize = 24
+const (
+	MaxGeoBufPoolSize = 24 * 1024
+	MinGeoBufPoolSize = 24
+)
 
 type GeoBufferPoolCallbackFunc func() *s2.GeoBufferPool
 
-// PreSearchKey indicates whether to perform a preliminary search to gather necessary
-// information which would be used in the actual search down the line.
-const PreSearchKey = "_presearch_key"
-
 // *PreSearchDataKey are used to store the data gathered during the presearch phase
 // which would be use in the actual search phase.
-const KnnPreSearchDataKey = "_knn_pre_search_data_key"
-const SynonymPreSearchDataKey = "_synonym_pre_search_data_key"
-const BM25PreSearchDataKey = "_bm25_pre_search_data_key"
-
-// SearchTypeKey is used to identify type of the search being performed.
-//
-// for consistent scoring in cases an index is partitioned/sharded (using an
-// index alias), GlobalScoring helps in aggregating the necessary stats across
-// all the child bleve indexes (shards/partitions) first before the actual search
-// is performed, such that the scoring involved using these stats would be at a
-// global level.
-const SearchTypeKey = "_search_type_key"
-
-// The following keys are used to invoke the callbacks at the start and end stages
-// of optimizing the disjunction/conjunction searcher creation.
-const SearcherStartCallbackKey = "_searcher_start_callback_key"
-const SearcherEndCallbackKey = "_searcher_end_callback_key"
-
-// FieldTermSynonymMapKey is used to store and transport the synonym definitions data
-// to the actual search phase which would use the synonyms to perform the search.
-const FieldTermSynonymMapKey = "_field_term_synonym_map_key"
+const (
+	KnnPreSearchDataKey     = "_knn_pre_search_data_key"
+	SynonymPreSearchDataKey = "_synonym_pre_search_data_key"
+	BM25PreSearchDataKey    = "_bm25_pre_search_data_key"
+)
 
 const GlobalScoring = "_global_scoring"
 
-// GetScoringModelCallbackKey is used to help the underlying searcher identify
-// which scoring mechanism to use based on index mapping.
-const GetScoringModelCallbackKey = "_get_scoring_model"
-
-type SearcherStartCallbackFn func(size uint64) error
-type SearcherEndCallbackFn func(size uint64) error
+type (
+	SearcherStartCallbackFn func(size uint64) error
+	SearcherEndCallbackFn   func(size uint64) error
+)
 
 type GetScoringModelCallbackFn func() string
 
@@ -199,8 +224,10 @@ func (f FieldTermSynonymMap) MergeWith(fts FieldTermSynonymMap) {
 // the default values are as per elastic search's implementation
 //   - https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-similarity.html#bm25
 //   - https://www.elastic.co/blog/practical-bm25-part-3-considerations-for-picking-b-and-k1-in-elasticsearch
-var BM25_k1 float64 = 1.2
-var BM25_b float64 = 0.75
+var (
+	BM25_k1 float64 = 1.2
+	BM25_b  float64 = 0.75
+)
 
 type BM25Stats struct {
 	DocCount         float64        `json:"doc_count"`

vendor/github.com/blevesearch/bleve/v2/search_knn.go

@@ -203,6 +203,9 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error {
 			r.KNN[i].FilterQuery = nil
 		} else {
 			r.KNN[i].FilterQuery, err = query.ParseQuery(knnReq.FilterQuery)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	r.KNNOperator = temp.KNNOperator
@@ -306,6 +309,17 @@ func validateKNN(req *SearchRequest) error {
 		if q.K > BleveMaxK {
 			return fmt.Errorf("k must be less than %d", BleveMaxK)
 		}
+		// since the DefaultField is not applicable for knn,
+		// the field must be specified.
+		if q.Field == "" {
+			return fmt.Errorf("knn query field must be non-empty")
+		}
+		if vfq, ok := q.FilterQuery.(query.ValidatableQuery); ok {
+			err := vfq.Validate()
+			if err != nil {
+				return fmt.Errorf("knn filter query is invalid: %v", err)
+			}
+		}
 	}
 	switch req.KNNOperator {
 	case knnOperatorAnd, knnOperatorOr, "":