chore(deps): bump github.com/blevesearch/bleve/v2 from 2.4.2 to 2.4.3

Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.4.2 to 2.4.3. - [Release notes](https://github.com/blevesearch/bleve/releases) - [Commits](https://github.com/blevesearch/bleve/compare/v2.4.2...v2.4.3) --- updated-dependencies: - dependency-name: github.com/blevesearch/bleve/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>
2024-11-14 08:38:51 +01:00
parent 926bc8e22a
commit 1104846219
51 changed files with 1321 additions and 269 deletions
@@ -9,28 +9,29 @@
 [![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)

-A modern indexing library in GO
+A modern indexing + search library in GO

 ## Features

-* Index any go data structure (including JSON)
-* Intelligent defaults backed up by powerful configuration
+* Index any GO data structure or JSON
+* Intelligent defaults backed up by powerful configuration ([scorch](https://github.com/blevesearch/bleve/blob/master/index/scorch/README.md))
 * Supported field types:
    * `text`, `number`, `datetime`, `boolean`, `geopoint`, `geoshape`, `IP`, `vector`
 * Supported query types:
-    * Term, Phrase, Match, Match Phrase, Prefix, Fuzzy
-    * Conjunction, Disjunction, Boolean (`must`/`should`/`must_not`)
-    * Term Range, Numeric Range, Date Range
-    * [Geo Spatial](https://github.com/blevesearch/bleve/blob/master/geo/README.md)
-    * Simple [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/)
-    * Approximate k-nearest neighbors over [vectors](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md)
-* [tf-idf](https://en.wikipedia.org/wiki/Tf-idf) Scoring
+    * `term`, `phrase`, `match`, `match_phrase`, `prefix`, `regexp`, `wildcard`, `fuzzy`
+    * term range, numeric range, date range, boolean field
+    * compound queries: `conjuncts`, `disjuncts`, boolean (`must`/`should`/`must_not`)
+    * [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/)
+    * [geo spatial search](https://github.com/blevesearch/bleve/blob/master/geo/README.md)
+    * approximate k-nearest neighbors via [vector search](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md)
+* [tf-idf](https://en.wikipedia.org/wiki/Tf-idf) scoring
+* Hybrid search: exact + semantic
 * Query time boosting
 * Search result match highlighting with document fragments
 * Aggregations/faceting support:
-    * Terms Facet
-    * Numeric Range Facet
-    * Date Range Facet
+    * terms facet
+    * numeric range facet
+    * date range facet

 ## Indexing

@@ -46,6 +46,9 @@ func (b *Batch) Index(id string, data interface{}) error {
 	if id == "" {
 		return ErrorEmptyID
 	}
+	if eventIndex, ok := b.index.(index.EventIndex); ok {
+		eventIndex.FireIndexEvent()
+	}
 	doc := document.NewDocument(id)
 	err := b.index.Mapping().MapDocument(doc, data)
 	if err != nil {
@@ -67,3 +67,7 @@ var EventKindMergeTaskIntroduction = EventKind(8)
 // EventKindPreMergeCheck is fired before the merge begins to check if
 // the caller should proceed with the merge.
 var EventKindPreMergeCheck = EventKind(9)
+
+// EventKindIndexStart is fired when Index() is invoked which
+// creates a new Document object from an interface using the index mapping.
+var EventKindIndexStart = EventKind(10)
@@ -23,6 +23,7 @@ import (
 	"sync"
 	"sync/atomic"

+	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/v2/search"
 	index "github.com/blevesearch/bleve_index_api"
 	segment_api "github.com/blevesearch/scorch_segment_api/v2"
@@ -34,6 +35,8 @@ type OptimizeVR struct {
 	totalCost uint64
 	// maps field to vector readers
 	vrs map[string][]*IndexSnapshotVectorReader
+	// if at least one of the vector readers requires filtered kNN.
+	requiresFiltering bool
 }

 // This setting _MUST_ only be changed during init and not after.
@@ -62,6 +65,11 @@ func (o *OptimizeVR) Finish() error {
 	var errorsM sync.Mutex
 	var errors []error

+	var snapshotGlobalDocNums map[int]*roaring.Bitmap
+	if o.requiresFiltering {
+		snapshotGlobalDocNums = o.snapshot.globalDocNums()
+	}
+
 	defer o.invokeSearcherEndCallback()

 	wg := sync.WaitGroup{}
@@ -77,7 +85,8 @@ func (o *OptimizeVR) Finish() error {
 					wg.Done()
 				}()
 				for field, vrs := range o.vrs {
-					vecIndex, err := segment.InterpretVectorIndex(field, origSeg.deleted)
+					vecIndex, err := segment.InterpretVectorIndex(field,
+						o.requiresFiltering, origSeg.deleted)
 					if err != nil {
 						errorsM.Lock()
 						errors = append(errors, err)
@@ -89,9 +98,37 @@ func (o *OptimizeVR) Finish() error {
 					vectorIndexSize := vecIndex.Size()
 					origSeg.cachedMeta.updateMeta(field, vectorIndexSize)
 					for _, vr := range vrs {
+						var pl segment_api.VecPostingsList
+						var err error
+
 						// for each VR, populate postings list and iterators
 						// by passing the obtained vector index and getting similar vectors.
-						pl, err := vecIndex.Search(vr.vector, vr.k, vr.searchParams)
+
+						// Only applies to filtered kNN.
+						if vr.eligibleDocIDs != nil && len(vr.eligibleDocIDs) > 0 {
+							eligibleVectorInternalIDs := vr.getEligibleDocIDs()
+							if snapshotGlobalDocNums != nil {
+								// Only the eligible documents belonging to this segment
+								// will get filtered out.
+								// There is no way to determine which doc belongs to which segment
+								eligibleVectorInternalIDs.And(snapshotGlobalDocNums[index])
+							}
+
+							eligibleLocalDocNums := make([]uint64,
+								eligibleVectorInternalIDs.GetCardinality())
+							// get the (segment-)local document numbers
+							for i, docNum := range eligibleVectorInternalIDs.ToArray() {
+								localDocNum := o.snapshot.localDocNumFromGlobal(index,
+									uint64(docNum))
+								eligibleLocalDocNums[i] = localDocNum
+							}
+
+							pl, err = vecIndex.SearchWithFilter(vr.vector, vr.k,
+								eligibleLocalDocNums, vr.searchParams)
+						} else {
+							pl, err = vecIndex.Search(vr.vector, vr.k, vr.searchParams)
+						}
+
 						if err != nil {
 							errorsM.Lock()
 							errors = append(errors, err)
@@ -140,6 +177,9 @@ func (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context,
 		return octx, nil
 	}
 	o.ctx = ctx
+	if !o.requiresFiltering {
+		o.requiresFiltering = len(s.eligibleDocIDs) > 0
+	}

 	if o.snapshot != s.snapshot {
 		o.invokeSearcherEndCallback()
@@ -49,7 +49,7 @@ type Scorch struct {

 	unsafeBatch bool

-	rootLock             sync.RWMutex
+	rootLock sync.RWMutex

 	root                 *IndexSnapshot // holds 1 ref-count on the root
 	rootPersisted        []chan error   // closed when root is persisted
@@ -376,6 +376,8 @@ func (s *Scorch) Delete(id string) error {
 func (s *Scorch) Batch(batch *index.Batch) (err error) {
 	start := time.Now()

+	// notify handlers that we're about to index a batch of data
+	s.fireEvent(EventKindBatchIntroductionStart, 0)
 	defer func() {
 		s.fireEvent(EventKindBatchIntroduction, time.Since(start))
 	}()
@@ -434,9 +436,6 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) {

 	indexStart := time.Now()

-	// notify handlers that we're about to introduce a segment
-	s.fireEvent(EventKindBatchIntroductionStart, 0)
-
 	var newSegment segment.Segment
 	var bufBytes uint64
 	stats := newFieldStats()
@@ -878,3 +877,8 @@ func (s *Scorch) CopyReader() index.CopyReader {
 	s.rootLock.Unlock()
 	return rv
 }
+
+// external API to fire a scorch event (EventKindIndexStart) externally from bleve
+func (s *Scorch) FireIndexEvent() {
+	s.fireEvent(EventKindIndexStart, 0)
+}
@@ -471,16 +471,44 @@ func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 	return rvd, nil
 }

+// In a multi-segment index, each document has:
+// 1. a local docnum - local to the segment
+// 2. a global docnum - unique identifier across the index
+// This function returns the segment index(the segment in which the docnum is present)
+// and local docnum of a document.
 func (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) {
 	segmentIndex := sort.Search(len(is.offsets),
 		func(x int) bool {
 			return is.offsets[x] > docNum
 		}) - 1

-	localDocNum := docNum - is.offsets[segmentIndex]
+	localDocNum := is.localDocNumFromGlobal(segmentIndex, docNum)
 	return int(segmentIndex), localDocNum
 }

+// This function returns the local docnum, given the segment index and global docnum
+func (is *IndexSnapshot) localDocNumFromGlobal(segmentIndex int, docNum uint64) uint64 {
+	return docNum - is.offsets[segmentIndex]
+}
+
+// Function to return a mapping of the segment index to the live global	doc nums
+// in the segment of the specified index snapshot.
+func (is *IndexSnapshot) globalDocNums() map[int]*roaring.Bitmap {
+	if len(is.segment) == 0 {
+		return nil
+	}
+
+	segmentIndexGlobalDocNums := make(map[int]*roaring.Bitmap)
+
+	for i := range is.segment {
+		segmentIndexGlobalDocNums[i] = roaring.NewBitmap()
+		for _, localDocNum := range is.segment[i].DocNumbersLive().ToArray() {
+			segmentIndexGlobalDocNums[i].Add(localDocNum + uint32(is.offsets[i]))
+		}
+	}
+	return segmentIndexGlobalDocNums
+}
+
 func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) {
 	docNum, err := docInternalToNumber(id)
 	if err != nil {
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"reflect"

+	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/bleve/v2/size"
 	index "github.com/blevesearch/bleve_index_api"
 	segment_api "github.com/blevesearch/scorch_segment_api/v2"
@@ -51,6 +52,31 @@ type IndexSnapshotVectorReader struct {
 	ctx           context.Context

 	searchParams json.RawMessage
+
+	// The following fields are only applicable for vector readers which will
+	// process pre-filtered kNN queries.
+	eligibleDocIDs []index.IndexInternalID
+}
+
+// Function to convert the internal IDs of the eligible documents to a type suitable
+// for addition to a bitmap.
+// Useful to have the eligible doc IDs in a bitmap to leverage the fast intersection
+// (AND) operations. Eg. finding the eligible doc IDs present in a segment.
+func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap {
+	res := roaring.NewBitmap()
+	if len(i.eligibleDocIDs) > 0 {
+		internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs))
+		// converts the doc IDs to uint32 and returns
+		for _, eligibleDocInternalID := range i.eligibleDocIDs {
+			internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID))
+			if err != nil {
+				continue
+			}
+			internalDocIDs = append(internalDocIDs, uint32(internalDocID))
+		}
+		res.AddMany(internalDocIDs)
+	}
+	return res
 }

 func (i *IndexSnapshotVectorReader) Size() int {
@@ -108,7 +134,17 @@ func (i *IndexSnapshotVectorReader) Advance(ID index.IndexInternalID,
 	preAlloced *index.VectorDoc) (*index.VectorDoc, error) {

 	if i.currPosting != nil && bytes.Compare(i.currID, ID) >= 0 {
-		i2, err := i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k, i.searchParams)
+		var i2 index.VectorReader
+		var err error
+
+		if len(i.eligibleDocIDs) > 0 {
+			i2, err = i.snapshot.VectorReaderWithFilter(i.ctx, i.vector, i.field,
+				i.k, i.searchParams, i.eligibleDocIDs)
+		} else {
+			i2, err = i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k,
+				i.searchParams)
+		}
+
 		if err != nil {
 			return nil, err
 		}
@@ -48,3 +48,29 @@ func (is *IndexSnapshot) VectorReader(ctx context.Context, vector []float32,

 	return rv, nil
 }
+
+func (is *IndexSnapshot) VectorReaderWithFilter(ctx context.Context, vector []float32,
+	field string, k int64, searchParams json.RawMessage,
+	filterIDs []index.IndexInternalID) (
+	index.VectorReader, error) {
+
+	rv := &IndexSnapshotVectorReader{
+		vector:         vector,
+		field:          field,
+		k:              k,
+		snapshot:       is,
+		searchParams:   searchParams,
+		eligibleDocIDs: filterIDs,
+	}
+
+	if rv.postings == nil {
+		rv.postings = make([]segment_api.VecPostingsList, len(is.segment))
+	}
+	if rv.iterators == nil {
+		rv.iterators = make([]segment_api.VecPostingsIterator, len(is.segment))
+	}
+
+	// initialize postings and iterators within the OptimizeVR's Finish()
+
+	return rv, nil
+}
@@ -256,6 +256,8 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) {
 		return ErrorIndexClosed
 	}

+	i.FireIndexEvent()
+
 	doc := document.NewDocument(id)
 	err = i.m.MapDocument(doc, data)
 	if err != nil {
@@ -1112,3 +1114,16 @@ func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser,
 	return os.OpenFile(filepath.Join(string(f), dir, file),
 		os.O_RDWR|os.O_CREATE, 0600)
 }
+
+func (i *indexImpl) FireIndexEvent() {
+	// get the internal index implementation
+	internalIndex, err := i.Advanced()
+	if err != nil {
+		return
+	}
+	// check if the internal index implementation supports events
+	if internalEventIndex, ok := internalIndex.(index.EventIndex); ok {
+		// fire the Index() event
+		internalEventIndex.FireIndexEvent()
+	}
+}
@@ -24,6 +24,7 @@ import (
 	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
+	faiss "github.com/blevesearch/go-faiss"
 )

 // Min and Max allowed dimensions for a vector field;
@@ -140,6 +141,10 @@ func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
 	if !ok {
 		return false
 	}
+	// normalize raw vector if similarity is cosine
+	if fm.Similarity == index.CosineSimilarity {
+		vector = NormalizeVector(vector)
+	}

 	fieldName := getFieldName(pathString, path, fm)
 	options := fm.Options()
@@ -163,6 +168,10 @@ func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interfac
 	if err != nil || len(decodedVector) != fm.Dims {
 		return
 	}
+	// normalize raw vector if similarity is cosine
+	if fm.Similarity == index.CosineSimilarity {
+		decodedVector = NormalizeVector(decodedVector)
+	}

 	fieldName := getFieldName(pathString, path, fm)
 	options := fm.Options()
@@ -252,3 +261,12 @@ func validateVectorFieldAlias(field *FieldMapping, parentName string,

 	return nil
 }
+
+func NormalizeVector(vec []float32) []float32 {
+	// make a copy of the vector to avoid modifying the original
+	// vector in-place
+	vecCopy := make([]float32, len(vec))
+	copy(vecCopy, vec)
+	// normalize the vector copy using in-place normalization provided by faiss
+	return faiss.NormalizeVector(vecCopy)
+}
@@ -0,0 +1,157 @@
+//  Copyright (c) 2024 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/blevesearch/bleve/v2/search"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type EligibleCollector struct {
+	size    int
+	total   uint64
+	took    time.Duration
+	results search.DocumentMatchCollection
+
+	ids []index.IndexInternalID
+}
+
+func NewEligibleCollector(size int) *EligibleCollector {
+	return newEligibleCollector(size)
+}
+
+func newEligibleCollector(size int) *EligibleCollector {
+	// No sort order & skip always 0 since this is only to filter eligible docs.
+	ec := &EligibleCollector{size: size,
+		ids: make([]index.IndexInternalID, 0, size),
+	}
+	return ec
+}
+
+func makeEligibleDocumentMatchHandler(ctx *search.SearchContext) (search.DocumentMatchHandler, error) {
+	if ec, ok := ctx.Collector.(*EligibleCollector); ok {
+		return func(d *search.DocumentMatch) error {
+			if d == nil {
+				return nil
+			}
+
+			copyOfID := make([]byte, len(d.IndexInternalID))
+			copy(copyOfID, d.IndexInternalID)
+			ec.ids = append(ec.ids, copyOfID)
+
+			// recycle the DocumentMatch
+			ctx.DocumentMatchPool.Put(d)
+
+			return nil
+		}, nil
+	}
+
+	return nil, fmt.Errorf("eligiblity collector not available")
+}
+
+func (ec *EligibleCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error {
+	startTime := time.Now()
+	var err error
+	var next *search.DocumentMatch
+
+	backingSize := ec.size
+	if backingSize > PreAllocSizeSkipCap {
+		backingSize = PreAllocSizeSkipCap + 1
+	}
+	searchContext := &search.SearchContext{
+		DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), 0),
+		Collector:         ec,
+		IndexReader:       reader,
+	}
+
+	dmHandler, err := makeEligibleDocumentMatchHandler(searchContext)
+	if err != nil {
+		return err
+	}
+	select {
+	case <-ctx.Done():
+		search.RecordSearchCost(ctx, search.AbortM, 0)
+		return ctx.Err()
+	default:
+		next, err = searcher.Next(searchContext)
+	}
+	for err == nil && next != nil {
+		if ec.total%CheckDoneEvery == 0 {
+			select {
+			case <-ctx.Done():
+				search.RecordSearchCost(ctx, search.AbortM, 0)
+				return ctx.Err()
+			default:
+			}
+		}
+		ec.total++
+
+		err = dmHandler(next)
+		if err != nil {
+			break
+		}
+
+		next, err = searcher.Next(searchContext)
+	}
+	if err != nil {
+		return err
+	}
+
+	// help finalize/flush the results in case
+	// of custom document match handlers.
+	err = dmHandler(nil)
+	if err != nil {
+		return err
+	}
+
+	// compute search duration
+	ec.took = time.Since(startTime)
+
+	return nil
+}
+
+func (ec *EligibleCollector) Results() search.DocumentMatchCollection {
+	return nil
+}
+
+func (ec *EligibleCollector) IDs() []index.IndexInternalID {
+	return ec.ids
+}
+
+func (ec *EligibleCollector) Total() uint64 {
+	return ec.total
+}
+
+// No concept of scoring in the eligible collector.
+func (ec *EligibleCollector) MaxScore() float64 {
+	return 0
+}
+
+func (ec *EligibleCollector) Took() time.Duration {
+	return ec.took
+}
+
+func (ec *EligibleCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) {
+	// facet unsupported for pre-filtering in KNN search
+}
+
+func (ec *EligibleCollector) FacetResults() search.FacetResults {
+	// facet unsupported for pre-filtering in KNN search
+	return nil
+}
@@ -14,7 +14,9 @@

 package collector

-import "github.com/blevesearch/bleve/v2/search"
+import (
+	"github.com/blevesearch/bleve/v2/search"
+)

 type collectStoreSlice struct {
 	slice   search.DocumentMatchCollection
@@ -74,6 +74,9 @@ func (q *BooleanQuery) SetMinShould(minShould float64) {
 }

 func (q *BooleanQuery) AddMust(m ...Query) {
+	if m == nil {
+		return
+	}
 	if q.Must == nil {
 		tmp := NewConjunctionQuery([]Query{})
 		tmp.queryStringMode = q.queryStringMode
@@ -85,6 +88,9 @@ func (q *BooleanQuery) AddMust(m ...Query) {
 }

 func (q *BooleanQuery) AddShould(m ...Query) {
+	if m == nil {
+		return
+	}
 	if q.Should == nil {
 		tmp := NewDisjunctionQuery([]Query{})
 		tmp.queryStringMode = q.queryStringMode
@@ -96,6 +102,9 @@ func (q *BooleanQuery) AddShould(m ...Query) {
 }

 func (q *BooleanQuery) AddMustNot(m ...Query) {
+	if m == nil {
+		return
+	}
 	if q.MustNot == nil {
 		tmp := NewDisjunctionQuery([]Query{})
 		tmp.queryStringMode = q.queryStringMode
@@ -35,7 +35,9 @@ type KNNQuery struct {
 	BoostVal    *Boost    `json:"boost,omitempty"`

 	// see KNNRequest.Params for description
-	Params json.RawMessage `json:"params"`
+	Params        json.RawMessage `json:"params"`
+	FilterQuery   Query           `json:"filter,omitempty"`
+	filterResults []index.IndexInternalID
 }

 func NewKNNQuery(vector []float32) *KNNQuery {
@@ -67,6 +69,14 @@ func (q *KNNQuery) SetParams(params json.RawMessage) {
 	q.Params = params
 }

+func (q *KNNQuery) SetFilterQuery(f Query) {
+	q.FilterQuery = f
+}
+
+func (q *KNNQuery) SetFilterResults(results []index.IndexInternalID) {
+	q.filterResults = results
+}
+
 func (q *KNNQuery) Searcher(ctx context.Context, i index.IndexReader,
 	m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
 	fieldMapping := m.FieldMappingForPath(q.VectorField)
@@ -77,6 +87,12 @@ func (q *KNNQuery) Searcher(ctx context.Context, i index.IndexReader,
 	if q.K <= 0 || len(q.Vector) == 0 {
 		return nil, fmt.Errorf("k must be greater than 0 and vector must be non-empty")
 	}
+	if similarityMetric == index.CosineSimilarity {
+		// normalize the vector
+		q.Vector = mapping.NormalizeVector(q.Vector)
+	}
+
 	return searcher.NewKNNSearcher(ctx, i, m, options, q.VectorField,
-		q.Vector, q.K, q.BoostVal.Value(), similarityMetric, q.Params)
+		q.Vector, q.K, q.BoostVal.Value(), similarityMetric, q.Params,
+		q.filterResults)
 }
@@ -49,11 +49,20 @@ type KNNSearcher struct {

 func NewKNNSearcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,
 	options search.SearcherOptions, field string, vector []float32, k int64,
-	boost float64, similarityMetric string, searchParams json.RawMessage) (
+	boost float64, similarityMetric string, searchParams json.RawMessage,
+	filterIDs []index.IndexInternalID) (
 	search.Searcher, error) {

 	if vr, ok := i.(index.VectorIndexReader); ok {
-		vectorReader, err := vr.VectorReader(ctx, vector, field, k, searchParams)
+		var vectorReader index.VectorReader
+		var err error
+
+		if len(filterIDs) > 0 {
+			vectorReader, err = vr.VectorReaderWithFilter(ctx, vector, field, k,
+				searchParams, filterIDs)
+		} else {
+			vectorReader, err = vr.VectorReader(ctx, vector, field, k, searchParams)
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -87,6 +87,10 @@ type KNNRequest struct {
 	//
 	// Consult go-faiss to know all supported search params
 	Params json.RawMessage `json:"params"`
+
+	// Filter query to use with kNN pre-filtering.
+	// Supports pre-filtering with all existing types of query clauses.
+	FilterQuery query.Query `json:"filter,omitempty"`
 }

 func (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost float64) {
@@ -99,6 +103,18 @@ func (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost fl
 	})
 }

+func (r *SearchRequest) AddKNNWithFilter(field string, vector []float32, k int64,
+	boost float64, filterQuery query.Query) {
+	b := query.Boost(boost)
+	r.KNN = append(r.KNN, &KNNRequest{
+		Field:       field,
+		Vector:      vector,
+		K:           k,
+		Boost:       &b,
+		FilterQuery: filterQuery,
+	})
+}
+
 func (r *SearchRequest) AddKNNOperator(operator knnOperator) {
 	r.KNNOperator = operator
 }
@@ -106,6 +122,16 @@ func (r *SearchRequest) AddKNNOperator(operator knnOperator) {
 // UnmarshalJSON deserializes a JSON representation of
 // a SearchRequest
 func (r *SearchRequest) UnmarshalJSON(input []byte) error {
+	type tempKNNReq struct {
+		Field        string          `json:"field"`
+		Vector       []float32       `json:"vector"`
+		VectorBase64 string          `json:"vector_base64"`
+		K            int64           `json:"k"`
+		Boost        *query.Boost    `json:"boost,omitempty"`
+		Params       json.RawMessage `json:"params"`
+		FilterQuery  json.RawMessage `json:"filter,omitempty"`
+	}
+
 	var temp struct {
 		Q                json.RawMessage   `json:"query"`
 		Size             *int              `json:"size"`
@@ -119,7 +145,7 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error {
 		Score            string            `json:"score"`
 		SearchAfter      []string          `json:"search_after"`
 		SearchBefore     []string          `json:"search_before"`
-		KNN              []*KNNRequest     `json:"knn"`
+		KNN              []*tempKNNReq     `json:"knn"`
 		KNNOperator      knnOperator       `json:"knn_operator"`
 		PreSearchData    json.RawMessage   `json:"pre_search_data"`
 	}
@@ -163,7 +189,22 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error {
 		r.From = 0
 	}

-	r.KNN = temp.KNN
+	r.KNN = make([]*KNNRequest, len(temp.KNN))
+	for i, knnReq := range temp.KNN {
+		r.KNN[i] = &KNNRequest{}
+		r.KNN[i].Field = temp.KNN[i].Field
+		r.KNN[i].Vector = temp.KNN[i].Vector
+		r.KNN[i].VectorBase64 = temp.KNN[i].VectorBase64
+		r.KNN[i].K = temp.KNN[i].K
+		r.KNN[i].Boost = temp.KNN[i].Boost
+		r.KNN[i].Params = temp.KNN[i].Params
+		if len(knnReq.FilterQuery) == 0 {
+			// Setting this to nil to avoid ParseQuery() setting it to a match none
+			r.KNN[i].FilterQuery = nil
+		} else {
+			r.KNN[i].FilterQuery, err = query.ParseQuery(knnReq.FilterQuery)
+		}
+	}
 	r.KNNOperator = temp.KNNOperator
 	if r.KNNOperator == "" {
 		r.KNNOperator = knnOperatorOr
@@ -209,7 +250,9 @@ var (
 	knnOperatorOr  = knnOperator("or")
 )

-func createKNNQuery(req *SearchRequest) (query.Query, []int64, int64, error) {
+func createKNNQuery(req *SearchRequest, eligibleDocsMap map[int][]index.IndexInternalID,
+	requiresFiltering map[int]bool) (
+	query.Query, []int64, int64, error) {
 	if requestHasKNN(req) {
 		// first perform validation
 		err := validateKNN(req)
@@ -219,12 +262,25 @@ func createKNNQuery(req *SearchRequest) (query.Query, []int64, int64, error) {
 		var subQueries []query.Query
 		kArray := make([]int64, 0, len(req.KNN))
 		sumOfK := int64(0)
-		for _, knn := range req.KNN {
+		for i, knn := range req.KNN {
+			// If it's a filtered kNN but has no eligible filter hits, then
+			// do not run the kNN query.
+			if requiresFiltering[i] && len(eligibleDocsMap[i]) <= 0 {
+				continue
+			}
+
 			knnQuery := query.NewKNNQuery(knn.Vector)
 			knnQuery.SetFieldVal(knn.Field)
 			knnQuery.SetK(knn.K)
 			knnQuery.SetBoost(knn.Boost.Value())
 			knnQuery.SetParams(knn.Params)
+			if len(eligibleDocsMap[i]) > 0 {
+				knnQuery.SetFilterQuery(knn.FilterQuery)
+				filterResults, exists := eligibleDocsMap[i]
+				if exists {
+					knnQuery.SetFilterResults(filterResults)
+				}
+			}
 			subQueries = append(subQueries, knnQuery)
 			kArray = append(kArray, knn.K)
 			sumOfK += knn.K
@@ -303,7 +359,62 @@ func addSortAndFieldsToKNNHits(req *SearchRequest, knnHits []*search.DocumentMat
 }

 func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) ([]*search.DocumentMatch, error) {
-	KNNQuery, kArray, sumOfK, err := createKNNQuery(req)
+	// maps the index of the KNN query in the req to the pre-filter hits aka
+	// eligible docs' internal IDs .
+	filterHitsMap := make(map[int][]index.IndexInternalID)
+	// Indicates if this query requires filtering downstream
+	// No filtering required if it's a match all query/no filters applied.
+	requiresFiltering := make(map[int]bool)
+
+	for idx, knnReq := range req.KNN {
+		// TODO Can use goroutines for this filter query stuff - do it if perf results
+		// show this to be significantly slow otherwise.
+		filterQ := knnReq.FilterQuery
+		if filterQ == nil {
+			requiresFiltering[idx] = false
+			continue
+		}
+
+		if _, ok := filterQ.(*query.MatchAllQuery); ok {
+			// Equivalent to not having a filter query.
+			requiresFiltering[idx] = false
+			continue
+		}
+
+		if _, ok := filterQ.(*query.MatchNoneQuery); ok {
+			// Filtering required since no hits are eligible.
+			requiresFiltering[idx] = true
+			// a match none query just means none the documents are eligible
+			// hence, we can save on running the query.
+			continue
+		}
+
+		// Applies to all supported types of queries.
+		filterSearcher, _ := filterQ.Searcher(ctx, reader, i.m, search.SearcherOptions{
+			Score: "none", // just want eligible hits --> don't compute scores if not needed
+		})
+		// Using the index doc count to determine collector size since we do not
+		// have an estimate of the number of eligible docs in the index yet.
+		indexDocCount, err := i.DocCount()
+		if err != nil {
+			return nil, err
+		}
+		filterColl := collector.NewEligibleCollector(int(indexDocCount))
+		err = filterColl.Collect(ctx, filterSearcher, reader)
+		if err != nil {
+			return nil, err
+		}
+		filterHits := filterColl.IDs()
+		if len(filterHits) > 0 {
+			filterHitsMap[idx] = filterHits
+		}
+		// set requiresFiltering regardless of whether there're filtered hits or
+		// not to later decide whether to consider the knnQuery or not
+		requiresFiltering[idx] = true
+	}
+
+	// Add the filter hits when creating the kNN query
+	KNNQuery, kArray, sumOfK, err := createKNNQuery(req, filterHitsMap, requiresFiltering)
 	if err != nil {
 		return nil, err
 	}