build(deps): bump github.com/blevesearch/bleve/v2 from 2.3.10 to 2.4.0

Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.3.10 to 2.4.0. - [Release notes](https://github.com/blevesearch/bleve/releases) - [Commits](https://github.com/blevesearch/bleve/compare/v2.3.10...v2.4.0) --- updated-dependencies: - dependency-name: github.com/blevesearch/bleve/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>
2024-04-03 16:05:32 +02:00
parent 8f432c4cdd
commit 68e4e81870
105 changed files with 15633 additions and 485 deletions
@@ -50,7 +50,8 @@ type DocumentMapping struct {
 	StructTagKey string `json:"struct_tag_key,omitempty"`
 }

-func (dm *DocumentMapping) Validate(cache *registry.Cache) error {
+func (dm *DocumentMapping) Validate(cache *registry.Cache,
+	parentName string, fieldAliasCtx map[string]*FieldMapping) error {
 	var err error
 	if dm.DefaultAnalyzer != "" {
 		_, err := cache.AnalyzerNamed(dm.DefaultAnalyzer)
@@ -58,8 +59,12 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache) error {
 			return err
 		}
 	}
-	for _, property := range dm.Properties {
-		err = property.Validate(cache)
+	for propertyName, property := range dm.Properties {
+		newParent := propertyName
+		if parentName != "" {
+			newParent = fmt.Sprintf("%s.%s", parentName, propertyName)
+		}
+		err = property.Validate(cache, newParent, fieldAliasCtx)
 		if err != nil {
 			return err
 		}
@@ -77,15 +82,25 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache) error {
 				return err
 			}
 		}
-		switch field.Type {
-		case "text", "datetime", "number", "boolean", "geopoint", "geoshape", "IP":
-		default:
-			return fmt.Errorf("unknown field type: '%s'", field.Type)
+
+		err := validateFieldMapping(field, parentName, fieldAliasCtx)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
 }

+func validateFieldType(field *FieldMapping) error {
+	switch field.Type {
+	case "text", "datetime", "number", "boolean", "geopoint", "geoshape", "IP":
+		return nil
+	default:
+		return fmt.Errorf("field: '%s', unknown field type: '%s'",
+			field.Name, field.Type)
+	}
+}
+
 // analyzerNameForPath attempts to first find the field
 // described by this path, then returns the analyzer
 // configured for that field
@@ -141,15 +156,20 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping {
 	return nil
 }

-// documentMappingForPath returns the EXACT and closest matches for a sub
+// documentMappingForPathElements returns the EXACT and closest matches for a sub
 // document or for an explicitly mapped field; the closest most specific
 // document mapping could be one that matches part of the provided path.
-func (dm *DocumentMapping) documentMappingForPath(path string) (
+func (dm *DocumentMapping) documentMappingForPathElements(pathElements []string) (
 	*DocumentMapping, *DocumentMapping) {
-	pathElements := decodePath(path)
+	var pathElementsCopy []string
+	if len(pathElements) == 0 {
+		pathElementsCopy = []string{""}
+	} else {
+		pathElementsCopy = pathElements
+	}
 	current := dm
 OUTER:
-	for i, pathElement := range pathElements {
+	for i, pathElement := range pathElementsCopy {
 		if subDocMapping, exists := current.Properties[pathElement]; exists {
 			current = subDocMapping
 			continue OUTER
@@ -157,7 +177,7 @@ OUTER:

 		// no subDocMapping matches this pathElement
 		// only if this is the last element check for field name
-		if i == len(pathElements)-1 {
+		if i == len(pathElementsCopy)-1 {
 			for _, field := range current.Fields {
 				if field.Name == pathElement {
 					break
@@ -170,6 +190,15 @@ OUTER:
 	return current, current
 }

+// documentMappingForPath returns the EXACT and closest matches for a sub
+// document or for an explicitly mapped field; the closest most specific
+// document mapping could be one that matches part of the provided path.
+func (dm *DocumentMapping) documentMappingForPath(path string) (
+	*DocumentMapping, *DocumentMapping) {
+	pathElements := decodePath(path)
+	return dm.documentMappingForPathElements(pathElements)
+}
+
 // NewDocumentMapping returns a new document mapping
 // with all the default values.
 func NewDocumentMapping() *DocumentMapping {
@@ -388,9 +417,8 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes
 }

 func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) {
-	pathString := encodePath(path)
 	// look to see if there is a mapping for this field
-	subDocMapping, closestDocMapping := dm.documentMappingForPath(pathString)
+	subDocMapping, closestDocMapping := dm.documentMappingForPathElements(path)

 	// check to see if we even need to do further processing
 	if subDocMapping != nil && !subDocMapping.Enabled {
@@ -402,6 +430,8 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 		// cannot do anything with the zero value
 		return
 	}
+
+	pathString := encodePath(path)
 	propertyType := propertyValue.Type()
 	switch propertyType.Kind() {
 	case reflect.String:
@@ -502,9 +532,20 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 			dm.walkDocument(property, path, indexes, context)
 		}
 	case reflect.Map, reflect.Slice:
+		var isPropertyVector bool
+		var isPropertyVectorInitialized bool
 		if subDocMapping != nil {
 			for _, fieldMapping := range subDocMapping.Fields {
 				switch fieldMapping.Type {
+				case "vector":
+					processed := fieldMapping.processVector(property, pathString, path,
+						indexes, context)
+					if !isPropertyVectorInitialized {
+						isPropertyVector = processed
+						isPropertyVectorInitialized = true
+					} else {
+						isPropertyVector = isPropertyVector && processed
+					}
 				case "geopoint":
 					fieldMapping.processGeoPoint(property, pathString, path, indexes, context)
 				case "IP":
@@ -517,7 +558,9 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
 				}
 			}
 		}
-		dm.walkDocument(property, path, indexes, context)
+		if !isPropertyVector {
+			dm.walkDocument(property, path, indexes, context)
+		}
 	case reflect.Ptr:
 		if !propertyValue.IsNil() {
 			switch property := property.(type) {
@@ -69,6 +69,17 @@ type FieldMapping struct {
 	// the processing of freq/norm details when the default score based relevancy
 	// isn't needed.
 	SkipFreqNorm bool `json:"skip_freq_norm,omitempty"`
+
+	// Dimensionality of the vector
+	Dims int `json:"dims,omitempty"`
+
+	// Similarity is the similarity algorithm used for scoring
+	// vector fields.
+	// See: index.DefaultSimilarityMetric & index.SupportedSimilarityMetrics
+	Similarity string `json:"similarity,omitempty"`
+
+	// Applicable to vector fields only - optimization string
+	VectorIndexOptimizedFor string `json:"vector_index_optimized_for,omitempty"`
 }

 // NewTextFieldMapping returns a default field mapping for text
@@ -448,6 +459,21 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "dims":
+			err := json.Unmarshal(v, &fm.Dims)
+			if err != nil {
+				return err
+			}
+		case "similarity":
+			err := json.Unmarshal(v, &fm.Similarity)
+			if err != nil {
+				return err
+			}
+		case "vector_index_optimized_for":
+			err := json.Unmarshal(v, &fm.VectorIndexOptimizedFor)
+			if err != nil {
+				return err
+			}
 		default:
 			invalidKeys = append(invalidKeys, k)
 		}
@@ -174,12 +174,14 @@ func (im *IndexMappingImpl) Validate() error {
 	if err != nil {
 		return err
 	}
-	err = im.DefaultMapping.Validate(im.cache)
+
+	fieldAliasCtx := make(map[string]*FieldMapping)
+	err = im.DefaultMapping.Validate(im.cache, "", fieldAliasCtx)
 	if err != nil {
 		return err
 	}
 	for _, docMapping := range im.TypeMapping {
-		err = docMapping.Validate(im.cache)
+		err = docMapping.Validate(im.cache, "", fieldAliasCtx)
 		if err != nil {
 			return err
 		}
@@ -431,6 +433,33 @@ func (im *IndexMappingImpl) FieldAnalyzer(field string) string {
 	return im.AnalyzerNameForPath(field)
 }

+// FieldMappingForPath returns the mapping for a specific field 'path'.
+func (im *IndexMappingImpl) FieldMappingForPath(path string) FieldMapping {
+	if im.TypeMapping != nil {
+		for _, v := range im.TypeMapping {
+			for field, property := range v.Properties {
+				for _, v1 := range property.Fields {
+					if field == path {
+						// Return field mapping if the name matches the path param.
+						return *v1
+					}
+				}
+			}
+		}
+	}
+
+	for field, property := range im.DefaultMapping.Properties {
+		for _, v1 := range property.Fields {
+			if field == path {
+				// Return field mapping if the name matches the path param.
+				return *v1
+			}
+		}
+	}
+
+	return FieldMapping{}
+}
+
 // wrapper to satisfy new interface

 func (im *IndexMappingImpl) DefaultSearchField() string {
@@ -55,4 +55,6 @@ type IndexMapping interface {

 	AnalyzerNameForPath(path string) string
 	AnalyzerNamed(name string) analysis.Analyzer
+
+	FieldMappingForPath(path string) FieldMapping
 }
@@ -0,0 +1,35 @@
+//  Copyright (c) 2023 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !vectors
+// +build !vectors
+
+package mapping
+
+func NewVectorFieldMapping() *FieldMapping {
+	return nil
+}
+
+func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) bool {
+	return false
+}
+
+// -----------------------------------------------------------------------------
+// document validation functions
+
+func validateFieldMapping(field *FieldMapping, parentName string,
+	fieldAliasCtx map[string]*FieldMapping) error {
+	return validateFieldType(field)
+}
@@ -0,0 +1,220 @@
+//  Copyright (c) 2023 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build vectors
+// +build vectors
+
+package mapping
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/util"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+// Min and Max allowed dimensions for a vector field
+const (
+	MinVectorDims = 1
+	MaxVectorDims = 2048
+)
+
+func NewVectorFieldMapping() *FieldMapping {
+	return &FieldMapping{
+		Type:         "vector",
+		Store:        false,
+		Index:        true,
+		IncludeInAll: false,
+		DocValues:    false,
+		SkipFreqNorm: true,
+	}
+}
+
+// validate and process a flat vector
+func processFlatVector(vecV reflect.Value, dims int) ([]float32, bool) {
+	if vecV.Len() != dims {
+		return nil, false
+	}
+
+	rv := make([]float32, dims)
+	for i := 0; i < vecV.Len(); i++ {
+		item := vecV.Index(i)
+		if !item.CanInterface() {
+			return nil, false
+		}
+		itemI := item.Interface()
+		itemFloat, ok := util.ExtractNumericValFloat32(itemI)
+		if !ok {
+			return nil, false
+		}
+		rv[i] = itemFloat
+	}
+
+	return rv, true
+}
+
+// validate and process a vector
+// max supported depth of nesting is 2 ([][]float32)
+func processVector(vecI interface{}, dims int) ([]float32, bool) {
+	vecV := reflect.ValueOf(vecI)
+	if !vecV.IsValid() || vecV.Kind() != reflect.Slice || vecV.Len() == 0 {
+		return nil, false
+	}
+
+	// Let's examine the first element (head) of the vector.
+	// If head is a slice, then vector is nested, otherwise flat.
+	head := vecV.Index(0)
+	if !head.CanInterface() {
+		return nil, false
+	}
+	headI := head.Interface()
+	headV := reflect.ValueOf(headI)
+	if !headV.IsValid() {
+		return nil, false
+	}
+	if headV.Kind() != reflect.Slice { // vector is flat
+		return processFlatVector(vecV, dims)
+	}
+
+	// # process nested vector
+
+	// pre-allocate memory for the flattened vector
+	// so that we can use copy() later
+	rv := make([]float32, dims*vecV.Len())
+
+	for i := 0; i < vecV.Len(); i++ {
+		subVec := vecV.Index(i)
+		if !subVec.CanInterface() {
+			return nil, false
+		}
+		subVecI := subVec.Interface()
+		subVecV := reflect.ValueOf(subVecI)
+		if !subVecV.IsValid() {
+			return nil, false
+		}
+
+		if subVecV.Kind() != reflect.Slice {
+			return nil, false
+		}
+
+		flatVector, ok := processFlatVector(subVecV, dims)
+		if !ok {
+			return nil, false
+		}
+
+		copy(rv[i*dims:(i+1)*dims], flatVector)
+	}
+
+	return rv, true
+}
+
+func (fm *FieldMapping) processVector(propertyMightBeVector interface{},
+	pathString string, path []string, indexes []uint64, context *walkContext) bool {
+	vector, ok := processVector(propertyMightBeVector, fm.Dims)
+	// Don't add field to document if vector is invalid
+	if !ok {
+		return false
+	}
+
+	fieldName := getFieldName(pathString, path, fm)
+	options := fm.Options()
+	field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, vector,
+		fm.Dims, fm.Similarity, fm.VectorIndexOptimizedFor, options)
+	context.doc.AddField(field)
+
+	// "_all" composite field is not applicable for vector field
+	context.excludedFromAll = append(context.excludedFromAll, fieldName)
+	return true
+}
+
+// -----------------------------------------------------------------------------
+// document validation functions
+
+func validateFieldMapping(field *FieldMapping, parentName string,
+	fieldAliasCtx map[string]*FieldMapping) error {
+	switch field.Type {
+	case "vector":
+		return validateVectorFieldAlias(field, parentName, fieldAliasCtx)
+	default: // non-vector field
+		return validateFieldType(field)
+	}
+}
+
+func validateVectorFieldAlias(field *FieldMapping, parentName string,
+	fieldAliasCtx map[string]*FieldMapping) error {
+
+	if field.Name == "" {
+		field.Name = parentName
+	}
+
+	if field.Similarity == "" {
+		field.Similarity = index.DefaultSimilarityMetric
+	}
+
+	if field.VectorIndexOptimizedFor == "" {
+		field.VectorIndexOptimizedFor = index.DefaultIndexOptimization
+	}
+	if _, exists := index.SupportedVectorIndexOptimizations[field.VectorIndexOptimizedFor]; !exists {
+		// if an unsupported config is provided, override to default
+		field.VectorIndexOptimizedFor = index.DefaultIndexOptimization
+	}
+
+	// following fields are not applicable for vector
+	// thus, we set them to default values
+	field.IncludeInAll = false
+	field.IncludeTermVectors = false
+	field.Store = false
+	field.DocValues = false
+	field.SkipFreqNorm = true
+
+	// # If alias is present, validate the field options as per the alias
+	// note: reading from a nil map is safe
+	if fieldAlias, ok := fieldAliasCtx[field.Name]; ok {
+		if field.Dims != fieldAlias.Dims {
+			return fmt.Errorf("field: '%s', invalid alias "+
+				"(different dimensions %d and %d)", fieldAlias.Name, field.Dims,
+				fieldAlias.Dims)
+		}
+
+		if field.Similarity != fieldAlias.Similarity {
+			return fmt.Errorf("field: '%s', invalid alias "+
+				"(different similarity values %s and %s)", fieldAlias.Name,
+				field.Similarity, fieldAlias.Similarity)
+		}
+
+		return nil
+	}
+
+	// # Validate field options
+
+	if field.Dims < MinVectorDims || field.Dims > MaxVectorDims {
+		return fmt.Errorf("field: '%s', invalid vector dimension: %d,"+
+			" value should be in range (%d, %d)", field.Name, field.Dims,
+			MinVectorDims, MaxVectorDims)
+	}
+
+	if _, ok := index.SupportedSimilarityMetrics[field.Similarity]; !ok {
+		return fmt.Errorf("field: '%s', invalid similarity "+
+			"metric: '%s', valid metrics are: %+v", field.Name, field.Similarity,
+			reflect.ValueOf(index.SupportedSimilarityMetrics).MapKeys())
+	}
+
+	if fieldAliasCtx != nil { // writing to a nil map is unsafe
+		fieldAliasCtx[field.Name] = field
+	}
+
+	return nil
+}