sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build(deps): bump github.com/blevesearch/bleve/v2 from 2.4.4 to 2.5.0

Browse Source 
Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.4.4 to 2.5.0.
- [Release notes](https://github.com/blevesearch/bleve/releases)
- [Commits](https://github.com/blevesearch/bleve/compare/v2.4.4...v2.5.0)

---
updated-dependencies:
- dependency-name: github.com/blevesearch/bleve/v2
  dependency-version: 2.5.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot]
2025-04-29 08:41:09 +00:00
committed by GitHub
parent 3d9de26f1c
commit 0c84ba3ad2
254 changed files with 17180 additions and 2192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/github.com/blevesearch/go-faiss/index.go
Generated Vendored
+55 -33
View File
@@ -44,9 +44,20 @@ type Index interface {
// AddWithIDs is like Add, but stores xids instead of sequential IDs.
AddWithIDs(x []float32, xids []int64) error
// Applicable only to IVF indexes: Return a map of centroid ID --> []vector IDs
// for the cluster.
ObtainClusterToVecIDsFromIVFIndex() (ids map[int64][]int64, err error)
// Returns true if the index is an IVF index.
IsIVFIndex() bool
// Applicable only to IVF indexes: Returns a map where the keys
// are cluster IDs and the values represent the count of input vectors that belong
// to each cluster.
// This method only considers the given vecIDs and does not account for all
// vectors in the index.
// Example:
// If vecIDs = [1, 2, 3, 4, 5], and:
// - Vectors 1 and 2 belong to cluster 1
// - Vectors 3, 4, and 5 belong to cluster 2
// The output will be: map[1:2, 2:3]
ObtainClusterVectorCountsFromIVFIndex(vecIDs []int64) (map[int64]int64, error)
// Applicable only to IVF indexes: Returns the centroid IDs in decreasing order
// of proximity to query 'x' and their distance from 'x'
@@ -65,7 +76,7 @@ type Index interface {
labels []int64, err error)
// Applicable only to IVF indexes: Search clusters whose IDs are in eligibleCentroidIDs
SearchClustersFromIVFIndex(selector Selector, nvecs int, eligibleCentroidIDs []int64,
SearchClustersFromIVFIndex(selector Selector, eligibleCentroidIDs []int64,
minEligibleCentroids int, k int64, x, centroidDis []float32,
params json.RawMessage) ([]float32, []int64, error)
@@ -140,24 +151,31 @@ func (idx *faissIndex) Add(x []float32) error {
return nil
}
func (idx *faissIndex) ObtainClusterToVecIDsFromIVFIndex() (map[int64][]int64, error) {
// This type assertion is required to determine whether to invoke
// ObtainClustersWithDistancesFromIVFIndex, SearchClustersFromIVFIndex or not.
func (idx *faissIndex) ObtainClusterVectorCountsFromIVFIndex(vecIDs []int64) (map[int64]int64, error) {
if !idx.IsIVFIndex() {
return nil, fmt.Errorf("index is not an IVF index")
}
clusterIDs := make([]int64, len(vecIDs))
if c := C.faiss_get_lists_for_keys(
idx.idx,
(*C.idx_t)(unsafe.Pointer(&vecIDs[0])),
(C.size_t)(len(vecIDs)),
(*C.idx_t)(unsafe.Pointer(&clusterIDs[0])),
); c != 0 {
return nil, getLastError()
}
rv := make(map[int64]int64, len(vecIDs))
for _, v := range clusterIDs {
rv[v]++
}
return rv, nil
}
func (idx *faissIndex) IsIVFIndex() bool {
if ivfIdx := C.faiss_IndexIVF_cast(idx.cPtr()); ivfIdx == nil {
return nil, nil
return false
}
clusterVectorIDMap := make(map[int64][]int64)
nlist := C.faiss_IndexIVF_nlist(idx.idx)
for i := 0; i < int(nlist); i++ {
list_size := C.faiss_IndexIVF_get_list_size(idx.idx, C.size_t(i))
invlist := make([]int64, list_size)
C.faiss_IndexIVF_invlists_get_ids(idx.idx, C.size_t(i), (*C.idx_t)(&invlist[0]))
clusterVectorIDMap[int64(i)] = invlist
}
return clusterVectorIDMap, nil
return true
}
func (idx *faissIndex) ObtainClustersWithDistancesFromIVFIndex(x []float32, centroidIDs []int64) (
@@ -169,10 +187,11 @@ func (idx *faissIndex) ObtainClustersWithDistancesFromIVFIndex(x []float32, cent
}
defer includeSelector.Delete()
params, err := NewSearchParams(idx, json.RawMessage{}, includeSelector.Get())
params, err := NewSearchParams(idx, json.RawMessage{}, includeSelector.Get(), nil)
if err != nil {
return nil, nil, err
}
defer params.Delete()
// Populate these with the centroids and their distances.
centroids := make([]int64, len(centroidIDs))
@@ -180,9 +199,14 @@ func (idx *faissIndex) ObtainClustersWithDistancesFromIVFIndex(x []float32, cent
n := len(x) / idx.D()
c := C.faiss_Search_closest_eligible_centroids(idx.idx, (C.int)(n),
(*C.float)(&x[0]), (C.int)(len(centroidIDs)),
(*C.float)(&centroidDistances[0]), (*C.idx_t)(&centroids[0]), params.sp)
c := C.faiss_Search_closest_eligible_centroids(
idx.idx,
(C.idx_t)(n),
(*C.float)(&x[0]),
(C.idx_t)(len(centroidIDs)),
(*C.float)(&centroidDistances[0]),
(*C.idx_t)(&centroids[0]),
params.sp)
if c != 0 {
return nil, nil, getLastError()
}
@@ -190,24 +214,22 @@ func (idx *faissIndex) ObtainClustersWithDistancesFromIVFIndex(x []float32, cent
return centroids, centroidDistances, nil
}
func (idx *faissIndex) SearchClustersFromIVFIndex(selector Selector, nvecs int,
func (idx *faissIndex) SearchClustersFromIVFIndex(selector Selector,
eligibleCentroidIDs []int64, minEligibleCentroids int, k int64, x,
centroidDis []float32, params json.RawMessage) ([]float32, []int64, error) {
defer selector.Delete()
tempParams := defaultSearchParamsIVF{
tempParams := &defaultSearchParamsIVF{
Nlist: len(eligibleCentroidIDs),
// Have to override nprobe so that more clusters will be searched for this
// query, if required.
Nprobe: minEligibleCentroids,
Nvecs: nvecs,
}
searchParams, err := NewSearchParamsIVF(idx, params, selector.Get(),
tempParams)
searchParams, err := NewSearchParams(idx, params, selector.Get(), tempParams)
if err != nil {
return nil, nil, err
}
defer searchParams.Delete()
n := len(x) / idx.D()
@@ -285,11 +307,11 @@ func (idx *faissIndex) SearchWithoutIDs(x []float32, k int64, exclude []int64, p
defer excludeSelector.Delete()
}
searchParams, err := NewSearchParams(idx, params, selector)
defer searchParams.Delete()
searchParams, err := NewSearchParams(idx, params, selector, nil)
if err != nil {
return nil, nil, err
}
defer searchParams.Delete()
distances, labels, err = idx.searchWithParams(x, k, searchParams.sp)
@@ -305,7 +327,7 @@ func (idx *faissIndex) SearchWithIDs(x []float32, k int64, include []int64,
}
defer includeSelector.Delete()
searchParams, err := NewSearchParams(idx, params, includeSelector.Get())
searchParams, err := NewSearchParams(idx, params, includeSelector.Get(), nil)
if err != nil {
return nil, nil, err
}
vendor/github.com/blevesearch/go-faiss/search_params.go
Generated Vendored
+23 -83
View File
@@ -34,7 +34,6 @@ type searchParamsIVF struct {
type defaultSearchParamsIVF struct {
Nprobe int `json:"ivf_nprobe,omitempty"`
Nlist int `json:"ivf_nlist,omitempty"`
Nvecs int `json:"ivf_nvecs,omitempty"`
}
func (s *searchParamsIVF) Validate() error {
@@ -55,119 +54,60 @@ func getNProbeFromSearchParams(params *SearchParams) int32 {
return int32(C.faiss_SearchParametersIVF_nprobe(params.sp))
}
func NewSearchParamsIVF(idx Index, params json.RawMessage, sel *C.FaissIDSelector,
defaultParams defaultSearchParamsIVF) (*SearchParams, error) {
rv := &SearchParams{}
if ivfIdx := C.faiss_IndexIVF_cast(idx.cPtr()); ivfIdx != nil {
rv.sp = C.faiss_SearchParametersIVF_cast(rv.sp)
if len(params) == 0 && sel == nil {
return rv, nil
}
var nprobe, maxCodes, nlist int
nlist = int(C.faiss_IndexIVF_nlist(ivfIdx))
// It's important to set nprobe to the value decided at the time of
// index creation. Otherwise, nprobe will be set to the default
// value of 1.
nprobe = int(C.faiss_IndexIVF_nprobe(ivfIdx))
nvecs := idx.Ntotal()
if defaultParams.Nlist > 0 {
nlist = defaultParams.Nlist
}
if defaultParams.Nprobe > 0 {
nprobe = defaultParams.Nprobe
}
var ivfParams searchParamsIVF
if len(params) > 0 {
if err := json.Unmarshal(params, &ivfParams); err != nil {
return rv, fmt.Errorf("failed to unmarshal IVF search params, "+
"err:%v", err)
}
if err := ivfParams.Validate(); err != nil {
return rv, err
}
}
if ivfParams.NprobePct > 0 {
// in the situation when the calculated nprobe happens to be
// between 0 and 1, we'll round it up.
nprobe = max(int(float32(nlist)*(ivfParams.NprobePct/100)), 1)
}
if ivfParams.MaxCodesPct > 0 {
maxCodes = int(float32(nvecs) * (ivfParams.MaxCodesPct / 100))
} // else, maxCodes will be set to the default value of 0, which means no limit
if c := C.faiss_SearchParametersIVF_new_with(
&rv.sp,
sel,
C.size_t(nprobe),
C.size_t(maxCodes),
); c != 0 {
return rv, fmt.Errorf("failed to create faiss IVF search params")
}
}
return rv, nil
}
// Always return a valid SearchParams object,
// Returns a valid SearchParams object,
// thus caller must clean up the object
// by invoking Delete() method, even if an error is returned.
// by invoking Delete() method.
func NewSearchParams(idx Index, params json.RawMessage, sel *C.FaissIDSelector,
) (*SearchParams, error) {
defaultParams *defaultSearchParamsIVF) (*SearchParams, error) {
rv := &SearchParams{}
if c := C.faiss_SearchParameters_new(&rv.sp, sel); c != 0 {
return rv, fmt.Errorf("failed to create faiss search params")
return nil, fmt.Errorf("failed to create faiss search params")
}
// check if the index is IVF and set the search params
if ivfIdx := C.faiss_IndexIVF_cast(idx.cPtr()); ivfIdx != nil {
rv.sp = C.faiss_SearchParametersIVF_cast(rv.sp)
if len(params) == 0 && sel == nil {
return rv, nil
}
var nlist, nprobe, nvecs, maxCodes int
nlist = int(C.faiss_IndexIVF_nlist(ivfIdx))
nprobe = int(C.faiss_IndexIVF_nprobe(ivfIdx))
nvecs = int(C.faiss_Index_ntotal(idx.cPtr()))
if defaultParams != nil {
if defaultParams.Nlist > 0 {
nlist = defaultParams.Nlist
}
if defaultParams.Nprobe > 0 {
nprobe = defaultParams.Nprobe
}
}
var ivfParams searchParamsIVF
if len(params) > 0 {
if err := json.Unmarshal(params, &ivfParams); err != nil {
return rv, fmt.Errorf("failed to unmarshal IVF search params, "+
rv.Delete()
return nil, fmt.Errorf("failed to unmarshal IVF search params, "+
"err:%v", err)
}
if err := ivfParams.Validate(); err != nil {
return rv, err
rv.Delete()
return nil, err
}
}
var nprobe, maxCodes int
if ivfParams.NprobePct > 0 {
nlist := float32(C.faiss_IndexIVF_nlist(ivfIdx))
// in the situation when the calculated nprobe happens to be
// between 0 and 1, we'll round it up.
nprobe = max(int(nlist*(ivfParams.NprobePct/100)), 1)
} else {
// it's important to set nprobe to the value decided at the time of
// index creation. Otherwise, nprobe will be set to the default
// value of 1.
nprobe = int(C.faiss_IndexIVF_nprobe(ivfIdx))
nprobe = max(int(float32(nlist)*(ivfParams.NprobePct/100)), 1)
}
if ivfParams.MaxCodesPct > 0 {
nvecs := C.faiss_Index_ntotal(idx.cPtr())
maxCodes = int(float32(nvecs) * (ivfParams.MaxCodesPct / 100))
} // else, maxCodes will be set to the default value of 0, which means no limit
if c := C.faiss_SearchParametersIVF_new_with(
&rv.sp,
sel,
C.size_t(nprobe),
C.size_t(maxCodes),
); c != 0 {
return rv, fmt.Errorf("failed to create faiss IVF search params")
rv.Delete()
return nil, fmt.Errorf("failed to create faiss IVF search params")
}
}
return rv, nil
}