From 4c23ced637f603f4961b4762c2fe678c2d7552f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 13 Apr 2022 11:10:09 +0200 Subject: [PATCH] Fix the index after the protobuf refactoring --- search/pkg/search/index/index.go | 54 +++++++++++++++++---------- search/pkg/search/index/index_test.go | 52 +++++++++++++++----------- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 5b6b80465..5bdd48259 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -25,11 +25,21 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/mapping" - "github.com/owncloud/ocis/search/pkg/search" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) +type indexDocument struct { + RootID string + Path string + ID string + + Name string + Size uint64 +} + // Index represents a bleve based search index type Index struct { bleveIndex bleve.Index @@ -54,19 +64,19 @@ func New(bleveIndex bleve.Index) (*Index, error) { // Add adds a new entity to the Index func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error { entity := toEntity(ref, ri) - return i.bleveIndex.Index(entity.ID, entity) + return i.bleveIndex.Index(idToBleveId(ri.Id), entity) } // Remove removes an entity from the index func (i *Index) Remove(ri *sprovider.ResourceInfo) error { - return i.bleveIndex.Delete(ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId()) + return i.bleveIndex.Delete(idToBleveId(ri.Id)) } // Search searches the index according to the criteria specified in the given SearchIndexRequest -func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { +func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery(req.Query), - bleve.NewQueryStringQuery("Path:"+req.Reference.Path+"*"), // Limit search to this directory in the space + bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) bleveReq := bleve.NewSearchRequest(query) bleveReq.Fields = []string{"*"} @@ -75,7 +85,7 @@ func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*se return nil, err } - matches := []search.Match{} + matches := []*searchmsg.Match{} for _, h := range res.Hits { match, err := fromFields(h.Fields) if err != nil { @@ -84,7 +94,7 @@ func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*se matches = append(matches, match) } - return &search.SearchIndexResult{ + return &searchsvc.SearchIndexResponse{ Matches: matches, }, nil } @@ -96,8 +106,8 @@ func BuildMapping() mapping.IndexMapping { return indexMapping } -func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { - return &Entity{ +func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument { + return &indexDocument{ RootID: ref.ResourceId.GetStorageId() + ":" + ref.ResourceId.GetOpaqueId(), Path: ref.Path, ID: ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId(), @@ -106,25 +116,29 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { } } -func fromFields(fields map[string]interface{}) (search.Match, error) { +func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { rootIDParts := strings.SplitN(fields["RootID"].(string), ":", 2) IDParts := strings.SplitN(fields["ID"].(string), ":", 2) - return search.Match{ - Reference: &sprovider.Reference{ - ResourceId: &sprovider.ResourceId{ - StorageId: rootIDParts[0], - OpaqueId: rootIDParts[1], + return &searchmsg.Match{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootIDParts[0], + OpaqueId: rootIDParts[1], + }, + Path: fields["Path"].(string), }, - Path: fields["Path"].(string), - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Id: &searchmsg.ResourceID{ StorageId: IDParts[0], OpaqueId: IDParts[1], }, - Path: fields["Name"].(string), + Name: fields["Name"].(string), Size: uint64(fields["Size"].(float64)), }, }, nil } + +func idToBleveId(id *sprovider.ResourceId) string { + return id.StorageId + ":" + id.OpaqueId +} diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 37bf24468..5e11cab2e 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -5,7 +5,8 @@ import ( "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/owncloud/ocis/search/pkg/search" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/search/pkg/search/index" . "github.com/onsi/ginkgo/v2" @@ -73,20 +74,23 @@ var _ = Describe("Index", func() { It("finds files by name, prefix or substring match", func() { queries := []string{"foo.pdf", "foo*", "*oo.p*"} for _, query := range queries { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, }, Query: query, }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(ref.ResourceId.OpaqueId)) + Expect(res.Matches[0].Entity.Ref.Path).To(Equal(ref.Path)) + Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId)) + Expect(res.Matches[0].Entity.Name).To(Equal(ri.Path)) + Expect(res.Matches[0].Entity.Size).To(Equal(ri.Size)) } }) @@ -119,28 +123,34 @@ var _ = Describe("Index", func() { It("finds files living deeper in the tree by filename, prefix or substring match", func() { queries := []string{"nestedpdf.pdf", "nested*", "*tedpdf.*"} for _, query := range queries { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, }, Query: query, }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(nestedRef.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(nestedRef.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(nestedRI.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(nestedRI.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(nestedRI.Size)) + Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(nestedRef.ResourceId.OpaqueId)) + Expect(res.Matches[0].Entity.Ref.Path).To(Equal(nestedRef.Path)) + Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(nestedRI.Id.OpaqueId)) + Expect(res.Matches[0].Entity.Name).To(Equal(nestedRI.Path)) + Expect(res.Matches[0].Entity.Size).To(Equal(nestedRI.Size)) } }) It("does not find the higher levels when limiting the searched directory", func() { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - Path: "./nested/", + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, + Path: "./nested/", }, Query: "foo.pdf", })