sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use spaceID

Browse Source
This commit is contained in:
Michael Barz
2022-07-11 10:44:09 +02:00
parent 84d9b390a6
commit 2a113e9f7d
15 changed files with 216 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/search/pkg/search/index/index.go
+25 -13
View File
@@ -33,6 +33,7 @@ import (
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/cs3org/reva/v2/pkg/storagespace"
"google.golang.org/protobuf/types/known/timestamppb"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -218,7 +219,11 @@ func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (
query := bleve.NewConjunctionQuery(
bleve.NewQueryStringQuery(req.Query),
deletedQuery, // Skip documents that have been marked as deleted
bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space
bleve.NewQueryStringQuery("RootID:"+idToBleveId(&sprovider.ResourceId{
StorageId: req.Ref.GetResourceId().GetStorageId(),
SpaceId: req.Ref.GetResourceId().GetSpaceId(),
OpaqueId: req.Ref.GetResourceId().GetOpaqueId(),
})), // Limit search to the space
bleve.NewQueryStringQuery("Path:"+queryEscape(utils.MakeRelativePath(path.Join(req.Ref.Path, "/"))+"*")), // Limit search to this directory in the space
)
bleveReq := bleve.NewSearchRequest(query)
@@ -307,22 +312,22 @@ func fieldsToEntity(fields map[string]interface{}) *indexDocument {
}
func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2)
IDParts := strings.SplitN(fields["ID"].(string), "!", 2)
rootID, err := storagespace.ParseID(fields["RootID"].(string))
if err != nil {
return nil, err
}
rID, err := storagespace.ParseID(fields["ID"].(string))
if err != nil {
return nil, err
}
match := &searchmsg.Match{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootIDParts[0],
OpaqueId: rootIDParts[1],
},
Path: fields["Path"].(string),
},
Id: &searchmsg.ResourceID{
StorageId: IDParts[0],
OpaqueId: IDParts[1],
ResourceId: resourceIDtoSearchID(rootID),
Path: fields["Path"].(string),
},
Id: resourceIDtoSearchID(rID),
Name: fields["Name"].(string),
Size: uint64(fields["Size"].(float64)),
Type: uint64(fields["Type"].(float64)),
@@ -342,7 +347,14 @@ func idToBleveId(id *sprovider.ResourceId) string {
if id == nil {
return ""
}
return id.StorageId + "!" + id.OpaqueId
return storagespace.FormatResourceID(*id)
}
func resourceIDtoSearchID(id sprovider.ResourceId) *searchmsg.ResourceID {
return &searchmsg.ResourceID{
StorageId: id.GetStorageId(),
SpaceId: id.GetSpaceId(),
OpaqueId: id.GetOpaqueId()}
}
func queryEscape(s string) string {
services/search/pkg/search/index/index_test.go
+21 -10
View File
@@ -21,7 +21,8 @@ var _ = Describe("Index", func() {
ctx context.Context
rootId = &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "rootopaqueid",
}
filename string
@@ -33,7 +34,8 @@ var _ = Describe("Index", func() {
}
parentRi = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "parentopaqueid",
},
Path: "sub d!r",
@@ -47,11 +49,13 @@ var _ = Describe("Index", func() {
}
childRi = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "childopaqueid",
},
ParentId: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "parentopaqueid",
},
Path: "child.pdf",
@@ -65,7 +69,8 @@ var _ = Describe("Index", func() {
Query: query,
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootId.StorageId,
StorageId: "provider-1",
SpaceId: rootId.SpaceId,
OpaqueId: rootId.OpaqueId,
},
},
@@ -96,11 +101,13 @@ var _ = Describe("Index", func() {
}
ri = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "opaqueid",
},
ParentId: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "someopaqueid",
},
Path: filename,
@@ -172,7 +179,8 @@ var _ = Describe("Index", func() {
It("scopes the search to the specified space", func() {
resourceId := &sprovider.ResourceId{
StorageId: "differentstorageid",
StorageId: "provider-1",
SpaceId: "differentspaceid",
OpaqueId: "differentopaqueid",
}
assertDocCount(resourceId, `Name:foo.pdf`, 0)
@@ -222,14 +230,16 @@ var _ = Describe("Index", func() {
BeforeEach(func() {
nestedRef = &sprovider.Reference{
ResourceId: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "rootopaqueid",
},
Path: "./nested/nestedpdf.pdf",
}
nestedRI = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
StorageId: "provider-1",
SpaceId: "spaceid",
OpaqueId: "nestedopaqueid",
},
Path: "nestedpdf.pdf",
@@ -251,6 +261,7 @@ var _ = Describe("Index", func() {
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: ref.ResourceId.StorageId,
SpaceId: ref.ResourceId.SpaceId,
OpaqueId: ref.ResourceId.OpaqueId,
},
Path: "./nested/",
services/search/pkg/search/provider/searchprovider.go
+12 -6
View File
@@ -92,6 +92,7 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
opaqueMap := sdk.DecodeOpaqueMap(space.Opaque)
grantSpaceId := storagespace.FormatResourceID(provider.ResourceId{
StorageId: opaqueMap["grantStorageID"],
SpaceId: opaqueMap["grantSpaceID"],
OpaqueId: opaqueMap["grantOpaqueID"],
})
mountpointMap[grantSpaceId] = space.Id.OpaqueId
@@ -122,25 +123,26 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s
continue
}
mountpointPrefix = utils.MakeRelativePath(gpRes.Path)
sid, oid, err := storagespace.SplitID(mountpointId)
sid, spid, oid, err := storagespace.SplitID(mountpointId)
if err != nil {
p.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Str("mountpointId", mountpointId).Msg("invalid mountpoint space id")
continue
}
mountpointRootId = &searchmsg.ResourceID{
StorageId: sid,
SpaceId: spid,
OpaqueId: oid,
}
p.logger.Debug().Interface("grantSpace", space).Interface("mountpointRootId", mountpointRootId).Msg("searching a grant")
}
_, rootStorageID := storagespace.SplitStorageID(space.Root.StorageId)
res, err := p.indexClient.Search(ctx, &searchsvc.SearchIndexRequest{
Query: formatQuery(req.Query),
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: space.Root.StorageId,
OpaqueId: rootStorageID,
SpaceId: space.Root.SpaceId,
OpaqueId: space.Root.OpaqueId,
},
Path: mountpointPrefix,
},
@@ -197,14 +199,18 @@ func (p *Provider) IndexSpace(ctx context.Context, req *searchsvc.IndexSpaceRequ
// Walk the space and index all files
walker := walker.NewWalker(p.gwClient)
rootId := &provider.ResourceId{StorageId: req.SpaceId, OpaqueId: req.SpaceId}
err = walker.Walk(ownerCtx, rootId, func(wd string, info *provider.ResourceInfo, err error) error {
rootId, err := storagespace.ParseID(req.SpaceId)
if err != nil {
p.logger.Error().Err(err).Msg(err.Error())
return nil, err
}
err = walker.Walk(ownerCtx, &rootId, func(wd string, info *provider.ResourceInfo, err error) error {
if err != nil {
p.logger.Error().Err(err).Msg("error walking the tree")
}
ref := &provider.Reference{
Path: utils.MakeRelativePath(filepath.Join(wd, info.Path)),
ResourceId: rootId,
ResourceId: &rootId,
}
err = p.indexClient.Add(ref, info)
if err != nil {