sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index with type

Browse Source 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2020-07-10 15:51:13 +02:00
parent 0f64397e79
commit b4f1a908a9
4 changed files with 75 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/proto/v0/bleve.go
+14
View File
@@ -0,0 +1,14 @@
package proto
// Bleve uses a private bleveClassifier interface to determine the type of a struct
// see https://github.com/blevesearch/bleve/blob/master/mapping/mapping.go#L32-L38
type BleveAccount struct {
Account
BleveType string `json:"bleve_type"`
}
type BleveGroup struct {
Group
BleveType string `json:"bleve_type"`
}
pkg/service/v0/accounts.go
+12 -8
View File
@@ -12,7 +12,6 @@ import (
"github.com/CiscoM31/godata"
"github.com/blevesearch/bleve"
"github.com/blevesearch/bleve/search/query"
"github.com/gofrs/uuid"
"github.com/golang/protobuf/ptypes/empty"
merrors "github.com/micro/go-micro/v2/errors"
@@ -42,8 +41,10 @@ func (s Service) indexAccounts(path string) (err error) {
return
}
for _, file := range list {
a := &proto.Account{}
if err = s.loadAccount(file.Name(), a); err != nil {
a := &proto.BleveAccount{
BleveType: "account",
}
if err = s.loadAccount(file.Name(), &a.Account); err != nil {
s.log.Error().Err(err).Str("account", file.Name()).Msg("could not load account")
continue
}
@@ -153,8 +154,6 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
var password string
var query query.Query
// check if this looks like an auth request
match := authQuery.FindStringSubmatch(in.Query)
if len(match) == 3 {
@@ -165,6 +164,12 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
}
}
// only search for accounts
tq := bleve.NewTermQuery("account")
tq.SetField("bleve_type")
query := bleve.NewConjunctionQuery(tq)
if in.Query != "" {
// parse the query like an odata filter
var q *godata.GoDataFilterQuery
@@ -174,13 +179,12 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
}
// convert to bleve query
query, err = provider.BuildBleveQuery(q)
bq, err := provider.BuildBleveQuery(q)
if err != nil {
s.log.Error().Err(err).Msg("could not build bleve query")
return merrors.InternalServerError(s.id, "could not build bleve query: %v", err.Error())
}
} else {
query = bleve.NewMatchAllQuery()
query.AddQuery(bq)
}
s.log.Debug().Interface("query", query).Msg("using query")
pkg/service/v0/groups.go
+13 -7
View File
@@ -9,7 +9,6 @@ import (
"github.com/CiscoM31/godata"
"github.com/blevesearch/bleve"
"github.com/blevesearch/bleve/search/query"
"github.com/gofrs/uuid"
"github.com/golang/protobuf/ptypes/empty"
merrors "github.com/micro/go-micro/v2/errors"
@@ -30,8 +29,10 @@ func (s Service) indexGroups(path string) (err error) {
return
}
for _, file := range list {
g := &proto.Group{}
if err = s.loadGroup(file.Name(), g); err != nil {
g := &proto.BleveGroup{
BleveType: "group",
}
if err = s.loadGroup(file.Name(), &g.Group); err != nil {
s.log.Error().Err(err).Str("group", file.Name()).Msg("could not load group")
continue
}
@@ -115,7 +116,11 @@ func (s Service) deflateMembers(g *proto.Group) {
// ListGroups implements the GroupsServiceHandler interface
func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out *proto.ListGroupsResponse) (err error) {
var query query.Query
// only search for groups
tq := bleve.NewTermQuery("group")
tq.SetField("bleve_type")
query := bleve.NewConjunctionQuery(tq)
if in.Query != "" {
// parse the query like an odata filter
@@ -126,13 +131,12 @@ func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out
}
// convert to bleve query
query, err = provider.BuildBleveQuery(q)
bq, err := provider.BuildBleveQuery(q)
if err != nil {
s.log.Error().Err(err).Msg("could not build bleve query")
return merrors.InternalServerError(s.id, "could not build bleve query: %v", err.Error())
}
} else {
query = bleve.NewMatchAllQuery()
query.AddQuery(bq)
}
s.log.Debug().Interface("query", query).Msg("using query")
@@ -320,6 +324,7 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
}
// FIXME update index!
// TODO rollback changes when only one of them failed?
// TODO store relation in another file?
// TODO return error if they are already related?
@@ -379,6 +384,7 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
}
// FIXME update index!
// TODO rollback changes when only one of them failed?
// TODO store relation in another file?
// TODO return error if they are not related?
pkg/service/v0/service.go
+36 -4
View File
@@ -11,6 +11,7 @@ import (
"github.com/blevesearch/bleve"
"github.com/blevesearch/bleve/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/analysis/analyzer/simple"
"github.com/owncloud/ocis-accounts/pkg/config"
"github.com/owncloud/ocis-accounts/pkg/proto/v0"
"github.com/owncloud/ocis-pkg/v2/log"
@@ -208,13 +209,44 @@ func New(opts ...Option) (s *Service, err error) {
}
}
mapping := bleve.NewIndexMapping()
indexMapping := bleve.NewIndexMapping()
// keep all symbols in terms to allow exact maching, eg. emails
mapping.DefaultAnalyzer = keyword.Name
indexMapping.DefaultAnalyzer = keyword.Name
// TODO don't bother to store fields as we will load the account from disk
//groupsFieldMapping := bleve.NewTextFieldMapping()
//blogMapping.AddFieldMappingsAt("memberOf", nameFieldMapping)
// TODO index groups and accounts as different types?
// TODO index groups and accounts as different types!
// Reusable mapping for text, uses english stop word removal
simpleTextFieldMapping := bleve.NewTextFieldMapping()
simpleTextFieldMapping.Analyzer = simple.Name
simpleTextFieldMapping.Store = false
// Reusable mapping for keyword text
keywordFieldMapping := bleve.NewTextFieldMapping()
keywordFieldMapping.Analyzer = keyword.Name
keywordFieldMapping.Store = false
// accounts
accountMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("account", accountMapping)
// Text
accountMapping.AddFieldMappingsAt("display_name", simpleTextFieldMapping)
accountMapping.AddFieldMappingsAt("description", simpleTextFieldMapping)
// Keywords
accountMapping.AddFieldMappingsAt("mail", keywordFieldMapping)
// groups
groupMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("group", groupMapping)
// Text
groupMapping.AddFieldMappingsAt("display_name", simpleTextFieldMapping)
groupMapping.AddFieldMappingsAt("description", simpleTextFieldMapping)
indexMapping.TypeField = "bleve_type"
s = &Service{
id: cfg.GRPC.Namespace + "." + cfg.Server.Name,
@@ -227,7 +259,7 @@ func New(opts ...Option) (s *Service, err error) {
if err = os.RemoveAll(indexDir); err != nil {
return nil, err
}
if s.index, err = bleve.New(indexDir, mapping); err != nil {
if s.index, err = bleve.New(indexDir, indexMapping); err != nil {
return
}
if err = s.indexAccounts(accountsDir); err != nil {