sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

persist accounts using json, index using bleve

Browse Source 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2020-06-15 16:04:35 +02:00
parent e78c4395de
commit c2668daae8
9 changed files with 308 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/service/v0/service.go
+79 -149
View File
@@ -2,12 +2,16 @@ package service
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"github.com/CiscoM31/godata"
"github.com/blevesearch/bleve"
"github.com/blevesearch/bleve/search/query"
"github.com/golang/protobuf/ptypes/empty"
mclient "github.com/micro/go-micro/v2/client"
"github.com/owncloud/ocis-accounts/pkg/config"
@@ -16,13 +20,55 @@ import (
olog "github.com/owncloud/ocis-pkg/v2/log"
settings "github.com/owncloud/ocis-settings/pkg/proto/v0"
"github.com/rs/zerolog/log"
"gopkg.in/ldap.v2"
)
// New returns a new instance of Service
func New(cfg *config.Config) Service {
// read all user and group records
// for now recreate index on every start
os.RemoveAll(filepath.Join(cfg.Server.AccountsDataPath, "index.bleve"))
os.MkdirAll(filepath.Join(cfg.Server.AccountsDataPath, "accounts"), 0700)
mapping := bleve.NewIndexMapping()
// TODO don't bother to store fields as we will load the account from disk
index, err := bleve.New(filepath.Join(cfg.Server.AccountsDataPath, "index.bleve"), mapping)
if err != nil {
panic(err)
}
f, err := os.Open(filepath.Join(cfg.Server.AccountsDataPath, "accounts"))
if err != nil {
log.Error().Err(err).Str("dir", filepath.Join(cfg.Server.AccountsDataPath, "accounts")).Msg("could not open acconts folder")
panic(err)
}
list, err := f.Readdir(-1)
f.Close()
if err != nil {
log.Error().Err(err).Str("dir", filepath.Join(cfg.Server.AccountsDataPath, "accounts")).Msg("could not list accounts folder")
panic(err)
}
for _, file := range list {
path := filepath.Join(cfg.Server.AccountsDataPath, "accounts", file.Name())
data, err := ioutil.ReadFile(path)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("could not read account")
continue
}
a := proto.Account{}
err = json.Unmarshal(data, &a)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("could not unmarshal account")
continue
}
log.Debug().Interface("account", a).Msg("found account")
index.Index(a.Id, a)
}
// TODO watch folders for new records
s := Service{
Config: cfg,
index: index,
}
return s
@@ -31,53 +77,7 @@ func New(cfg *config.Config) Service {
// Service implements the AccountsServiceHandler interface
type Service struct {
Config *config.Config
}
func (s Service) getBoundConnection(binddn string, password string) (l *ldap.Conn, err error) {
l, err = ldap.DialTLS("tcp", fmt.Sprintf("%s:%d", s.Config.LDAP.Hostname, s.Config.LDAP.Port), &tls.Config{InsecureSkipVerify: true})
if err != nil {
return nil, err
}
err = l.Bind(binddn, password)
if err != nil {
l.Close()
return nil, err
}
return
}
func (s Service) lookupDN(username string) (binddn string, err error) {
l, err := s.getBoundConnection(s.Config.LDAP.BindDN, s.Config.LDAP.BindPassword)
if err != nil {
return "", err
}
defer l.Close()
filter := fmt.Sprintf("(%s=%s)", s.Config.LDAP.Schema.Username, ldap.EscapeFilter(username))
// Search for the given username
searchRequest := ldap.NewSearchRequest(
s.Config.LDAP.BaseDN,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
filter,
[]string{"dn"},
nil,
)
sr, err := l.Search(searchRequest)
if err != nil {
return "", err
}
switch len(sr.Entries) {
case 0: // TODO return not found error
case 1:
return sr.Entries[0].DN, nil
default: // TODO return too many results error?
}
return "", fmt.Errorf("dn not found for %s", filter)
index bleve.Index
}
// the auth request is currently hardcoded and has to macth this regex
@@ -89,82 +89,50 @@ var authQuery = regexp.MustCompile(`^username eq '(.*)' and password eq '(.*)'$`
// TODO id vs onpremiseimmutableid
func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest, res *proto.ListAccountsResponse) (err error) {
var binddn string
var password string
// check if this looks like an auth request
match := authQuery.FindStringSubmatch(in.Query)
if len(match) == 3 {
binddn, err = s.lookupDN(match[1])
if err != nil {
log.Error().Err(err).Msg("ListAccounts with auth request")
return
}
log.Debug().Str("username", match[1]).Str("binddn", binddn).Msg("ListAccounts with auth request")
password = match[2]
// remove password from query
in.Query = fmt.Sprintf("username eq '%s'", match[1])
} else {
log.Debug().Str("query", in.Query).Int32("page-size", in.PageSize).Str("page-token", in.PageToken).Msg("ListAccounts")
binddn = s.Config.LDAP.BindDN
password = s.Config.LDAP.BindPassword
}
filter := "(&)" // see Absolute True and False Filters in https://tools.ietf.org/html/rfc4526#section-2
var query query.Query
if in.Query != "" {
// parse the query like an odata filter
var q *godata.GoDataFilterQuery
if q, err = godata.ParseFilterString(in.Query); err != nil {
log.Error().Err(err).Msg("could not parse query")
return
}
// convert to ldap filter
filter, err = provider.BuildLDAPFilter(q, &s.Config.LDAP.Schema)
// convert to bleve query
query, err = provider.BuildBleveQuery(q)
if err != nil {
log.Error().Err(err).Msg("could not build bleve query")
return
}
} else {
query = bleve.NewMatchAllQuery()
}
log.Debug().Str("filter", filter).Msg("using filter")
log.Debug().Interface("query", query).Msg("using query")
var l *ldap.Conn
l, err = s.getBoundConnection(binddn, password)
if err != nil {
return
}
defer l.Close()
// TODO combine the parsed query with a query filter from the config, eg. fmt.Sprintf(s.Config.LDAP.UserFilter, clientID)
// Search for the given clientID
searchRequest := ldap.NewSearchRequest(
s.Config.LDAP.BaseDN,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
filter,
[]string{"dn", s.Config.LDAP.Schema.AccountID, s.Config.LDAP.Schema.Username, s.Config.LDAP.Schema.DisplayName, s.Config.LDAP.Schema.Mail, s.Config.LDAP.Schema.Groups}, // TODO Groups, Identities?
nil,
)
sr, err := l.Search(searchRequest)
if err != nil {
return err
}
log.Debug().Interface("entries", sr.Entries).Msg("entries")
searchRequest := bleve.NewSearchRequest(query)
searchResult, err := s.index.Search(searchRequest)
log.Debug().Interface("result", searchResult).Msg("result")
res.Accounts = make([]*proto.Account, 0)
for i := range sr.Entries {
res.Accounts = append(res.Accounts, &proto.Account{
Id: sr.Entries[i].GetAttributeValue(s.Config.LDAP.Schema.AccountID),
// TODO identities
Username: sr.Entries[i].GetAttributeValue(s.Config.LDAP.Schema.Username),
DisplayName: sr.Entries[i].GetAttributeValue(s.Config.LDAP.Schema.DisplayName),
Mail: sr.Entries[i].GetAttributeValue(s.Config.LDAP.Schema.Mail),
//Groups: sr.Entries[i].GetAttributeValues(s.Config.LDAP.Schema.Groups),
})
for _, hit := range searchResult.Hits {
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", hit.ID)
data, err := ioutil.ReadFile(path)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("could not read account")
continue
}
a := proto.Account{}
err = json.Unmarshal(data, &a)
if err != nil {
log.Error().Err(err).Str("path", path).Msg("could not unmarshal account")
continue
}
log.Debug().Interface("account", a).Msg("found account")
res.Accounts = append(res.Accounts, &a)
}
return nil
@@ -172,45 +140,7 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
// GetAccount implements the AccountsServiceHandler interface
func (s Service) GetAccount(c context.Context, req *proto.GetAccountRequest, res *proto.Account) (err error) {
l, err := s.getBoundConnection(s.Config.LDAP.BindDN, s.Config.LDAP.BindPassword)
if err != nil {
return err
}
defer l.Close()
// TODO combine the parsed query with a query filter from the config, eg. fmt.Sprintf(s.Config.LDAP.UserFilter, clientID)
filter := fmt.Sprintf("(%s=%s)", s.Config.LDAP.Schema.AccountID, ldap.EscapeFilter(req.Id))
// Search for the given clientID
searchRequest := ldap.NewSearchRequest(
s.Config.LDAP.BaseDN,
ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
filter,
[]string{"dn", s.Config.LDAP.Schema.AccountID, s.Config.LDAP.Schema.Username, s.Config.LDAP.Schema.DisplayName, s.Config.LDAP.Schema.Mail, s.Config.LDAP.Schema.Groups}, // TODO Groups, Identities?
nil,
)
sr, err := l.Search(searchRequest)
if err != nil {
return err
}
log.Debug().Interface("entries", sr.Entries).Msg("entries")
switch len(sr.Entries) {
case 0: // TODO return not found error
case 1:
res.Id = sr.Entries[0].GetAttributeValue(s.Config.LDAP.Schema.AccountID)
// TODO identities?
res.Username = sr.Entries[0].GetAttributeValue(s.Config.LDAP.Schema.Username)
res.DisplayName = sr.Entries[0].GetAttributeValue(s.Config.LDAP.Schema.DisplayName)
res.Mail = sr.Entries[0].GetAttributeValue(s.Config.LDAP.Schema.Mail)
// TODO groups
default: // TODO return too many results error?
}
return nil
return errors.New("not implemented")
}
// CreateAccount implements the AccountsServiceHandler interface