sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #623 from owncloud/extract_storage_layer

Browse Source 
Extract storage layer + cs3impl
This commit is contained in:
Benedikt Kulmann
2020-10-05 17:19:58 +02:00
committed by GitHub
parent fd3f0b7aae ff4c802ee2
commit bb5ccae157
22 changed files with 1155 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files
accounts/changelog/unreleased/configurable-storage.md
+7
View File
@@ -0,0 +1,7 @@
Change: Choose disk or cs3 storage for accounts and groups
The accounts service now has an abstraction layer for the storage. In addition to the local disk implementation
we implemented a cs3 storage, which is the new default for the accounts service.
https://github.com/owncloud/ocis/pull/623
accounts/go.mod
+3
View File
@@ -5,6 +5,8 @@ go 1.13
require (
github.com/CiscoM31/godata v0.0.0-20191007193734-c2c4ebb1b415
github.com/blevesearch/bleve v1.0.9
github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd
github.com/cs3org/reva v1.1.0
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
@@ -32,6 +34,7 @@ require (
github.com/tredoe/osutil v1.0.5
golang.org/x/net v0.0.0-20200822124328-c89045814202
google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
google.golang.org/grpc v1.31.0
google.golang.org/protobuf v1.25.0
)
accounts/pkg/config/config.go
+28
View File
@@ -61,6 +61,32 @@ type Log struct {
Color bool
}
// Repo defines which storage implementation is to be used.
type Repo struct {
Disk Disk
CS3 CS3
}
// Disk is the local disk implementation of the storage.
type Disk struct {
Path string
}
// CS3 is the cs3 implementation of the storage.
type CS3 struct {
ProviderAddr string
DataURL string
DataPrefix string
}
// ServiceUser defines the user required for EOS
type ServiceUser struct {
UUID string
Username string
UID int64
GID int64
}
// Config merges all Account config parameters.
type Config struct {
LDAP LDAP
@@ -70,6 +96,8 @@ type Config struct {
Asset Asset
Log Log
TokenManager TokenManager
Repo Repo
ServiceUser ServiceUser
}
// New returns a new config.
accounts/pkg/flagset/flagset.go
+56
View File
@@ -99,6 +99,62 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"ACCOUNTS_JWT_SECRET"},
Destination: &cfg.TokenManager.JWTSecret,
},
&cli.StringFlag{
Name: "storage-disk-path",
Value: "",
Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts",
EnvVars: []string{"ACCOUNTS_STORAGE_DISK_PATH"},
Destination: &cfg.Repo.Disk.Path,
},
&cli.StringFlag{
Name: "storage-cs3-provider-addr",
Value: "localhost:9215",
Usage: "bind address for the metadata storage provider",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"},
Destination: &cfg.Repo.CS3.ProviderAddr,
},
&cli.StringFlag{
Name: "storage-cs3-data-url",
Value: "http://localhost:9216",
Usage: "http endpoint of the metadata storage",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_URL"},
Destination: &cfg.Repo.CS3.DataURL,
},
&cli.StringFlag{
Name: "storage-cs3-data-prefix",
Value: "data",
Usage: "path prefix for the http endpoint of the metadata storage, without leading slash",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"},
Destination: &cfg.Repo.CS3.DataPrefix,
},
&cli.StringFlag{
Name: "service-user-uuid",
Value: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
Usage: "uuid of the internal service user (required on EOS)",
EnvVars: []string{"ACCOUNTS_SERVICE_USER_UUID"},
Destination: &cfg.ServiceUser.UUID,
},
&cli.StringFlag{
Name: "service-user-username",
Value: "",
Usage: "username of the internal service user (required on EOS)",
EnvVars: []string{"ACCOUNTS_SERVICE_USER_USERNAME"},
Destination: &cfg.ServiceUser.Username,
},
&cli.Int64Flag{
Name: "service-user-uid",
Value: 0,
Usage: "uid of the internal service user (required on EOS)",
EnvVars: []string{"ACCOUNTS_SERVICE_USER_UID"},
Destination: &cfg.ServiceUser.UID,
},
&cli.Int64Flag{
Name: "service-user-gid",
Value: 0,
Usage: "gid of the internal service user (required on EOS)",
EnvVars: []string{"ACCOUNTS_SERVICE_USER_GID"},
Destination: &cfg.ServiceUser.GID,
},
}
}
accounts/pkg/proto/v0/accounts.pb.micro_test.go
+3 -27
View File
@@ -163,6 +163,7 @@ func init() {
cfg := config.New()
cfg.Server.AccountsDataPath = dataPath
cfg.Repo.Disk.Path = dataPath
var hdlr *svc.Service
var err error
@@ -754,7 +755,6 @@ func TestGetGroupInvalidID(t *testing.T) {
assert.IsType(t, &proto.Group{}, resp)
assert.Empty(t, resp)
assert.Error(t, err)
assert.Equal(t, "{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/42: no such file or directory\",\"status\":\"Not Found\"}", err.Error())
cleanUp(t)
}
@@ -804,11 +804,6 @@ func TestDeleteGroupNotExisting(t *testing.T) {
assert.IsType(t, &empty.Empty{}, res)
assert.Empty(t, res)
assert.Error(t, err)
assert.Equal(
t,
fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read group: open accounts-store/groups/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
err.Error(),
)
}
cleanUp(t)
}
@@ -826,17 +821,12 @@ func TestDeleteGroupInvalidId(t *testing.T) {
client := service.Client()
cl := proto.NewGroupsService("com.owncloud.api.accounts", client)
for id, val := range invalidIds {
for id := range invalidIds {
req := &proto.DeleteGroupRequest{Id: id}
res, err := cl.DeleteGroup(context.Background(), req)
assert.IsType(t, &empty.Empty{}, res)
assert.Empty(t, res)
assert.Error(t, err)
assert.Equal(
t,
fmt.Sprintf("{\"id\":\".\",\"code\":500,\"detail\":\"could not clean up group id: invalid id %v\",\"status\":\"Internal Server Error\"}", val),
err.Error(),
)
}
cleanUp(t)
}
@@ -859,11 +849,7 @@ func TestUpdateGroup(t *testing.T) {
assert.IsType(t, &proto.Group{}, res)
assert.Empty(t, res)
assert.Error(t, err)
assert.Equal(
t,
"{\"id\":\".\",\"code\":500,\"detail\":\"not implemented\",\"status\":\"Internal Server Error\"}",
err.Error(),
)
cleanUp(t)
}
@@ -953,11 +939,6 @@ func TestAddMemberNonExisting(t *testing.T) {
assert.IsType(t, &proto.Group{}, res)
assert.Empty(t, res)
assert.Error(t, err)
assert.Equal(
t,
fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
err.Error(),
)
}
// Check group is not changed
@@ -1029,11 +1010,6 @@ func TestRemoveMemberNonExistingUser(t *testing.T) {
assert.IsType(t, &proto.Group{}, res)
assert.Empty(t, res)
assert.Error(t, err)
assert.Equal(
t,
fmt.Sprintf("{\"id\":\".\",\"code\":404,\"detail\":\"could not read account: open accounts-store/accounts/%v: no such file or directory\",\"status\":\"Not Found\"}", id),
err.Error(),
)
}
// Check group is not changed
accounts/pkg/service/v0/accounts.go
+70 -90
View File
@@ -2,10 +2,7 @@ package service
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"sync"
@@ -19,6 +16,7 @@ import (
merrors "github.com/micro/go-micro/v2/errors"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
"github.com/owncloud/ocis/accounts/pkg/provider"
"github.com/owncloud/ocis/accounts/pkg/storage"
"github.com/owncloud/ocis/ocis-pkg/roles"
settings "github.com/owncloud/ocis/settings/pkg/proto/v0"
settings_svc "github.com/owncloud/ocis/settings/pkg/service/v0"
@@ -37,33 +35,11 @@ import (
// accLock mutually exclude readers from writers on account files
var accLock sync.Mutex
func (s Service) indexAccounts(path string) (err error) {
var f *os.File
if f, err = os.Open(path); err != nil {
s.log.Error().Err(err).Str("dir", path).Msg("could not open accounts folder")
return
}
list, err := f.Readdir(-1)
f.Close()
if err != nil {
s.log.Error().Err(err).Str("dir", path).Msg("could not list accounts folder")
return
}
for _, file := range list {
err = s.indexAccount(file.Name())
if err != nil {
s.log.Error().Err(err).Str("file", file.Name()).Msg("could not index account")
}
}
return
}
func (s Service) indexAccount(id string) error {
a := &proto.BleveAccount{
BleveType: "account",
}
if err := s.loadAccount(id, &a.Account); err != nil {
if err := s.repo.LoadAccount(context.Background(), id, &a.Account); err != nil {
s.log.Error().Err(err).Str("account", id).Msg("could not load account")
return err
}
@@ -79,37 +55,6 @@ func (s Service) indexAccount(id string) error {
// login eq \"teddy\" and password eq \"F&1!b90t111!\"
var authQuery = regexp.MustCompile(`^login eq '(.*)' and password eq '(.*)'$`) // TODO how is ' escaped in the password?
func (s Service) loadAccount(id string, a *proto.Account) (err error) {
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
return merrors.NotFound(s.id, "could not read account: %v", err.Error())
}
if err = json.Unmarshal(data, a); err != nil {
return merrors.InternalServerError(s.id, "could not unmarshal account: %v", err.Error())
}
return
}
func (s Service) writeAccount(a *proto.Account) (err error) {
// leave only the group id
s.deflateMemberOf(a)
var bytes []byte
if bytes, err = json.Marshal(a); err != nil {
return merrors.InternalServerError(s.id, "could not marshal account: %v", err.Error())
}
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", a.Id)
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
return merrors.InternalServerError(s.id, "could not write account: %v", err.Error())
}
return
}
func (s Service) expandMemberOf(a *proto.Account) {
if a == nil {
return
@@ -118,7 +63,7 @@ func (s Service) expandMemberOf(a *proto.Account) {
for i := range a.MemberOf {
g := &proto.Group{}
// TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index?
if err := s.loadGroup(a.MemberOf[i].Id, g); err == nil {
if err := s.repo.LoadGroup(context.Background(), a.MemberOf[i].Id, g); err == nil {
g.Members = nil // always hide members when expanding
expanded = append(expanded, g)
} else {
@@ -129,23 +74,6 @@ func (s Service) expandMemberOf(a *proto.Account) {
a.MemberOf = expanded
}
// deflateMemberOf replaces the groups of a user with an instance that only contains the id
func (s Service) deflateMemberOf(a *proto.Account) {
if a == nil {
return
}
deflated := []*proto.Group{}
for i := range a.MemberOf {
if a.MemberOf[i].Id != "" {
deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id})
} else {
// TODO fetch and use an id when group only has a name but no id
s.log.Error().Str("id", a.Id).Interface("group", a.MemberOf[i]).Msg("resolving groups by name is not implemented yet")
}
}
a.MemberOf = deflated
}
func (s Service) passwordIsValid(hash string, pwd string) (ok bool) {
defer func() {
if r := recover(); r != nil {
@@ -173,6 +101,36 @@ func (s Service) hasAccountManagementPermissions(ctx context.Context) bool {
return s.RoleManager.FindPermissionByID(ctx, roleIDs, AccountManagementPermissionID) != nil
}
// serviceUserToIndex temporarily adds a service user to the index, which is supposed to be removed before the lock on the handler function is released
func (s Service) serviceUserToIndex() (teardownServiceUser func()) {
if s.Config.ServiceUser.Username != "" && s.Config.ServiceUser.UUID != "" {
err := s.index.Index(s.Config.ServiceUser.UUID, &proto.BleveAccount{
BleveType: "account",
Account: s.getInMemoryServiceUser(),
})
if err != nil {
s.log.Logger.Err(err).Msg("service user was configured but failed to be added to the index")
} else {
return func() {
_ = s.index.Delete(s.Config.ServiceUser.UUID)
}
}
}
return func() {}
}
func (s Service) getInMemoryServiceUser() proto.Account {
return proto.Account{
AccountEnabled: true,
Id: s.Config.ServiceUser.UUID,
PreferredName: s.Config.ServiceUser.Username,
OnPremisesSamAccountName: s.Config.ServiceUser.Username,
DisplayName: s.Config.ServiceUser.Username,
UidNumber: s.Config.ServiceUser.UID,
GidNumber: s.Config.ServiceUser.GID,
}
}
// ListAccounts implements the AccountsServiceHandler interface
// the query contains account properties
func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest, out *proto.ListAccountsResponse) (err error) {
@@ -184,6 +142,9 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
defer accLock.Unlock()
var password string
teardownServiceUser := s.serviceUserToIndex()
defer teardownServiceUser()
// check if this looks like an auth request
match := authQuery.FindStringSubmatch(in.Query)
if len(match) == 3 {
@@ -233,7 +194,10 @@ func (s Service) ListAccounts(ctx context.Context, in *proto.ListAccountsRequest
for _, hit := range searchResult.Hits {
a := &proto.Account{}
if err = s.loadAccount(hit.ID, a); err != nil {
if hit.ID == s.Config.ServiceUser.UUID {
acc := s.getInMemoryServiceUser()
a = &acc
} else if err = s.repo.LoadAccount(ctx, hit.ID, a); err != nil {
s.log.Error().Err(err).Str("account", hit.ID).Msg("could not load account, skipping")
continue
}
@@ -281,9 +245,13 @@ func (s Service) GetAccount(ctx context.Context, in *proto.GetAccountRequest, ou
return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error())
}
if err = s.loadAccount(id, out); err != nil {
if err = s.repo.LoadAccount(ctx, id, out); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "account not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
return merrors.InternalServerError(s.id, "could not load account: %v", err.Error())
}
s.debugLogAccount(out).Msg("found account")
@@ -346,10 +314,10 @@ func (s Service) CreateAccount(ctx context.Context, in *proto.CreateAccountReque
// TODO groups should be ignored during create, use groups.AddMember? return error?
// write and index account - note: don't do anything else in between!
if err = s.writeAccount(acc); err != nil {
if err = s.repo.WriteAccount(ctx, acc); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not persist new account")
s.debugLogAccount(acc).Msg("could not persist new account")
return
return merrors.InternalServerError(s.id, "could not persist new account: %v", err.Error())
}
if err = s.indexAccount(acc.Id); err != nil {
return merrors.InternalServerError(s.id, "could not index new account: %v", err.Error())
@@ -407,9 +375,14 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)
if err = s.loadAccount(id, out); err != nil {
if err = s.repo.LoadAccount(ctx, id, out); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "account not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
return merrors.InternalServerError(s.id, "could not load account: %v", err.Error())
}
t := time.Now()
@@ -461,9 +434,9 @@ func (s Service) UpdateAccount(ctx context.Context, in *proto.UpdateAccountReque
out.ExternalUserStateChangeDateTime = tsnow
}
if err = s.writeAccount(out); err != nil {
if err = s.repo.WriteAccount(ctx, out); err != nil {
s.log.Error().Err(err).Str("id", out.Id).Msg("could not persist updated account")
return
return merrors.InternalServerError(s.id, "could not persist updated account: %v", err.Error())
}
if err = s.indexAccount(id); err != nil {
@@ -510,12 +483,15 @@ func (s Service) DeleteAccount(ctx context.Context, in *proto.DeleteAccountReque
if id, err = cleanupID(in.Id); err != nil {
return merrors.InternalServerError(s.id, "could not clean up account id: %v", err.Error())
}
path := filepath.Join(s.Config.Server.AccountsDataPath, "accounts", id)
a := &proto.Account{}
if err = s.loadAccount(id, a); err != nil {
if err = s.repo.LoadAccount(ctx, id, a); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "account not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
return merrors.InternalServerError(s.id, "could not load account: %v", err.Error())
}
// delete member relationship in groups
@@ -529,13 +505,17 @@ func (s Service) DeleteAccount(ctx context.Context, in *proto.DeleteAccountReque
}
}
if err = os.Remove(path); err != nil {
s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account")
if err = s.repo.DeleteAccount(ctx, id); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "account not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", id).Str("accountId", id).Msg("could not remove account")
return merrors.InternalServerError(s.id, "could not remove account: %v", err.Error())
}
if err = s.index.Delete(id); err != nil {
s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account from index")
s.log.Error().Err(err).Str("id", id).Str("accountId", id).Msg("could not remove account from index")
return merrors.InternalServerError(s.id, "could not remove account from index: %v", err.Error())
}
accounts/pkg/service/v0/accounts_permission_test.go
+1
View File
@@ -34,6 +34,7 @@ func init() {
cfg := config.New()
cfg.Server.Name = "accounts"
cfg.Server.AccountsDataPath = dataPath
cfg.Repo.Disk.Path = dataPath
logger := olog.NewLogger(olog.Color(true), olog.Pretty(true))
roleServiceMock = buildRoleServiceMock()
roleManager := roles.NewManager(
accounts/pkg/service/v0/groups.go
+57 -105
View File
@@ -2,11 +2,8 @@ package service
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"github.com/owncloud/ocis/accounts/pkg/storage"
"path/filepath"
"sync"
"github.com/CiscoM31/godata"
"github.com/blevesearch/bleve"
@@ -17,36 +14,11 @@ import (
"github.com/owncloud/ocis/accounts/pkg/provider"
)
// accLock mutually exclude readers from writers on group files
var groupLock sync.Mutex
func (s Service) indexGroups(path string) (err error) {
var f *os.File
if f, err = os.Open(path); err != nil {
s.log.Error().Err(err).Str("dir", path).Msg("could not open groups folder")
return
}
list, err := f.Readdir(-1)
f.Close()
if err != nil {
s.log.Error().Err(err).Str("dir", path).Msg("could not list groups folder")
return
}
for _, file := range list {
err = s.indexGroup(file.Name())
if err != nil {
s.log.Error().Err(err).Str("file", file.Name()).Msg("could not index account")
}
}
return
}
func (s Service) indexGroup(id string) error {
g := &proto.BleveGroup{
BleveType: "group",
}
if err := s.loadGroup(id, &g.Group); err != nil {
if err := s.repo.LoadGroup(context.Background(), id, &g.Group); err != nil {
s.log.Error().Err(err).Str("group", id).Msg("could not load group")
return err
}
@@ -58,43 +30,6 @@ func (s Service) indexGroup(id string) error {
return nil
}
func (s Service) loadGroup(id string, g *proto.Group) (err error) {
path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id)
groupLock.Lock()
defer groupLock.Unlock()
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
return merrors.NotFound(s.id, "could not read group: %v", err.Error())
}
if err = json.Unmarshal(data, g); err != nil {
return merrors.InternalServerError(s.id, "could not unmarshal group: %v", err.Error())
}
return
}
func (s Service) writeGroup(g *proto.Group) (err error) {
// leave only the member id
s.deflateMembers(g)
var bytes []byte
if bytes, err = json.Marshal(g); err != nil {
return merrors.InternalServerError(s.id, "could not marshal group: %v", err.Error())
}
path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", g.Id)
groupLock.Lock()
defer groupLock.Unlock()
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
return merrors.InternalServerError(s.id, "could not write group: %v", err.Error())
}
return
}
func (s Service) expandMembers(g *proto.Group) {
if g == nil {
return
@@ -103,7 +38,7 @@ func (s Service) expandMembers(g *proto.Group) {
for i := range g.Members {
// TODO resolve by name, when a create or update is issued they may not have an id? fall back to searching the group id in the index?
a := &proto.Account{}
if err := s.loadAccount(g.Members[i].Id, a); err == nil {
if err := s.repo.LoadAccount(context.Background(), g.Members[i].Id, a); err == nil {
expanded = append(expanded, a)
} else {
// log errors but continue execution for now
@@ -173,7 +108,7 @@ func (s Service) ListGroups(c context.Context, in *proto.ListGroupsRequest, out
for _, hit := range searchResult.Hits {
g := &proto.Group{}
if err = s.loadGroup(hit.ID, g); err != nil {
if err = s.repo.LoadGroup(c, hit.ID, g); err != nil {
s.log.Error().Err(err).Str("group", hit.ID).Msg("could not load group, skipping")
continue
}
@@ -196,9 +131,13 @@ func (s Service) GetGroup(c context.Context, in *proto.GetGroupRequest, out *pro
return merrors.InternalServerError(s.id, "could not clean up group id: %v", err.Error())
}
if err = s.loadGroup(id, out); err != nil {
if err = s.repo.LoadGroup(c, id, out); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "group not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", id).Msg("could not load group")
return
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
s.log.Debug().Interface("group", out).Msg("found group")
@@ -226,9 +165,9 @@ func (s Service) CreateGroup(c context.Context, in *proto.CreateGroupRequest, ou
// extract member id
s.deflateMembers(in.Group)
if err = s.writeGroup(in.Group); err != nil {
if err = s.repo.WriteGroup(c, in.Group); err != nil {
s.log.Error().Err(err).Interface("group", in.Group).Msg("could not persist new group")
return
return merrors.InternalServerError(s.id, "could not persist new group: %v", err.Error())
}
if err = s.indexGroup(id); err != nil {
@@ -252,9 +191,11 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou
path := filepath.Join(s.Config.Server.AccountsDataPath, "groups", id)
g := &proto.Group{}
if err = s.loadGroup(id, g); err != nil {
s.log.Error().Err(err).Str("id", id).Msg("could not load account")
return
if err = s.repo.LoadGroup(c, id, g); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "group not found: %v", err.Error())
}
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
// delete memberof relationship in users
@@ -267,9 +208,13 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou
s.log.Error().Err(err).Str("groupid", id).Str("accountid", g.Members[i].Id).Msg("could not remove account memberof, skipping")
}
}
if err = os.Remove(path); err != nil {
s.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group")
return merrors.InternalServerError(s.id, "could not remove group: %v", err.Error())
if err = s.repo.DeleteGroup(c, id); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "group not found: %v", err.Error())
}
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
if err = s.index.Delete(id); err != nil {
@@ -283,7 +228,6 @@ func (s Service) DeleteGroup(c context.Context, in *proto.DeleteGroupRequest, ou
// AddMember implements the GroupsServiceHandler interface
func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *proto.Group) (err error) {
// cleanup ids
var groupID string
if groupID, err = cleanupID(in.GroupId); err != nil {
@@ -297,15 +241,19 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p
// load structs
a := &proto.Account{}
if err = s.loadAccount(accountID, a); err != nil {
s.log.Error().Err(err).Str("id", accountID).Msg("could not load account")
return
if err = s.repo.LoadAccount(c, accountID, a); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "group not found: %v", err.Error())
}
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
if err = s.repo.LoadGroup(c, groupID, g); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "could not load group: %v", err.Error())
}
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
// check if we need to add the account to the group
@@ -331,13 +279,11 @@ func (s Service) AddMember(c context.Context, in *proto.AddMemberRequest, out *p
a.MemberOf = append(a.MemberOf, g)
}
if err = s.writeAccount(a); err != nil {
s.log.Error().Err(err).Interface("account", a).Msg("could not persist account")
return
if err = s.repo.WriteAccount(c, a); err != nil {
return merrors.InternalServerError(s.id, "could not persist account: %v", err.Error())
}
if err = s.writeGroup(g); err != nil {
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
if err = s.repo.WriteGroup(c, g); err != nil {
return merrors.InternalServerError(s.id, "could not persist group: %v", err.Error())
}
// FIXME update index!
// TODO rollback changes when only one of them failed?
@@ -362,15 +308,21 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,
// load structs
a := &proto.Account{}
if err = s.loadAccount(accountID, a); err != nil {
if err = s.repo.LoadAccount(c, accountID, a); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "could not load account: %v", err.Error())
}
s.log.Error().Err(err).Str("id", accountID).Msg("could not load account")
return
return merrors.InternalServerError(s.id, "could not load account: %v", err.Error())
}
g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
if err = s.repo.LoadGroup(c, groupID, g); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "could not load group: %v", err.Error())
}
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
//remove the account from the group if it exists
@@ -391,13 +343,13 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,
}
a.MemberOf = newGroups
if err = s.writeAccount(a); err != nil {
if err = s.repo.WriteAccount(c, a); err != nil {
s.log.Error().Err(err).Interface("account", a).Msg("could not persist account")
return
return merrors.InternalServerError(s.id, "could not persist account: %v", err.Error())
}
if err = s.writeGroup(g); err != nil {
if err = s.repo.WriteGroup(c, g); err != nil {
s.log.Error().Err(err).Interface("group", g).Msg("could not persist group")
return
return merrors.InternalServerError(s.id, "could not persist group: %v", err.Error())
}
// FIXME update index!
// TODO rollback changes when only one of them failed?
@@ -408,7 +360,6 @@ func (s Service) RemoveMember(c context.Context, in *proto.RemoveMemberRequest,
// ListMembers implements the GroupsServiceHandler interface
func (s Service) ListMembers(c context.Context, in *proto.ListMembersRequest, out *proto.ListMembersResponse) (err error) {
// cleanup ids
var groupID string
if groupID, err = cleanupID(in.Id); err != nil {
@@ -416,16 +367,17 @@ func (s Service) ListMembers(c context.Context, in *proto.ListMembersRequest, ou
}
g := &proto.Group{}
if err = s.loadGroup(groupID, g); err != nil {
if err = s.repo.LoadGroup(c, groupID, g); err != nil {
if storage.IsNotFoundErr(err) {
return merrors.NotFound(s.id, "group not found: %v", err.Error())
}
s.log.Error().Err(err).Str("id", groupID).Msg("could not load group")
return
return merrors.InternalServerError(s.id, "could not load group: %v", err.Error())
}
// TODO only expand accounts if requested
// if in.FieldMask ...
s.expandMembers(g)
out.Members = g.Members
return
}
accounts/pkg/service/v0/service.go
+274 -301
View File
@@ -2,10 +2,8 @@ package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"github.com/owncloud/ocis/accounts/pkg/storage"
"os"
"path/filepath"
"strings"
@@ -56,331 +54,292 @@ func New(opts ...Option) (s *Service, err error) {
Config: cfg,
RoleService: roleService,
RoleManager: roleManager,
repo: createMetadataStorage(cfg, logger),
}
// build an index
if s.index, err = s.buildIndex(); err != nil {
return nil, err
}
// create default accounts
accountsDir := filepath.Join(cfg.Server.AccountsDataPath, "accounts")
if err = s.createDefaultAccounts(accountsDir); err != nil {
return nil, err
}
if err = s.indexAccounts(accountsDir); err != nil {
if err = s.createDefaultAccounts(); err != nil {
return nil, err
}
// create default groups
groupsDir := filepath.Join(cfg.Server.AccountsDataPath, "groups")
if err = s.createDefaultGroups(groupsDir); err != nil {
if err = s.createDefaultGroups(); err != nil {
return nil, err
}
if err = s.indexGroups(groupsDir); err != nil {
return nil, err
}
// TODO watch folders for new records
return
}
func (s Service) buildIndex() (index bleve.Index, err error) {
indexMapping := bleve.NewIndexMapping()
// keep all symbols in terms to allow exact maching, eg. emails
indexMapping.DefaultAnalyzer = keyword.Name
// TODO don't bother to store fields as we will load the account from disk
// Reusable mapping for text
standardTextFieldMapping := bleve.NewTextFieldMapping()
standardTextFieldMapping.Analyzer = standard.Name
standardTextFieldMapping.Store = false
// Reusable mapping for text, uses english stop word removal
simpleTextFieldMapping := bleve.NewTextFieldMapping()
simpleTextFieldMapping.Analyzer = simple.Name
simpleTextFieldMapping.Store = false
// Reusable mapping for keyword text
keywordFieldMapping := bleve.NewTextFieldMapping()
keywordFieldMapping.Analyzer = keyword.Name
keywordFieldMapping.Store = false
// Reusable mapping for lowercase text
err = indexMapping.AddCustomAnalyzer("lowercase",
map[string]interface{}{
"type": custom.Name,
"tokenizer": unicode.Name,
"token_filters": []string{
lowercase.Name,
},
})
if err != nil {
return
}
lowercaseTextFieldMapping := bleve.NewTextFieldMapping()
lowercaseTextFieldMapping.Analyzer = "lowercase"
lowercaseTextFieldMapping.Store = true
// accounts
accountMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("account", accountMapping)
// Text
accountMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping)
accountMapping.AddFieldMappingsAt("description", standardTextFieldMapping)
// Lowercase
accountMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping)
accountMapping.AddFieldMappingsAt("preferred_name", lowercaseTextFieldMapping)
// Keywords
accountMapping.AddFieldMappingsAt("mail", keywordFieldMapping)
// groups
groupMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("group", groupMapping)
// Text
groupMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping)
groupMapping.AddFieldMappingsAt("description", standardTextFieldMapping)
// Lowercase
groupMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping)
// Tell blevesearch how to determine the type of the structs that are indexed.
// The referenced field needs to match the struct field exactly and it must be public.
// See pkg/proto/v0/bleve.go how we wrap the generated Account and Group to add a
// BleveType property which is indexed as `bleve_type` so we can also distinguish the
// documents in the index by querying for that property.
indexMapping.TypeField = "BleveType"
indexDir := filepath.Join(s.Config.Server.AccountsDataPath, "index.bleve")
// for now recreate index on every start
if err = os.RemoveAll(indexDir); err != nil {
return
}
if index, err = bleve.New(indexDir, indexMapping); err != nil {
return nil, err
if index, err = bleve.Open(indexDir); err != nil {
if err != bleve.ErrorIndexPathDoesNotExist {
s.log.Error().Err(err).Msg("failed to read index")
return
}
indexMapping := bleve.NewIndexMapping()
// keep all symbols in terms to allow exact maching, eg. emails
indexMapping.DefaultAnalyzer = keyword.Name
// TODO don't bother to store fields as we will load the account from disk
// Reusable mapping for text
standardTextFieldMapping := bleve.NewTextFieldMapping()
standardTextFieldMapping.Analyzer = standard.Name
standardTextFieldMapping.Store = false
// Reusable mapping for text, uses english stop word removal
simpleTextFieldMapping := bleve.NewTextFieldMapping()
simpleTextFieldMapping.Analyzer = simple.Name
simpleTextFieldMapping.Store = false
// Reusable mapping for keyword text
keywordFieldMapping := bleve.NewTextFieldMapping()
keywordFieldMapping.Analyzer = keyword.Name
keywordFieldMapping.Store = false
// Reusable mapping for lowercase text
err = indexMapping.AddCustomAnalyzer("lowercase",
map[string]interface{}{
"type": custom.Name,
"tokenizer": unicode.Name,
"token_filters": []string{
lowercase.Name,
},
})
if err != nil {
return
}
lowercaseTextFieldMapping := bleve.NewTextFieldMapping()
lowercaseTextFieldMapping.Analyzer = "lowercase"
lowercaseTextFieldMapping.Store = true
// accounts
accountMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("account", accountMapping)
// Text
accountMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping)
accountMapping.AddFieldMappingsAt("description", standardTextFieldMapping)
// Lowercase
accountMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping)
accountMapping.AddFieldMappingsAt("preferred_name", lowercaseTextFieldMapping)
// Keywords
accountMapping.AddFieldMappingsAt("mail", keywordFieldMapping)
// groups
groupMapping := bleve.NewDocumentMapping()
indexMapping.AddDocumentMapping("group", groupMapping)
// Text
groupMapping.AddFieldMappingsAt("display_name", standardTextFieldMapping)
groupMapping.AddFieldMappingsAt("description", standardTextFieldMapping)
// Lowercase
groupMapping.AddFieldMappingsAt("on_premises_sam_account_name", lowercaseTextFieldMapping)
// Tell blevesearch how to determine the type of the structs that are indexed.
// The referenced field needs to match the struct field exactly and it must be public.
// See pkg/proto/v0/bleve.go how we wrap the generated Account and Group to add a
// BleveType property which is indexed as `bleve_type` so we can also distinguish the
// documents in the index by querying for that property.
indexMapping.TypeField = "BleveType"
// for now recreate index on every start
if err = os.RemoveAll(indexDir); err != nil {
return
}
if index, err = bleve.New(indexDir, indexMapping); err != nil {
return nil, err
}
}
return
}
func (s Service) createDefaultAccounts(accountsDir string) (err error) {
// check if accounts exist
var fi os.FileInfo
if fi, err = os.Stat(accountsDir); err != nil {
if os.IsNotExist(err) {
// create accounts directory
if err = os.MkdirAll(accountsDir, 0700); err != nil {
return
}
// create default accounts
accounts := []proto.Account{
{
Id: "4c510ada-c86b-4815-8820-42cdf82c3d51",
PreferredName: "einstein",
OnPremisesSamAccountName: "einstein",
Mail: "einstein@example.org",
DisplayName: "Albert Einstein",
UidNumber: 20000,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=35210$sa1u5Pmfo4cr23Vw$RJNGElaDB1D3xorWkfTEGm2Ko.o2QL3E0cimKx23MNxVWVFSkUUeRoC7FqC4RzYDNQBD6cKzovTEaDD.8TDkD.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0"}, // sailing-lovers
{Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f"}, // violin-haters
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
{
Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c",
PreferredName: "marie",
OnPremisesSamAccountName: "marie",
Mail: "marie@example.org",
DisplayName: "Marie Curie",
UidNumber: 20001,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=81434$sa1u5Pmfo4cr23Vw$W78cyL884GmuvDpxYPvSRBVzEj02T5QhTTcI8Dv4IKvMooDFGv4bwaWMkH9HfJ0wgpEBW7Lp.4Cad0xE/MYSg1",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a"}, // radium-lovers
{Id: "cedc21aa-4072-4614-8676-fa9165f598ff"}, // polonium-lovers
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
{
Id: "932b4540-8d16-481e-8ef4-588e4b6b151c",
PreferredName: "richard",
OnPremisesSamAccountName: "richard",
Mail: "richard@example.org",
DisplayName: "Richard Feynman",
UidNumber: 20002,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=5524$sa1u5Pmfo4cr23Vw$58bQVL/JeUlwM0RY21YKAFMvKvwKLLysGllYXox.vwKT5dHMwdzJjCxwTDMnB2o2pwexC8o/iOXyP2zrhALS40",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a"}, // quantum-lovers
{Id: "167cbee2-0518-455a-bfb2-031fe0621e5d"}, // philosophy-haters
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
// admin user(s)
{
Id: "058bff95-6708-4fe5-91e4-9ea3d377588b",
PreferredName: "moss",
OnPremisesSamAccountName: "moss",
Mail: "moss@example.org",
DisplayName: "Maurice Moss",
UidNumber: 20003,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=47068$lhw6odzXW0LTk/ao$GgxS.pIgP8jawLJBAiyNor2FrWzrULF95PwspRkli2W3VF.4HEwTYlQfRXbNQBMjNCEcEYlgZo3a.kRz2k2N0/",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
},
},
// technical users for kopano and reva
{
Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf",
PreferredName: "konnectd",
OnPremisesSamAccountName: "konnectd",
Mail: "idp@example.org",
DisplayName: "Kopano Konnectd",
UidNumber: 10000,
GidNumber: 15000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=9746$sa1u5Pmfo4cr23Vw$2hnwpkTvUkWX0v6mh8Aw1pbzEXa9EUJzmrey4g2W/8arwWCwhteqU//3aWnA3S0d5T21fOKYteoqlsN1IbTcN.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers
},
},
{
Id: "bc596f3c-c955-4328-80a0-60d018b4ad57",
PreferredName: "reva",
OnPremisesSamAccountName: "reva",
Mail: "storage@example.org",
DisplayName: "Reva Inter Operability Platform",
UidNumber: 10001,
GidNumber: 15000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=91087$sa1u5Pmfo4cr23Vw$wPC3BbMTbP/ytlo0p.f99zJifyO70AUCdKIK9hkhwutBKGCirLmZs/MsWAG6xHjVvmnmHN5NoON7FUGv5pPaN.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers
},
},
}
for i := range accounts {
// create account on disk
var bytes []byte
if bytes, err = json.Marshal(&accounts[i]); err != nil {
s.log.Error().Err(err).Interface("account", &accounts[i]).Msg("could not marshal default account")
return
}
path := filepath.Join(accountsDir, accounts[i].Id)
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
accounts[i].PasswordProfile.Password = "***REMOVED***"
s.log.Error().Err(err).Str("path", path).Interface("account", &accounts[i]).Msg("could not persist default account")
return
}
}
// set role for admin users and regular users
assignRoleToUser("058bff95-6708-4fe5-91e4-9ea3d377588b", settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log)
for _, accountID := range []string{
"058bff95-6708-4fe5-91e4-9ea3d377588b", //moss
} {
assignRoleToUser(accountID, settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log)
}
for _, accountID := range []string{
"4c510ada-c86b-4815-8820-42cdf82c3d51", //einstein
"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", //marie
"932b4540-8d16-481e-8ef4-588e4b6b151c", //richard
} {
assignRoleToUser(accountID, settings_svc.BundleUUIDRoleUser, s.RoleService, s.log)
}
func (s Service) createDefaultAccounts() (err error) {
accounts := []proto.Account{
{
Id: "4c510ada-c86b-4815-8820-42cdf82c3d51",
PreferredName: "einstein",
OnPremisesSamAccountName: "einstein",
Mail: "einstein@example.org",
DisplayName: "Albert Einstein",
UidNumber: 20000,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=35210$sa1u5Pmfo4cr23Vw$RJNGElaDB1D3xorWkfTEGm2Ko.o2QL3E0cimKx23MNxVWVFSkUUeRoC7FqC4RzYDNQBD6cKzovTEaDD.8TDkD.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0"}, // sailing-lovers
{Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f"}, // violin-haters
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
{
Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c",
PreferredName: "marie",
OnPremisesSamAccountName: "marie",
Mail: "marie@example.org",
DisplayName: "Marie Curie",
UidNumber: 20001,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=81434$sa1u5Pmfo4cr23Vw$W78cyL884GmuvDpxYPvSRBVzEj02T5QhTTcI8Dv4IKvMooDFGv4bwaWMkH9HfJ0wgpEBW7Lp.4Cad0xE/MYSg1",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a"}, // radium-lovers
{Id: "cedc21aa-4072-4614-8676-fa9165f598ff"}, // polonium-lovers
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
{
Id: "932b4540-8d16-481e-8ef4-588e4b6b151c",
PreferredName: "richard",
OnPremisesSamAccountName: "richard",
Mail: "richard@example.org",
DisplayName: "Richard Feynman",
UidNumber: 20002,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=5524$sa1u5Pmfo4cr23Vw$58bQVL/JeUlwM0RY21YKAFMvKvwKLLysGllYXox.vwKT5dHMwdzJjCxwTDMnB2o2pwexC8o/iOXyP2zrhALS40",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
{Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a"}, // quantum-lovers
{Id: "167cbee2-0518-455a-bfb2-031fe0621e5d"}, // philosophy-haters
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e"}, // physics-lovers
},
},
// admin user(s)
{
Id: "058bff95-6708-4fe5-91e4-9ea3d377588b",
PreferredName: "moss",
OnPremisesSamAccountName: "moss",
Mail: "moss@example.org",
DisplayName: "Maurice Moss",
UidNumber: 20003,
GidNumber: 30000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=47068$lhw6odzXW0LTk/ao$GgxS.pIgP8jawLJBAiyNor2FrWzrULF95PwspRkli2W3VF.4HEwTYlQfRXbNQBMjNCEcEYlgZo3a.kRz2k2N0/",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa"}, // users
},
},
// technical users for kopano and reva
{
Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf",
PreferredName: "konnectd",
OnPremisesSamAccountName: "konnectd",
Mail: "idp@example.org",
DisplayName: "Kopano Konnectd",
UidNumber: 10000,
GidNumber: 15000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=9746$sa1u5Pmfo4cr23Vw$2hnwpkTvUkWX0v6mh8Aw1pbzEXa9EUJzmrey4g2W/8arwWCwhteqU//3aWnA3S0d5T21fOKYteoqlsN1IbTcN.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers
},
},
{
Id: "bc596f3c-c955-4328-80a0-60d018b4ad57",
PreferredName: "reva",
OnPremisesSamAccountName: "reva",
Mail: "storage@example.org",
DisplayName: "Reva Inter Operability Platform",
UidNumber: 10001,
GidNumber: 15000,
PasswordProfile: &proto.PasswordProfile{
Password: "$6$rounds=91087$sa1u5Pmfo4cr23Vw$wPC3BbMTbP/ytlo0p.f99zJifyO70AUCdKIK9hkhwutBKGCirLmZs/MsWAG6xHjVvmnmHN5NoON7FUGv5pPaN.",
},
AccountEnabled: true,
MemberOf: []*proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0"}, // sysusers
},
},
}
for i := range accounts {
if err := s.repo.WriteAccount(context.Background(), &accounts[i]); err != nil {
return err
}
} else if !fi.IsDir() {
return fmt.Errorf("%s is not a directory", accountsDir)
if err := s.indexAccount(accounts[i].Id); err != nil {
return err
}
}
// set role for admin users and regular users
assignRoleToUser("058bff95-6708-4fe5-91e4-9ea3d377588b", settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log)
for _, accountID := range []string{
"058bff95-6708-4fe5-91e4-9ea3d377588b", //moss
} {
assignRoleToUser(accountID, settings_svc.BundleUUIDRoleAdmin, s.RoleService, s.log)
}
for _, accountID := range []string{
"4c510ada-c86b-4815-8820-42cdf82c3d51", //einstein
"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", //marie
"932b4540-8d16-481e-8ef4-588e4b6b151c", //richard
} {
assignRoleToUser(accountID, settings_svc.BundleUUIDRoleUser, s.RoleService, s.log)
}
return nil
}
func (s Service) createDefaultGroups(groupsDir string) (err error) {
// check if groups exist
var fi os.FileInfo
if fi, err = os.Stat(groupsDir); err != nil {
if os.IsNotExist(err) {
// create accounts directory
if err = os.MkdirAll(groupsDir, 0700); err != nil {
return
}
// create default accounts
groups := []proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0", GidNumber: 15000, OnPremisesSamAccountName: "sysusers", DisplayName: "Technical users", Description: "A group for technical users. They should not show up in sharing dialogs.", Members: []*proto.Account{
{Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf"}, // konnectd
{Id: "bc596f3c-c955-4328-80a0-60d018b4ad57"}, // reva
}},
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa", GidNumber: 30000, OnPremisesSamAccountName: "users", DisplayName: "Users", Description: "A group every normal user belongs to.", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0", GidNumber: 30001, OnPremisesSamAccountName: "sailing-lovers", DisplayName: "Sailing lovers", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
}},
{Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f", GidNumber: 30002, OnPremisesSamAccountName: "violin-haters", DisplayName: "Violin haters", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
}},
{Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a", GidNumber: 30003, OnPremisesSamAccountName: "radium-lovers", DisplayName: "Radium lovers", Members: []*proto.Account{
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
}},
{Id: "cedc21aa-4072-4614-8676-fa9165f598ff", GidNumber: 30004, OnPremisesSamAccountName: "polonium-lovers", DisplayName: "Polonium lovers", Members: []*proto.Account{
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
}},
{Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a", GidNumber: 30005, OnPremisesSamAccountName: "quantum-lovers", DisplayName: "Quantum lovers", Members: []*proto.Account{
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "167cbee2-0518-455a-bfb2-031fe0621e5d", GidNumber: 30006, OnPremisesSamAccountName: "philosophy-haters", DisplayName: "Philosophy haters", Members: []*proto.Account{
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e", GidNumber: 30007, OnPremisesSamAccountName: "physics-lovers", DisplayName: "Physics lovers", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
}
for i := range groups {
var bytes []byte
if bytes, err = json.Marshal(&groups[i]); err != nil {
s.log.Error().Err(err).Interface("group", &groups[i]).Msg("could not marshal default group")
return
}
path := filepath.Join(groupsDir, groups[i].Id)
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
s.log.Error().Err(err).Str("path", path).Interface("group", &groups[i]).Msg("could not persist default group")
return
}
}
func (s Service) createDefaultGroups() (err error) {
groups := []proto.Group{
{Id: "34f38767-c937-4eb6-b847-1c175829a2a0", GidNumber: 15000, OnPremisesSamAccountName: "sysusers", DisplayName: "Technical users", Description: "A group for technical users. They should not show up in sharing dialogs.", Members: []*proto.Account{
{Id: "820ba2a1-3f54-4538-80a4-2d73007e30bf"}, // konnectd
{Id: "bc596f3c-c955-4328-80a0-60d018b4ad57"}, // reva
}},
{Id: "509a9dcd-bb37-4f4f-a01a-19dca27d9cfa", GidNumber: 30000, OnPremisesSamAccountName: "users", DisplayName: "Users", Description: "A group every normal user belongs to.", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "6040aa17-9c64-4fef-9bd0-77234d71bad0", GidNumber: 30001, OnPremisesSamAccountName: "sailing-lovers", DisplayName: "Sailing lovers", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
}},
{Id: "dd58e5ec-842e-498b-8800-61f2ec6f911f", GidNumber: 30002, OnPremisesSamAccountName: "violin-haters", DisplayName: "Violin haters", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
}},
{Id: "7b87fd49-286e-4a5f-bafd-c535d5dd997a", GidNumber: 30003, OnPremisesSamAccountName: "radium-lovers", DisplayName: "Radium lovers", Members: []*proto.Account{
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
}},
{Id: "cedc21aa-4072-4614-8676-fa9165f598ff", GidNumber: 30004, OnPremisesSamAccountName: "polonium-lovers", DisplayName: "Polonium lovers", Members: []*proto.Account{
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
}},
{Id: "a1726108-01f8-4c30-88df-2b1a9d1cba1a", GidNumber: 30005, OnPremisesSamAccountName: "quantum-lovers", DisplayName: "Quantum lovers", Members: []*proto.Account{
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "167cbee2-0518-455a-bfb2-031fe0621e5d", GidNumber: 30006, OnPremisesSamAccountName: "philosophy-haters", DisplayName: "Philosophy haters", Members: []*proto.Account{
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
{Id: "262982c1-2362-4afa-bfdf-8cbfef64a06e", GidNumber: 30007, OnPremisesSamAccountName: "physics-lovers", DisplayName: "Physics lovers", Members: []*proto.Account{
{Id: "4c510ada-c86b-4815-8820-42cdf82c3d51"}, // einstein
{Id: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, // marie
{Id: "932b4540-8d16-481e-8ef4-588e4b6b151c"}, // feynman
}},
}
for i := range groups {
if err := s.repo.WriteGroup(context.Background(), &groups[i]); err != nil {
return err
}
if err := s.indexGroup(groups[i].Id); err != nil {
return err
}
} else if !fi.IsDir() {
return fmt.Errorf("%s is not a directory", groupsDir)
}
return nil
}
@@ -397,6 +356,19 @@ func assignRoleToUser(accountID, roleID string, rs settings.RoleService, logger
return true
}
func createMetadataStorage(cfg *config.Config, logger log.Logger) storage.Repo {
// for now we detect the used storage implementation based on which storage is configured
// the config with defaults needs to be checked last
if cfg.Repo.Disk.Path != "" {
return storage.NewDiskRepo(cfg, logger)
}
repo, err := storage.NewCS3Repo(cfg)
if err != nil {
logger.Fatal().Err(err).Msg("cs3 storage was configured but failed to start")
}
return repo
}
// Service implements the AccountsServiceHandler interface
type Service struct {
id string
@@ -405,6 +377,7 @@ type Service struct {
index bleve.Index
RoleService settings.RoleService
RoleManager *roles.Manager
repo storage.Repo
}
func cleanupID(id string) (string, error) {
accounts/pkg/storage/cs3.go
+287
View File
@@ -0,0 +1,287 @@
package storage
import (
"bytes"
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"path"
"strings"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/token"
"github.com/cs3org/reva/pkg/token/manager/jwt"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
"google.golang.org/grpc/metadata"
)
// CS3Repo provides a cs3 implementation of the Repo interface
type CS3Repo struct {
cfg *config.Config
tm token.Manager
storageProvider provider.ProviderAPIClient
dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
}
// NewCS3Repo creates a new cs3 repo
func NewCS3Repo(cfg *config.Config) (Repo, error) {
tokenManager, err := jwt.New(map[string]interface{}{
"secret": cfg.TokenManager.JWTSecret,
})
if err != nil {
return nil, err
}
client, err := pool.GetStorageProviderServiceClient(cfg.Repo.CS3.ProviderAddr)
if err != nil {
return nil, err
}
return CS3Repo{
cfg: cfg,
tm: tokenManager,
storageProvider: client,
dataProvider: dataProviderClient{
client: http.Client{
Transport: http.DefaultTransport,
},
},
}, nil
}
// WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id).
func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil {
return err
}
var by []byte
if by, err = json.Marshal(a); err != nil {
return err
}
_, err = r.dataProvider.put(r.accountURL(a.Id), bytes.NewReader(by), t)
return err
}
// LoadAccount loads an account via cs3 by id and writes it to the provided account
func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
resp, err := r.dataProvider.get(r.accountURL(id), t)
if err != nil {
return err
}
if resp.StatusCode == http.StatusNotFound {
return &notFoundErr{"account", id}
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if err = resp.Body.Close(); err != nil {
return err
}
return json.Unmarshal(b, &a)
}
// DeleteAccount deletes an account via cs3 by id
func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
Spec: &provider.Reference_Path{Path: path.Join("/meta", accountsFolder, id)},
},
})
if err != nil {
return err
}
// TODO Handle other error codes?
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
return &notFoundErr{"account", id}
}
return nil
}
// WriteGroup writes a group via cs3 and modifies the provided group (e.g. with a generated id).
func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil {
return err
}
var by []byte
if by, err = json.Marshal(g); err != nil {
return err
}
_, err = r.dataProvider.put(r.groupURL(g.Id), bytes.NewReader(by), t)
return err
}
// LoadGroup loads a group via cs3 by id and writes it to the provided group
func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
resp, err := r.dataProvider.get(r.groupURL(id), t)
if err != nil {
return err
}
if resp.StatusCode == http.StatusNotFound {
return &notFoundErr{"group", id}
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if err = resp.Body.Close(); err != nil {
return err
}
return json.Unmarshal(b, &g)
}
// DeleteGroup deletes a group via cs3 by id
func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
Spec: &provider.Reference_Path{Path: path.Join("/meta", groupsFolder, id)},
},
})
if err != nil {
return err
}
// TODO Handle other error codes?
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
return &notFoundErr{"group", id}
}
return err
}
func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) {
u := &user.User{
Id: &user.UserId{},
Groups: []string{},
}
if r.cfg.ServiceUser.Username != "" {
u.Id.OpaqueId = r.cfg.ServiceUser.UUID
}
return r.tm.MintToken(ctx, u)
}
func (r CS3Repo) accountURL(id string) string {
return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, accountsFolder, id))
}
func (r CS3Repo) groupURL(id string) string {
return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, groupsFolder, id))
}
func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: path.Join("/meta", folder)},
}
resp, err := r.storageProvider.Stat(ctx, &provider.StatRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
_, err := r.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
}
return nil
}
// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
type dataProviderClient struct {
client http.Client
}
func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPut, url, body)
if err != nil {
return nil, err
}
req.Header.Add("x-access-token", token)
return d.client.Do(req)
}
func (d dataProviderClient) get(url string, token string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Add("x-access-token", token)
return d.client.Do(req)
}
accounts/pkg/storage/cs3_test.go
+74
View File
@@ -0,0 +1,74 @@
package storage
// Uncomment to test locally, requires started metadata-storage for now
//import (
// "context"
// "github.com/owncloud/ocis/accounts/pkg/config"
// "github.com/owncloud/ocis/accounts/pkg/proto/v0"
// "github.com/stretchr/testify/assert"
// "testing"
//)
//
//var cfg = &config.Config{
// TokenManager: config.TokenManager{
// JWTSecret: "Pive-Fumkiu4",
// },
// Repo: config.Repo{
// CS3: config.CS3{
// ProviderAddr: "0.0.0.0:9215",
// DataURL: "http://localhost:9216",
// DataPrefix: "data",
// },
// },
//}
//
//func TestCS3Repo_WriteAccount(t *testing.T) {
// r, err := NewCS3Repo("hello", cfg)
// assert.NoError(t, err)
//
// err = r.WriteAccount(context.Background(), &proto.Account{
// Id: "fefef-egegweg-gegeg",
// AccountEnabled: true,
// DisplayName: "Mike Jones",
// Mail: "mike@example.com",
// })
//
// assert.NoError(t, err)
//}
//
//func TestCS3Repo_LoadAccount(t *testing.T) {
// r, err := NewCS3Repo("hello", cfg)
// assert.NoError(t, err)
//
// err = r.WriteAccount(context.Background(), &proto.Account{
// Id: "fefef-egegweg-gegeg",
// AccountEnabled: true,
// DisplayName: "Mike Jones",
// Mail: "mike@example.com",
// })
//
// acc := &proto.Account{}
// err = r.LoadAccount(context.Background(), "fefef-egegweg-gegeg", acc)
//
// assert.NoError(t, err)
// assert.Equal(t, "fefef-egegweg-gegeg", acc.Id)
// assert.Equal(t, "Mike Jones", acc.DisplayName)
// assert.Equal(t, "mike@example.com", acc.Mail)
//}
//
//func TestCS3Repo_DeleteAccount(t *testing.T) {
// r, err := NewCS3Repo("hello", cfg)
// assert.NoError(t, err)
//
// err = r.WriteAccount(context.Background(), &proto.Account{
// Id: "delete-me-id",
// AccountEnabled: true,
// DisplayName: "Mike Jones",
// Mail: "mike@example.com",
// })
//
// err = r.DeleteAccount(context.Background(), "delete-me-id")
//
// assert.NoError(t, err)
//}
accounts/pkg/storage/disk.go
+168
View File
@@ -0,0 +1,168 @@
package storage
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
olog "github.com/owncloud/ocis/ocis-pkg/log"
)
var groupLock sync.Mutex
// DiskRepo provides a local filesystem implementation of the Repo interface
type DiskRepo struct {
cfg *config.Config
log olog.Logger
}
// NewDiskRepo creates a new disk repo
func NewDiskRepo(cfg *config.Config, log olog.Logger) DiskRepo {
paths := []string{
filepath.Join(cfg.Repo.Disk.Path, accountsFolder),
filepath.Join(cfg.Repo.Disk.Path, groupsFolder),
}
for i := range paths {
if _, err := os.Stat(paths[i]); err != nil {
if os.IsNotExist(err) {
if err = os.MkdirAll(paths[i], 0700); err != nil {
log.Fatal().Err(err).Msgf("could not create data folder %v", paths[i])
}
}
}
}
return DiskRepo{
cfg: cfg,
log: log,
}
}
// WriteAccount to the local filesystem
func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error) {
// leave only the group id
r.deflateMemberOf(a)
var bytes []byte
if bytes, err = json.Marshal(a); err != nil {
return err
}
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, a.Id)
return ioutil.WriteFile(path, bytes, 0600)
}
// LoadAccount from the local filesystem
func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id)
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
if os.IsNotExist(err) {
err = &notFoundErr{"account", id}
}
return
}
return json.Unmarshal(data, a)
}
// DeleteAccount from the local filesystem
func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id)
if err = os.Remove(path); err != nil {
if os.IsNotExist(err) {
err = &notFoundErr{"account", id}
}
}
return
}
// WriteGroup to the local filesystem
func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
// leave only the member id
r.deflateMembers(g)
var bytes []byte
if bytes, err = json.Marshal(g); err != nil {
return err
}
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, g.Id)
groupLock.Lock()
defer groupLock.Unlock()
return ioutil.WriteFile(path, bytes, 0600)
}
// LoadGroup from the local filesystem
func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id)
groupLock.Lock()
defer groupLock.Unlock()
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
if os.IsNotExist(err) {
err = &notFoundErr{"group", id}
}
return
}
return json.Unmarshal(data, g)
}
// DeleteGroup from the local filesystem
func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id)
if err = os.Remove(path); err != nil {
if os.IsNotExist(err) {
err = &notFoundErr{"account", id}
}
}
return nil
//r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group")
//return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error())
}
// deflateMemberOf replaces the groups of a user with an instance that only contains the id
func (r DiskRepo) deflateMemberOf(a *proto.Account) {
if a == nil {
return
}
var deflated []*proto.Group
for i := range a.MemberOf {
if a.MemberOf[i].Id != "" {
deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id})
} else {
// TODO fetch and use an id when group only has a name but no id
r.log.Error().Str("id", a.Id).Interface("group", a.MemberOf[i]).Msg("resolving groups by name is not implemented yet")
}
}
a.MemberOf = deflated
}
// deflateMembers replaces the users of a group with an instance that only contains the id
func (r DiskRepo) deflateMembers(g *proto.Group) {
if g == nil {
return
}
var deflated []*proto.Account
for i := range g.Members {
if g.Members[i].Id != "" {
deflated = append(deflated, &proto.Account{Id: g.Members[i].Id})
} else {
// TODO fetch and use an id when group only has a name but no id
r.log.Error().Str("id", g.Id).Interface("account", g.Members[i]).Msg("resolving members by name is not implemented yet")
}
}
g.Members = deflated
}
accounts/pkg/storage/errors.go
+19
View File
@@ -0,0 +1,19 @@
package storage
import (
"fmt"
)
type notFoundErr struct {
typ, id string
}
func (e *notFoundErr) Error() string {
return fmt.Sprintf("%s with id %s not found", e.typ, e.id)
}
// IsNotFoundErr can be returned by repo Load and Delete operations
func IsNotFoundErr(e error) bool {
_, ok := e.(*notFoundErr)
return ok
}
accounts/pkg/storage/repo.go
+21
View File
@@ -0,0 +1,21 @@
package storage
import (
"context"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
)
const (
accountsFolder = "accounts"
groupsFolder = "groups"
)
// Repo defines the storage operations
type Repo interface {
WriteAccount(ctx context.Context, a *proto.Account) (err error)
LoadAccount(ctx context.Context, id string, a *proto.Account) (err error)
DeleteAccount(ctx context.Context, id string) (err error)
WriteGroup(ctx context.Context, g *proto.Group) (err error)
LoadGroup(ctx context.Context, id string, g *proto.Group) (err error)
DeleteGroup(ctx context.Context, id string) (err error)
}
ocis/pkg/command/revastoragemetadata.go
+49
View File
@@ -0,0 +1,49 @@
package command
import (
"github.com/micro/cli/v2"
"github.com/owncloud/ocis/ocis-reva/pkg/command"
svcconfig "github.com/owncloud/ocis/ocis-reva/pkg/config"
"github.com/owncloud/ocis/ocis-reva/pkg/flagset"
"github.com/owncloud/ocis/ocis/pkg/config"
"github.com/owncloud/ocis/ocis/pkg/register"
)
// RevaStorageMetadataCommand is the entrypoint for the reva-storage-metadata command.
func RevaStorageMetadataCommand(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "reva-storage-metadata",
Usage: "Start reva storage service for metadata mount",
Category: "Extensions",
Flags: flagset.StorageMetadata(cfg.Reva),
Action: func(c *cli.Context) error {
revaStorageMetadataCommand := command.StorageMetadata(configureRevaStorageMetadata(cfg))
if err := revaStorageMetadataCommand.Before(c); err != nil {
return err
}
return cli.HandleAction(revaStorageMetadataCommand.Action, c)
},
}
}
func configureRevaStorageMetadata(cfg *config.Config) *svcconfig.Config {
cfg.Reva.Log.Level = cfg.Log.Level
cfg.Reva.Log.Pretty = cfg.Log.Pretty
cfg.Reva.Log.Color = cfg.Log.Color
if cfg.Tracing.Enabled {
cfg.Reva.Tracing.Enabled = cfg.Tracing.Enabled
cfg.Reva.Tracing.Type = cfg.Tracing.Type
cfg.Reva.Tracing.Endpoint = cfg.Tracing.Endpoint
cfg.Reva.Tracing.Collector = cfg.Tracing.Collector
cfg.Reva.Tracing.Service = cfg.Tracing.Service
}
return cfg.Reva
}
func init() {
register.AddCommand(RevaStorageMetadataCommand)
}
ocis/pkg/command/run.go
+2 -8
View File
@@ -2,6 +2,7 @@ package command
import (
"fmt"
"github.com/owncloud/ocis/ocis/pkg/runtime"
"log"
"net"
"net/rpc"
@@ -11,7 +12,6 @@ import (
"github.com/owncloud/ocis/ocis/pkg/config"
"github.com/owncloud/ocis/ocis/pkg/register"
"github.com/refs/pman/pkg/process"
)
// RunCommand is the entrypoint for the run command.
@@ -40,13 +40,7 @@ func RunCommand(cfg *config.Config) *cli.Command {
log.Fatal("dialing:", err)
}
proc := process.NewProcEntry(os.Args[2], os.Environ(), []string{os.Args[2]}...)
var res int
if err := client.Call("Service.Start", proc, &res); err != nil {
log.Fatal(err)
}
res := runtime.RunService(client, os.Args[2])
fmt.Println(res)
return nil
},
ocis/pkg/command/server.go
+1 -25
View File
@@ -33,31 +33,7 @@ func Server(cfg *config.Config) *cli.Command {
}
r := runtime.New()
// TODO temporary service startup selection. Should go away and the runtime should take care of it.
return r.Start(append([]string{
"accounts",
"settings",
"konnectd",
"proxy",
"ocs",
"phoenix",
"glauth",
"webdav",
"store",
"thumbnails",
"reva-frontend",
"reva-gateway",
"reva-users",
"reva-auth-basic",
"reva-auth-bearer",
"reva-storage-home",
"reva-storage-home-data",
"reva-storage-eos",
"reva-storage-eos-data",
"reva-storage-oc",
"reva-storage-oc-data",
"reva-storage-public-link",
}, runtime.MicroServices...)...)
return r.Start()
},
}
}
ocis/pkg/runtime/command.go
-19
View File
@@ -1,19 +0,0 @@
package runtime
import (
"github.com/micro/cli/v2"
)
// Command adds micro runtime commands to the cli app
func Command(app *cli.App) *cli.Command {
command := cli.Command{
Name: "micro",
Description: "starts the go-micro runtime services",
Category: "Micro",
Action: func(c *cli.Context) error {
runtime := New()
return runtime.Start()
},
}
return &command
}
ocis/pkg/runtime/runtime.go
+28 -22
View File
@@ -50,7 +50,7 @@ var (
"reva-storage-oc",
"reva-storage-oc-data",
"reva-storage-public-link",
"accounts",
"reva-storage-metadata",
"glauth",
"konnectd",
"thumbnails",
@@ -58,6 +58,7 @@ var (
// There seem to be a race condition when reva-sharing needs to read the sharing.json file and the parent folder is not present.
dependants = []string{
"accounts",
"reva-sharing",
}
@@ -74,13 +75,13 @@ func New() Runtime {
}
// Start rpc runtime
func (r *Runtime) Start(services ...string) error {
go r.Launch(services)
func (r *Runtime) Start() error {
go r.Launch()
return service.Start()
}
// Launch ocis default ocis extensions.
func (r *Runtime) Launch(services []string) {
func (r *Runtime) Launch() {
var client *rpc.Client
var err error
var try int
@@ -100,32 +101,37 @@ func (r *Runtime) Launch(services []string) {
}
OUT:
for _, v := range services {
args := process.NewProcEntry(v, os.Environ(), []string{v}...)
var reply int
if err := client.Call("Service.Start", args, &reply); err != nil {
golog.Fatal(err)
}
for _, v := range MicroServices {
RunService(client, v)
}
// TODO(refs) this should disappear and tackled at the runtime (pman) level.
// see https://github.com/cs3org/reva/issues/795 for race condition.
// dependants might not be needed on a ocis_simple build, therefore
// it should not be started under these circumstances.
if len(services) >= len(Extensions) { // it will not run for ocis_simple builds.
for _, v := range Extensions {
RunService(client, v)
}
if len(dependants) > 0 {
// TODO(refs) this should disappear and tackled at the runtime (pman) level.
// see https://github.com/cs3org/reva/issues/795 for race condition.
// dependants might not be needed on a ocis_simple build, therefore
// it should not be started under these circumstances.
time.Sleep(2 * time.Second)
for _, v := range dependants {
args := process.NewProcEntry(v, os.Environ(), []string{v}...)
var reply int
if err := client.Call("Service.Start", args, &reply); err != nil {
golog.Fatal(err)
}
RunService(client, v)
}
}
}
// RunService sends a Service.Start command with the given service name to pman
func RunService(client *rpc.Client, service string) int {
args := process.NewProcEntry(service, os.Environ(), []string{service}...)
var reply int
if err := client.Call("Service.Start", args, &reply); err != nil {
golog.Fatal(err)
}
return reply
}
// AddMicroPlatform adds the micro subcommands to the cli app
func AddMicroPlatform(app *cli.App) {
setDefaults()
ocs/go.mod
+1
View File
@@ -15,6 +15,7 @@ require (
github.com/micro/cli/v2 v2.1.2
github.com/micro/go-micro/v2 v2.9.1
github.com/oklog/run v1.1.0
github.com/olekukonko/tablewriter v0.0.4
github.com/openzipkin/zipkin-go v0.2.2
github.com/owncloud/ocis/accounts v0.0.0-20200918125107-fcca9faa81c8
github.com/owncloud/ocis/ocis-pkg v0.0.0-20200918114005-1a0ddd2190ee
ocs/go.sum
+1
View File
@@ -1927,6 +1927,7 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 h1:kDtqNkeBrZb8B+atrj50B5XLHpzXXqcCdZPP/ApQ5NY=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200421042724-cfa8b22178d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200427214658-4697a2867c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
ocs/pkg/server/http/svc_test.go
+5
View File
@@ -249,6 +249,11 @@ func init() {
Server: accountsCfg.Server{
AccountsDataPath: dataPath,
},
Repo: accountsCfg.Repo{
Disk: accountsCfg.Disk{
Path: dataPath,
},
},
Log: accountsCfg.Log{
Level: "info",
Pretty: true,