sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into rebuild-index

Browse Source
This commit is contained in:
A.Unger
2020-10-23 22:58:35 +02:00
parent 55d1f78bda 542b5a2b1b
commit 142f094f43
25 changed files with 355 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
accounts/pkg/indexer/index/cs3/autoincrement.go
+16 -36
View File
@@ -12,9 +12,10 @@ import (
"strconv"
"strings"
"github.com/owncloud/ocis/accounts/pkg/storage"
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
@@ -40,6 +41,7 @@ type Autoincrement struct {
dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
cs3conf *Config
bound *option.Bound
}
func init() {
@@ -57,15 +59,15 @@ func NewAutoincrementIndex(o ...option.Option) index.Index {
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
bound: opts.Bound,
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUserName: opts.ServiceUserName,
ServiceUserUUID: opts.ServiceUserUUID,
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
dataProvider: dataProviderClient{
baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
@@ -284,7 +286,6 @@ func (idx *Autoincrement) createSymlink(oldname, newname string) error {
}
return nil
}
func (idx *Autoincrement) resolveSymlink(name string) (string, error) {
@@ -317,37 +318,11 @@ func (idx *Autoincrement) resolveSymlink(name string) (string, error) {
}
func (idx *Autoincrement) makeDirIfNotExists(ctx context.Context, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)},
}
resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
_, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
}
return nil
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
}
func (idx *Autoincrement) authenticate(ctx context.Context) (token string, err error) {
u := &user.User{
Id: &user.UserId{OpaqueId: idx.cs3conf.ServiceUserUUID},
Groups: []string{},
}
return idx.tokenManager.MintToken(ctx, u)
return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
}
func (idx *Autoincrement) next() (int, error) {
@@ -383,6 +358,11 @@ func (idx *Autoincrement) next() (int, error) {
if err != nil {
return -1, err
}
if int64(latest) < idx.bound.Lower {
return int(idx.bound.Lower), nil
}
return latest + 1, nil
}
accounts/pkg/indexer/index/cs3/non_unique.go
+12 -39
View File
@@ -10,7 +10,8 @@ import (
"path/filepath"
"strings"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/owncloud/ocis/accounts/pkg/storage"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
@@ -66,12 +67,11 @@ func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUserName: opts.ServiceUserName,
ServiceUserUUID: opts.ServiceUserUUID,
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
dataProvider: dataProviderClient{
baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
@@ -315,38 +315,8 @@ func (idx *NonUnique) FilesDir() string {
return idx.filesDir
}
func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) {
u := &user.User{
Id: &user.UserId{OpaqueId: idx.cs3conf.ServiceUserUUID},
Groups: []string{},
}
return idx.tokenManager.MintToken(ctx, u)
}
func (idx *NonUnique) makeDirIfNotExists(ctx context.Context, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)},
}
resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
_, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
}
return nil
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
}
func (idx *NonUnique) createSymlink(oldname, newname string) error {
@@ -368,7 +338,6 @@ func (idx *NonUnique) createSymlink(oldname, newname string) error {
}
return nil
}
func (idx *NonUnique) resolveSymlink(name string) (string, error) {
@@ -418,3 +387,7 @@ func (idx *NonUnique) Delete() error {
return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir)
}
func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) {
return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
}
accounts/pkg/indexer/index/cs3/unique.go
+16 -42
View File
@@ -10,7 +10,10 @@ import (
"path/filepath"
"strings"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/owncloud/ocis/accounts/pkg/storage"
"github.com/owncloud/ocis/accounts/pkg/config"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
@@ -41,12 +44,11 @@ type Unique struct {
// Config represents cs3conf. Should be deprecated in favor of config.Config.
type Config struct {
ProviderAddr string
DataURL string
DataPrefix string
JWTSecret string
ServiceUserName string
ServiceUserUUID string
ProviderAddr string
DataURL string
DataPrefix string
JWTSecret string
ServiceUser config.ServiceUser
}
func init() {
@@ -69,12 +71,11 @@ func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUserName: opts.ServiceUserName,
ServiceUserUUID: opts.ServiceUserUUID,
ProviderAddr: opts.ProviderAddr,
DataURL: opts.DataURL,
DataPrefix: opts.DataPrefix,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
dataProvider: dataProviderClient{
baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
@@ -305,7 +306,6 @@ func (idx *Unique) createSymlink(oldname, newname string) error {
}
return nil
}
func (idx *Unique) resolveSymlink(name string) (string, error) {
@@ -338,37 +338,11 @@ func (idx *Unique) resolveSymlink(name string) (string, error) {
}
func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)},
}
resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
_, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{
Ref: rootPathRef,
})
if err != nil {
return err
}
}
return nil
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
}
func (idx *Unique) authenticate(ctx context.Context) (token string, err error) {
u := &user.User{
Id: &user.UserId{OpaqueId: idx.cs3conf.ServiceUserUUID},
Groups: []string{},
}
return idx.tokenManager.MintToken(ctx, u)
return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
}
func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
accounts/pkg/indexer/indexer.go
+1 -2
View File
@@ -86,8 +86,7 @@ func (i Indexer) AddIndex(t interface{}, indexBy, pkName, entityDirName, indexTy
option.WithDataPrefix(i.config.Repo.CS3.DataPrefix),
option.WithJWTSecret(i.config.Repo.CS3.JWTSecret),
option.WithProviderAddr(i.config.Repo.CS3.ProviderAddr),
option.WithServiceUserUUID(i.config.ServiceUser.UUID),
option.WithServiceUserName(i.config.ServiceUser.Username),
option.WithServiceUser(i.config.ServiceUser),
)
}
accounts/pkg/indexer/option/option.go
+10 -16
View File
@@ -1,5 +1,7 @@
package option
import "github.com/owncloud/ocis/accounts/pkg/config"
// Option defines a single option function.
type Option func(o *Options)
@@ -25,12 +27,11 @@ type Options struct {
Entity interface{}
// CS3 options
DataURL string
DataPrefix string
JWTSecret string
ProviderAddr string
ServiceUserUUID string
ServiceUserName string
DataURL string
DataPrefix string
JWTSecret string
ProviderAddr string
ServiceUser config.ServiceUser
}
// CaseInsensitive sets the CaseInsensitive field.
@@ -117,16 +118,9 @@ func WithProviderAddr(val string) Option {
}
}
// WithServiceUserUUID sets the option ServiceUserUUID.
func WithServiceUserUUID(val string) Option {
// WithServiceUser sets the option ServiceUser.
func WithServiceUser(val config.ServiceUser) Option {
return func(o *Options) {
o.ServiceUserUUID = val
}
}
// WithServiceUserName sets the option ServiceUserName.
func WithServiceUserName(val string) Option {
return func(o *Options) {
o.ServiceUserName = val
o.ServiceUser = val
}
}
accounts/pkg/indexer/test/test.go
+11
View File
@@ -3,6 +3,9 @@ package test
import (
"context"
"flag"
"net"
"time"
"github.com/micro/cli/v2"
"github.com/owncloud/ocis/storage/pkg/command"
mcfg "github.com/owncloud/ocis/storage/pkg/config"
@@ -20,4 +23,12 @@ func setupMetadataStorage() {
}
_ = app.Command("storage-metadata").Run(cli.NewContext(&app, &flag.FlagSet{}, &cli.Context{Context: context.Background()}))
// wait until port is open
d := net.Dialer{Timeout: 5 * time.Second}
conn, err := d.Dial("tcp", "localhost:9125")
if err != nil {
panic("timeout waiting for storage")
}
conn.Close()
}
accounts/pkg/service/v0/accounts.go
+1 -1
View File
@@ -230,7 +230,7 @@ func (s Service) findAccountsByQuery(ctx context.Context, query string) ([]strin
qID, qSam := match[1], match[2]
tmp := &proto.Account{}
err = s.repo.LoadAccount(ctx, qID, tmp)
if err != nil {
if err != nil && !storage.IsNotFoundErr(err) {
return nil, err
}
searchResults, err = s.index.FindBy(&proto.Account{}, "OnPremisesSamAccountName", qSam)
accounts/pkg/storage/cs3.go
+29 -5
View File
@@ -9,11 +9,13 @@ import (
"net/http"
"path"
"path/filepath"
"strconv"
"strings"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/token"
"github.com/cs3org/reva/pkg/token/manager/jwt"
@@ -131,7 +133,7 @@ func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error {
return err
}
if resp.StatusCode == http.StatusNotFound {
if resp.StatusCode != http.StatusOK {
return &notFoundErr{"account", id}
}
@@ -287,13 +289,30 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
}
func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) {
return AuthenticateCS3(ctx, r.cfg.ServiceUser, r.tm)
}
// AuthenticateCS3 mints an auth token for communicating with cs3 storage based on a service user from config
func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manager) (token string, err error) {
u := &user.User{
Id: &user.UserId{
OpaqueId: r.cfg.ServiceUser.UUID,
OpaqueId: su.UUID,
},
Groups: []string{},
Opaque: &types.Opaque{
Map: map[string]*types.OpaqueEntry{
"uid": {
Decoder: "plain",
Value: []byte(strconv.FormatInt(su.UID, 10)),
},
"gid": {
Decoder: "plain",
Value: []byte(strconv.FormatInt(su.GID, 10)),
},
},
},
}
return r.tm.MintToken(ctx, u)
return tm.MintToken(ctx, u)
}
func (r CS3Repo) accountURL(id string) string {
@@ -305,11 +324,16 @@ func (r CS3Repo) groupURL(id string) string {
}
func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error {
return MakeDirIfNotExist(ctx, r.storageProvider, folder)
}
// MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context.
func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: path.Join("/meta", folder)},
}
resp, err := r.storageProvider.Stat(ctx, &provider.StatRequest{
resp, err := sp.Stat(ctx, &provider.StatRequest{
Ref: rootPathRef,
})
@@ -318,7 +342,7 @@ func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
_, err := r.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{
_, err := sp.CreateContainer(ctx, &provider.CreateContainerRequest{
Ref: rootPathRef,
})