adapt metadata storage to make use the spaces datatx

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

accounts/pkg/storage/cs3.go

@@ -7,10 +7,13 @@ import (
 	"path/filepath"
 
 	"github.com/cs3org/reva/pkg/auth/scope"
+	"github.com/cs3org/reva/pkg/errtypes"
+	"github.com/cs3org/reva/pkg/utils"
 
 	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
 	revactx "github.com/cs3org/reva/pkg/ctx"
 	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/token"
@@ -32,6 +35,7 @@ type CS3Repo struct {
 	tm              token.Manager
 	storageProvider provider.ProviderAPIClient
 	metadataStorage metadatastorage.MetadataStorage
+	space           *provider.StorageSpace
 }
 
 // NewCS3Repo creates a new cs3 repo
@@ -54,12 +58,60 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) {
 		return nil, err
 	}
 
-	return CS3Repo{
+	repo := CS3Repo{
 		cfg:             cfg,
 		tm:              tokenManager,
 		storageProvider: client,
 		metadataStorage: ms,
-	}, nil
+	}
+
+	if err := repo.Init(); err != nil {
+		return nil, err
+	}
+
+	return &repo, nil
+}
+
+// init creates the metadata space
+func (r *CS3Repo) Init() (err error) {
+	ctx := context.Background()
+	ctx, err = r.getAuthenticatedContext(ctx)
+	if err != nil {
+		return err
+	}
+	// FIXME change CS3 api to allow sending a space id
+	cssr, err := r.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{
+		Opaque: &typesv1beta1.Opaque{
+			Map: map[string]*typesv1beta1.OpaqueEntry{
+				"spaceid": {
+					Decoder: "plain",
+					Value:   []byte(r.cfg.ServiceUser.UUID),
+				},
+			},
+		},
+		Owner: &user.User{
+			Id: &user.UserId{
+				OpaqueId: r.cfg.ServiceUser.UUID,
+			},
+			Groups:    []string{},
+			UidNumber: r.cfg.ServiceUser.UID,
+			GidNumber: r.cfg.ServiceUser.GID,
+		},
+		Name: "Metadata",
+		Type: "metadata",
+	})
+	switch {
+	case err != nil:
+		return err
+	case cssr.Status.Code == v1beta11.Code_CODE_OK:
+		// continue
+	case cssr.Status.Code != v1beta11.Code_CODE_ALREADY_EXISTS:
+		// continue
+	default:
+		return errtypes.NewErrtypeFromStatus(cssr.Status)
+	}
+	r.metadataStorage.SpaceRoot = cssr.StorageSpace.Root
+	return nil
 }
 
 // WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id).
@@ -102,7 +154,8 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err
 
 	res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", accountsFolder),
+			ResourceId: r.space.Root,
+			Path:       utils.MakeRelativePath(accountsFolder),
 		},
 	})
 	if err != nil {
@@ -142,7 +195,8 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
 
 	resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", accountsFolder, id),
+			ResourceId: r.space.Root,
+			Path:       utils.MakeRelativePath(filepath.Join("/", accountsFolder, id)),
 		},
 	})
 
@@ -197,7 +251,8 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error)
 
 	res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", groupsFolder),
+			ResourceId: r.space.Root,
+			Path:       utils.MakeRelativePath(groupsFolder),
 		},
 	})
 	if err != nil {
@@ -237,7 +292,8 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
 
 	resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", groupsFolder, id),
+			ResourceId: r.space.Root,
+			Path:       utils.MakeRelativePath(filepath.Join(groupsFolder, id)),
 		},
 	})
 
@@ -289,13 +345,14 @@ func (r CS3Repo) groupURL(id string) string {
 }
 
 func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error {
-	return MakeDirIfNotExist(ctx, r.storageProvider, folder)
+	return MakeDirIfNotExist(ctx, r.storageProvider, r.space.Root, folder)
 }
 
 // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context.
-func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folder string) error {
+func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, root *provider.ResourceId, folder string) error {
 	var rootPathRef = &provider.Reference{
-		Path: path.Join("/", folder),
+		ResourceId: root,
+		Path:       utils.MakeRelativePath(folder),
 	}
 
 	resp, err := sp.Stat(ctx, &provider.StatRequest{

ocis-pkg/indexer/index/cs3/autoincrement.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/token"
 	"github.com/cs3org/reva/pkg/token/manager/jwt"
+	"github.com/cs3org/reva/pkg/utils"
 	"google.golang.org/grpc/metadata"
 
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
@@ -165,7 +166,11 @@ func (idx *Autoincrement) Remove(id string, v string) error {
 	deletePath := path.Join("/", idx.indexRootDir, v)
 	resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: deletePath,
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(deletePath),
 		},
 	})
 
@@ -203,7 +208,11 @@ func (idx *Autoincrement) Search(pattern string) ([]string, error) {
 
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", idx.indexRootDir),
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(idx.indexRootDir),
 		},
 	})
 
@@ -289,7 +298,6 @@ func (idx *Autoincrement) makeDirIfNotExists(folder string) error {
 	if err != nil {
 		return err
 	}
-
 	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
 }
 
@@ -301,11 +309,11 @@ func (idx *Autoincrement) next() (int, error) {
 
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-<<<<<<< HEAD
+			ResourceId: &provider.ResourceId{
-			Path: path.Join("/meta", idx.indexRootDir), //TODO:
+				StorageId: idx.cs3conf.ServiceUser.UUID,
-=======
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
-			Path: path.Join("/", idx.indexRootDir),
+			},
->>>>>>> 7a5fb4c2e (drop /meta mount prefix)
+			Path: utils.MakeRelativePath(idx.indexRootDir),
 		},
 	})
 
@@ -352,5 +360,5 @@ func (idx *Autoincrement) Delete() error {
 		return err
 	}
 
-	return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir)
+	return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
 }

ocis-pkg/indexer/index/cs3/data_provider_client.go

@@ -0,0 +1,47 @@
+package cs3
+
+import (
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+)
+
+type dataProviderClient struct {
+	client  http.Client
+	spaceid string
+	baseURL string
+}
+
+func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("x-access-token", token)
+	return d.client.Do(req)
+}
+
+func (d dataProviderClient) get(url string, token string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("x-access-token", token)
+	return d.client.Do(req)
+}
+
+// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg
+func singleJoiningSlash(a, b string) string {
+	aslash := strings.HasSuffix(a, "/")
+	bslash := strings.HasPrefix(b, "/")
+	switch {
+	case aslash && bslash:
+		return a + b[1:]
+	case !aslash && !bslash:
+		return a + "/" + b
+	}
+	return a + b
+}

ocis-pkg/indexer/index/cs3/helper.go

@@ -3,16 +3,20 @@ package cs3
 import (
 	"context"
 	"fmt"
-	"path"
 
 	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/utils"
 )
 
-func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, indexRootDir string) error {
+func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, spaceid, indexRootDir string) error {
 	res, err := storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", indexRootDir),
+			ResourceId: &provider.ResourceId{
+				StorageId: spaceid,
+				OpaqueId:  spaceid,
+			},
+			Path: utils.MakeRelativePath(indexRootDir),
 		},
 	})
 	if err != nil {

ocis-pkg/indexer/index/cs3/non_unique.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/token"
 	"github.com/cs3org/reva/pkg/token/manager/jwt"
+	"github.com/cs3org/reva/pkg/utils"
 	idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/option"
@@ -119,7 +120,11 @@ func (idx *NonUnique) Lookup(v string) ([]string, error) {
 
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", idx.indexRootDir, v),
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(path.Join("/", idx.indexRootDir, v)),
 		},
 	})
 
@@ -175,7 +180,11 @@ func (idx *NonUnique) Remove(id string, v string) error {
 	deletePath := path.Join("/", idx.indexRootDir, v, id)
 	resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: deletePath,
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(deletePath),
 		},
 	})
 
@@ -190,7 +199,11 @@ func (idx *NonUnique) Remove(id string, v string) error {
 	toStat := path.Join("/", idx.indexRootDir, v)
 	lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: toStat,
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(toStat),
 		},
 	})
 	if err != nil {
@@ -201,7 +214,11 @@ func (idx *NonUnique) Remove(id string, v string) error {
 		deletePath = path.Join("/", idx.indexRootDir, v)
 		_, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
 			Ref: &provider.Reference{
-				Path: deletePath,
+				ResourceId: &provider.ResourceId{
+					StorageId: idx.cs3conf.ServiceUser.UUID,
+					OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+				},
+				Path: utils.MakeRelativePath(deletePath),
 			},
 		})
 		if err != nil {
@@ -245,7 +262,11 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) {
 	matches := make([]string, 0)
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", idx.indexRootDir),
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(idx.indexRootDir),
 		},
 	})
 
@@ -266,7 +287,11 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) {
 	for i := range foldersMatched {
 		res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 			Ref: &provider.Reference{
-				Path: foldersMatched[i],
+				ResourceId: &provider.ResourceId{
+					StorageId: idx.cs3conf.ServiceUser.UUID,
+					OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+				},
+				Path: utils.MakeRelativePath(foldersMatched[i]),
 			},
 		})
 
@@ -356,5 +381,5 @@ func (idx *NonUnique) Delete() error {
 		return err
 	}
 
-	return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir)
+	return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
 }

ocis-pkg/indexer/index/cs3/unique.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/token"
 	"github.com/cs3org/reva/pkg/token/manager/jwt"
+	"github.com/cs3org/reva/pkg/utils"
 	idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/option"
@@ -162,15 +163,14 @@ func (idx *Unique) Remove(id string, v string) error {
 		return err
 	}
 
-<<<<<<< HEAD
-	deletePath := path.Join("/meta", idx.indexRootDir, v)
-=======
 	deletePath := path.Join("/", idx.indexRootDir, v)
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
->>>>>>> 7a5fb4c2e (drop /meta mount prefix)
 	resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: deletePath,
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(deletePath),
 		},
 	})
 
@@ -217,7 +217,11 @@ func (idx *Unique) Search(pattern string) ([]string, error) {
 
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/", idx.indexRootDir),
+			ResourceId: &provider.ResourceId{
+				StorageId: idx.cs3conf.ServiceUser.UUID,
+				OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+			},
+			Path: utils.MakeRelativePath(idx.indexRootDir),
 		},
 	})
 
@@ -304,7 +308,10 @@ func (idx *Unique) makeDirIfNotExists(folder string) error {
 	if err != nil {
 		return err
 	}
-	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
+	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
+		StorageId: idx.cs3conf.ServiceUser.UUID,
+		OpaqueId:  idx.cs3conf.ServiceUser.UUID,
+	}, folder)
 }
 
 func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
@@ -323,5 +330,5 @@ func (idx *Unique) Delete() error {
 		return err
 	}
 
-	return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir)
+	return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
 }

ocis-pkg/metadata_storage/metadata_storage.go

@@ -36,17 +36,19 @@ func NewMetadataStorage(providerAddr string) (s MetadataStorage, err error) {
 type MetadataStorage struct {
 	storageProvider   provider.ProviderAPIClient
 	dataGatewayClient *http.Client
+	SpaceRoot         *provider.ResourceId
 }
 
-func (r MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error {
+func (ms MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error {
 
 	ref := provider.InitiateFileUploadRequest{
 		Ref: &provider.Reference{
-			Path: path.Join(storageMountPath, uploadpath),
+			ResourceId: ms.SpaceRoot,
+			Path:       path.Join(storageMountPath, uploadpath),
 		},
 	}
 
-	res, err := r.storageProvider.InitiateFileUpload(ctx, &ref)
+	res, err := ms.storageProvider.InitiateFileUpload(ctx, &ref)
 	if err != nil {
 		return err
 	}
@@ -70,7 +72,7 @@ func (r MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, co
 
 	md, _ := metadata.FromOutgoingContext(ctx)
 	req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0])
-	resp, err := r.dataGatewayClient.Do(req)
+	resp, err := ms.dataGatewayClient.Do(req)
 	if err != nil {
 		return err
 	}
@@ -80,14 +82,15 @@ func (r MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, co
 	return nil
 }
 
-func (r MetadataStorage) SimpleDownload(ctx context.Context, downloadpath string) (content []byte, err error) {
+func (ms MetadataStorage) SimpleDownload(ctx context.Context, downloadpath string) (content []byte, err error) {
 	ref := provider.InitiateFileDownloadRequest{
 		Ref: &provider.Reference{
-			Path: path.Join(storageMountPath, downloadpath),
+			ResourceId: ms.SpaceRoot,
+			Path:       path.Join(storageMountPath, downloadpath),
 		},
 	}
 
-	res, err := r.storageProvider.InitiateFileDownload(ctx, &ref)
+	res, err := ms.storageProvider.InitiateFileDownload(ctx, &ref)
 	if err != nil {
 		return []byte{}, err
 	}
@@ -111,7 +114,7 @@ func (r MetadataStorage) SimpleDownload(ctx context.Context, downloadpath string
 
 	md, _ := metadata.FromOutgoingContext(ctx)
 	req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0])
-	resp, err := r.dataGatewayClient.Do(req)
+	resp, err := ms.dataGatewayClient.Do(req)
 	if err != nil {
 		return []byte{}, err
 	}