sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump reva

Browse Source
This commit is contained in:
André Duffeck
2025-02-13 10:08:22 +01:00
parent 5b85029813
commit 52e61d46d1
208 changed files with 11004 additions and 5254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace/grace.go
Generated Vendored
+2 -2
View File
@@ -298,7 +298,7 @@ func (w *Watcher) TrapSignals() {
// TODO: Ideally this would call exit() but properly return an error. The
// exit() is problematic (i.e. racey) especiaily when orchestrating multiple
// reva services from some external runtime (like in the "ocis server" case
// reva services from some external runtime (like in the "opencloud server" case
func gracefulShutdown(w *Watcher) {
w.log.Info().Int("Timeout", w.gracefulShutdownTimeout).Msg("preparing for a graceful shutdown with deadline")
go func() {
@@ -336,7 +336,7 @@ func gracefulShutdown(w *Watcher) {
// TODO: Ideally this would call exit() but properly return an error. The
// exit() is problematic (i.e. racey) especiaily when orchestrating multiple
// reva services from some external runtime (like in the "ocis server" case
// reva services from some external runtime (like in the "opencloud server" case
func hardShutdown(w *Watcher) {
w.log.Info().Msg("preparing for hard shutdown, aborting all conns")
for _, s := range w.ss {
vendor/github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider/storageprovider.go
Generated Vendored
+2 -3
View File
@@ -291,7 +291,6 @@ func (s *Service) InitiateFileDownload(ctx context.Context, req *provider.Initia
// TODO(labkode): maybe add short-lived token?
// We now simply point the client to the data server.
// For example, https://data-server.example.org/home/docs/myfile.txt
// or ownclouds://data-server.example.org/home/docs/myfile.txt
log := appctx.GetLogger(ctx)
u := *s.dataServerURL
log.Info().Str("data-server", u.String()).Interface("ref", req.Ref).Msg("file download")
@@ -399,7 +398,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate
if req.Opaque.Map["Upload-Checksum"] != nil {
metadata["checksum"] = string(req.Opaque.Map["Upload-Checksum"].Value)
}
// ownCloud mtime to set for the uploaded file
// OpenCloud mtime to set for the uploaded file
if req.Opaque.Map["X-OC-Mtime"] != nil {
metadata["mtime"] = string(req.Opaque.Map["X-OC-Mtime"].Value)
}
@@ -430,7 +429,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate
// - it is also unassigned
// - ends in 9 as the 409 conflict
// - is near the 4xx errors about conditions: 415 Unsupported Media Type, 416 Range Not Satisfiable or 417 Expectation Failed
// owncloud only expects a 400 Bad request so InvalidArg is good enough for now
// OpenCloud only expects a 400 Bad request so InvalidArg is good enough for now
// seealso errtypes.StatusChecksumMismatch
case errtypes.PermissionDenied:
st = status.NewPermissionDenied(ctx, err, "permission denied")
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go
Generated Vendored
+2 -2
View File
@@ -1604,7 +1604,7 @@ func mdToPropResponse(ctx context.Context, pf *XML, md *provider.ResourceInfo, p
// see everts stance on this https://stackoverflow.com/a/31621912, he points to http://tools.ietf.org/html/rfc4918#section-15.3
// > Purpose: Contains the Content-Length header returned by a GET without accept headers.
// which only would make sense when eg. rendering a plain HTML filelisting when GETing a collection,
// which is not the case ... so we don't return it on collections. owncloud has oc:size for that
// which is not the case ... so we don't return it on collections. OpenCloud has oc:size for that
// TODO we cannot find out if md.Size is set or not because ints in go default to 0
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
appendToNotFound(prop.NotFound("d:getcontentlength"))
@@ -1638,7 +1638,7 @@ func mdToPropResponse(ctx context.Context, pf *XML, md *provider.ResourceInfo, p
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
// always returns the current usage,
// in oc10 there seems to be a bug that makes the size in webdav differ from the one in the user properties, not taking shares into account
// in ocis we plan to always mak the quota a property of the storage space
// in OpenCloud we plan to always mak the quota a property of the storage space
appendToOK(prop.Escaped("d:quota-used-bytes", size))
} else {
appendToNotFound(prop.NotFound("d:quota-used-bytes"))
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/put.go
Generated Vendored
+2 -2
View File
@@ -246,7 +246,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
utils.AppendPlainToOpaque(opaque, net.HeaderUploadLength, strconv.FormatInt(length, 10))
// curl -X PUT https://demo.owncloud.com/remote.php/webdav/testcs.bin -u demo:demo -d '123' -v -H 'OC-Checksum: SHA1:40bd001563085fc35165329ea1ff5c5ecbdbbeef'
// curl -X PUT https://demo.example.org/remote.php/webdav/testcs.bin -u demo:demo -d '123' -v -H 'OC-Checksum: SHA1:40bd001563085fc35165329ea1ff5c5ecbdbbeef'
var cparts []string
// TUS Upload-Checksum header takes precedence
@@ -257,7 +257,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
w.WriteHeader(http.StatusBadRequest)
return
}
// Then try owncloud header
// Then try OpenCloud header
} else if checksum := r.Header.Get(net.HeaderOCChecksum); checksum != "" {
cparts = strings.SplitN(checksum, ":", 2)
if len(cparts) != 2 {
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go
Generated Vendored
+1 -1
View File
@@ -121,7 +121,7 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http.
// r.Header.Get(net.HeaderOCChecksum)
// TODO must be SHA1, ADLER32 or MD5 ... in capital letters????
// curl -X PUT https://demo.owncloud.com/remote.php/webdav/testcs.bin -u demo:demo -d '123' -v -H 'OC-Checksum: SHA1:40bd001563085fc35165329ea1ff5c5ecbdbbeef'
// curl -X PUT https://demo.example.org/remote.php/webdav/testcs.bin -u demo:demo -d '123' -v -H 'OC-Checksum: SHA1:40bd001563085fc35165329ea1ff5c5ecbdbbeef'
// TODO check Expect: 100-continue
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/sharees/sharees.go
Generated Vendored
+1 -1
View File
@@ -35,7 +35,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
)
// Handler implements the ownCloud sharing API
// Handler implements the open collaboration service sharing API
type Handler struct {
gatewayAddr string
additionalInfoAttribute string
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go
Generated Vendored
+1 -1
View File
@@ -75,7 +75,7 @@ var (
errParsingSpaceReference = errors.New("could not parse space reference")
)
// Handler implements the shares part of the ownCloud sharing API
// Handler implements the shares part of the open collaboration service sharing API
type Handler struct {
gatewayAddr string
machineAuthAPIKey string
vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/user/user.go
Generated Vendored
+1 -1
View File
@@ -52,7 +52,7 @@ func (h *Handler) GetSelf(w http.ResponseWriter, r *http.Request) {
// User holds user data
type User struct {
ID string `json:"id" xml:"id"` // UserID in ocs is the owncloud internal username
ID string `json:"id" xml:"id"` // UserID in ocs is the username
DisplayName string `json:"display-name" xml:"display-name"` // is used in ocs/v(1|2).php/cloud/user - yes this is different from the users endpoint
Email string `json:"email" xml:"email"`
UserType string `json:"user-type" xml:"user-type"`
vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/xcloud/xcloud.go
Generated Vendored
+1 -1
View File
@@ -59,7 +59,7 @@ type CloudDriver struct {
}
func (d *CloudDriver) refresh() error {
// endpoint example: https://mybox.com or https://mybox.com/owncloud
// endpoint example: https://mybox.com
endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance)
req, err := http.NewRequest("GET", endpoint, nil)
vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/ocm.go
Generated Vendored
+1 -1
View File
@@ -294,7 +294,7 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc
}
if t == provider.ResourceType_RESOURCE_TYPE_FILE {
// get SHA1 checksum from owncloud specific properties if available
// get SHA1 checksum from OpenCloud specific properties if available
propstat := webdavFile.Sys().(gowebdav.Props)
ri.Checksum = extractChecksum(propstat)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/upload.go
Generated Vendored
+1 -1
View File
@@ -255,7 +255,7 @@ func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (i
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
// However, for the ocis driver it's not important whether the stream has ended
// However, for the ocm driver it's not important whether the stream has ended
// on purpose or accidentally.
if err != nil && err != io.ErrUnexpectedEOF {
return n, err
vendor/github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/cs3/cs3.go
Generated Vendored
-620
View File
@@ -1,620 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package cs3
import (
"context"
"encoding/json"
"fmt"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"golang.org/x/crypto/bcrypt"
"google.golang.org/protobuf/proto"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/publicshare"
"github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer"
indexerErrors "github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
func init() {
registry.Register("cs3", NewDefault)
}
// Manager implements a publicshare manager using a cs3 storage backend
type Manager struct {
gatewayClient gateway.GatewayAPIClient
sync.RWMutex
storage metadata.Storage
indexer indexer.Indexer
passwordHashCost int
initialized bool
}
type config struct {
GatewayAddr string `mapstructure:"gateway_addr"`
ProviderAddr string `mapstructure:"provider_addr"`
ServiceUserID string `mapstructure:"service_user_id"`
ServiceUserIdp string `mapstructure:"service_user_idp"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
}
// NewDefault returns a new manager instance with default dependencies
func NewDefault(m map[string]interface{}) (publicshare.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error creating a new manager")
return nil, err
}
s, err := metadata.NewCS3Storage(c.GatewayAddr, c.ProviderAddr, c.ServiceUserID, c.ServiceUserIdp, c.MachineAuthAPIKey)
if err != nil {
return nil, err
}
indexer := indexer.CreateIndexer(s)
client, err := pool.GetGatewayServiceClient(c.GatewayAddr)
if err != nil {
return nil, err
}
return New(client, s, indexer, bcrypt.DefaultCost)
}
// New returns a new manager instance
func New(gatewayClient gateway.GatewayAPIClient, storage metadata.Storage, indexer indexer.Indexer, passwordHashCost int) (*Manager, error) {
return &Manager{
gatewayClient: gatewayClient,
storage: storage,
indexer: indexer,
passwordHashCost: passwordHashCost,
initialized: false,
}, nil
}
func (m *Manager) initialize() error {
if m.initialized {
return nil
}
m.Lock()
defer m.Unlock()
if m.initialized { // check if initialization happened while grabbing the lock
return nil
}
err := m.storage.Init(context.Background(), "public-share-manager-metadata")
if err != nil {
return err
}
if err := m.storage.MakeDirIfNotExist(context.Background(), "publicshares"); err != nil {
return err
}
err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByField("Id.OpaqueId"), "Token", "publicshares", "unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByFunc{
Name: "Owner",
Func: indexOwnerFunc,
}, "Token", "publicshares", "non_unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByFunc{
Name: "Creator",
Func: indexCreatorFunc,
}, "Token", "publicshares", "non_unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&link.PublicShare{}, option.IndexByFunc{
Name: "ResourceId",
Func: indexResourceIDFunc,
}, "Token", "publicshares", "non_unique", nil, true)
if err != nil {
return err
}
m.initialized = true
return nil
}
// Dump exports public shares to channels (e.g. during migration)
func (m *Manager) Dump(ctx context.Context, shareChan chan<- *publicshare.WithPassword) error {
if err := m.initialize(); err != nil {
return err
}
pshares, err := m.storage.ListDir(ctx, "publicshares")
if err != nil {
return err
}
for _, v := range pshares {
var local publicshare.WithPassword
ps, err := m.getByToken(ctx, v.Name)
if err != nil {
return err
}
local.Password = ps.Password
proto.Merge(&local.PublicShare, &ps.PublicShare)
shareChan <- &local
}
return nil
}
// Load imports public shares and received shares from channels (e.g. during migration)
func (m *Manager) Load(ctx context.Context, shareChan <-chan *publicshare.WithPassword) error {
log := appctx.GetLogger(ctx)
if err := m.initialize(); err != nil {
return err
}
for ps := range shareChan {
if err := m.persist(context.Background(), ps); err != nil {
log.Error().Err(err).Interface("publicshare", ps).Msg("error loading public share")
}
}
return nil
}
// CreatePublicShare creates a new public share
func (m *Manager) CreatePublicShare(ctx context.Context, u *user.User, ri *provider.ResourceInfo, g *link.Grant) (*link.PublicShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
id := &link.PublicShareId{
OpaqueId: utils.RandString(15),
}
tkn := utils.RandString(15)
now := time.Now().UnixNano()
displayName, quicklink := tkn, false
if ri.ArbitraryMetadata != nil {
metadataName, ok := ri.ArbitraryMetadata.Metadata["name"]
if ok {
displayName = metadataName
}
quicklink, _ = strconv.ParseBool(ri.ArbitraryMetadata.Metadata["quicklink"])
}
var passwordProtected bool
password := g.Password
if password != "" {
h, err := bcrypt.GenerateFromPassword([]byte(password), m.passwordHashCost)
if err != nil {
return nil, errors.Wrap(err, "could not hash share password")
}
password = string(h)
passwordProtected = true
}
createdAt := &typespb.Timestamp{
Seconds: uint64(now / int64(time.Second)),
Nanos: uint32(now % int64(time.Second)),
}
s := &publicshare.WithPassword{
PublicShare: link.PublicShare{
Id: id,
Owner: ri.GetOwner(),
Creator: u.Id,
ResourceId: ri.Id,
Token: tkn,
Permissions: g.Permissions,
Ctime: createdAt,
Mtime: createdAt,
PasswordProtected: passwordProtected,
Expiration: g.Expiration,
DisplayName: displayName,
Quicklink: quicklink,
},
Password: password,
}
err := m.persist(ctx, s)
if err != nil {
return nil, err
}
return &s.PublicShare, nil
}
// UpdatePublicShare updates an existing public share
func (m *Manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
ps, err := m.getWithPassword(ctx, req.Ref)
if err != nil {
return nil, err
}
switch req.Update.Type {
case link.UpdatePublicShareRequest_Update_TYPE_DISPLAYNAME:
ps.PublicShare.DisplayName = req.Update.DisplayName
case link.UpdatePublicShareRequest_Update_TYPE_PERMISSIONS:
ps.PublicShare.Permissions = req.Update.Grant.Permissions
case link.UpdatePublicShareRequest_Update_TYPE_EXPIRATION:
ps.PublicShare.Expiration = req.Update.Grant.Expiration
case link.UpdatePublicShareRequest_Update_TYPE_PASSWORD:
if req.Update.Grant.Password == "" {
ps.Password = ""
ps.PublicShare.PasswordProtected = false
} else {
h, err := bcrypt.GenerateFromPassword([]byte(req.Update.Grant.Password), m.passwordHashCost)
if err != nil {
return nil, errors.Wrap(err, "could not hash share password")
}
ps.Password = string(h)
ps.PublicShare.PasswordProtected = true
}
default:
return nil, errtypes.BadRequest("no valid update type given")
}
err = m.persist(ctx, ps)
if err != nil {
return nil, err
}
return &ps.PublicShare, nil
}
// GetPublicShare returns an existing public share
func (m *Manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference, sign bool) (*link.PublicShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
ps, err := m.getWithPassword(ctx, ref)
if err != nil {
return nil, err
}
if ps.PublicShare.PasswordProtected && sign {
err = publicshare.AddSignature(&ps.PublicShare, ps.Password)
if err != nil {
return nil, err
}
}
return &ps.PublicShare, nil
}
func (m *Manager) getWithPassword(ctx context.Context, ref *link.PublicShareReference) (*publicshare.WithPassword, error) {
switch {
case ref.GetToken() != "":
return m.getByToken(ctx, ref.GetToken())
case ref.GetId().GetOpaqueId() != "":
return m.getByID(ctx, ref.GetId().GetOpaqueId())
default:
return nil, errtypes.BadRequest("neither id nor token given")
}
}
func (m *Manager) getByID(ctx context.Context, id string) (*publicshare.WithPassword, error) {
tokens, err := m.indexer.FindBy(&link.PublicShare{},
indexer.NewField("Id.OpaqueId", id),
)
if err != nil {
return nil, err
}
if len(tokens) == 0 {
return nil, errtypes.NotFound("publicshare with the given id not found")
}
return m.getByToken(ctx, tokens[0])
}
func (m *Manager) getByToken(ctx context.Context, token string) (*publicshare.WithPassword, error) {
fn := path.Join("publicshares", token)
data, err := m.storage.SimpleDownload(ctx, fn)
if err != nil {
return nil, err
}
ps := &publicshare.WithPassword{}
err = json.Unmarshal(data, ps)
if err != nil {
return nil, err
}
id := storagespace.UpdateLegacyResourceID(ps.PublicShare.ResourceId)
ps.PublicShare.ResourceId = id
return ps, nil
}
// ListPublicShares lists existing public shares matching the given filters
func (m *Manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
log := appctx.GetLogger(ctx)
var rIDs []*provider.ResourceId
if len(filters) != 0 {
grouped := publicshare.GroupFiltersByType(filters)
for _, g := range grouped {
for _, f := range g {
if f.GetResourceId() != nil {
rIDs = append(rIDs, f.GetResourceId())
}
}
}
}
var (
createdShareTokens []string
err error
)
// in spaces, always use the resourceId
if len(rIDs) != 0 {
for _, rID := range rIDs {
shareTokens, err := m.indexer.FindBy(&link.PublicShare{},
indexer.NewField("ResourceId", resourceIDToIndex(rID)),
)
if err != nil {
return nil, err
}
createdShareTokens = append(createdShareTokens, shareTokens...)
}
} else {
// fallback for legacy use
createdShareTokens, err = m.indexer.FindBy(&link.PublicShare{},
indexer.NewField("Owner", userIDToIndex(u.Id)),
indexer.NewField("Creator", userIDToIndex(u.Id)),
)
if err != nil {
return nil, err
}
}
// We use shareMem as a temporary lookup store to check which shares were
// already added. This is to prevent duplicates.
shareMem := make(map[string]struct{})
result := []*link.PublicShare{}
for _, token := range createdShareTokens {
ps, err := m.getByToken(ctx, token)
if err != nil {
return nil, err
}
if !publicshare.MatchesFilters(&ps.PublicShare, filters) {
continue
}
if publicshare.IsExpired(&ps.PublicShare) {
ref := &link.PublicShareReference{
Spec: &link.PublicShareReference_Id{
Id: ps.PublicShare.Id,
},
}
if err := m.RevokePublicShare(ctx, u, ref); err != nil {
log.Error().Err(err).
Str("public_share_token", ps.PublicShare.Token).
Str("public_share_id", ps.PublicShare.Id.OpaqueId).
Msg("failed to revoke expired public share")
}
continue
}
if ps.PublicShare.PasswordProtected && sign {
err = publicshare.AddSignature(&ps.PublicShare, ps.Password)
if err != nil {
return nil, err
}
}
result = append(result, &ps.PublicShare)
shareMem[ps.PublicShare.Token] = struct{}{}
}
// If a user requests to list shares which have not been created by them
// we have to explicitly fetch these shares and check if the user is
// allowed to list the shares.
// Only then can we add these shares to the result.
grouped := publicshare.GroupFiltersByType(filters)
idFilter, ok := grouped[link.ListPublicSharesRequest_Filter_TYPE_RESOURCE_ID]
if !ok {
return result, nil
}
var tokens []string
if len(idFilter) > 0 {
idFilters := make([]indexer.Field, 0, len(idFilter))
for _, filter := range idFilter {
resourceID := filter.GetResourceId()
idFilters = append(idFilters, indexer.NewField("ResourceId", resourceIDToIndex(resourceID)))
}
tokens, err = m.indexer.FindBy(&link.PublicShare{}, idFilters...)
if err != nil {
return nil, err
}
}
// statMem is used as a local cache to prevent statting resources which
// already have been checked.
statMem := make(map[string]struct{})
for _, token := range tokens {
if _, handled := shareMem[token]; handled {
// We don't want to add a share multiple times when we added it
// already.
continue
}
s, err := m.getByToken(ctx, token)
if err != nil {
return nil, err
}
if _, checked := statMem[resourceIDToIndex(s.PublicShare.GetResourceId())]; !checked {
sReq := &provider.StatRequest{
Ref: &provider.Reference{ResourceId: s.PublicShare.GetResourceId()},
}
sRes, err := m.gatewayClient.Stat(ctx, sReq)
if err != nil {
continue
}
if sRes.Status.Code != rpc.Code_CODE_OK {
continue
}
if !sRes.Info.PermissionSet.ListGrants {
continue
}
statMem[resourceIDToIndex(s.PublicShare.GetResourceId())] = struct{}{}
}
if publicshare.MatchesFilters(&s.PublicShare, filters) {
result = append(result, &s.PublicShare)
shareMem[s.PublicShare.Token] = struct{}{}
}
}
return result, nil
}
// RevokePublicShare revokes an existing public share
func (m *Manager) RevokePublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference) error {
if err := m.initialize(); err != nil {
return err
}
ps, err := m.GetPublicShare(ctx, u, ref, false)
if err != nil {
return err
}
err = m.storage.Delete(ctx, path.Join("publicshares", ps.Token))
if err != nil {
if _, ok := err.(errtypes.NotFound); !ok {
return err
}
}
return m.indexer.Delete(ps)
}
// GetPublicShareByToken gets an existing public share in an unauthenticated context using either a password or a signature
func (m *Manager) GetPublicShareByToken(ctx context.Context, token string, auth *link.PublicShareAuthentication, sign bool) (*link.PublicShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
ps, err := m.getByToken(ctx, token)
if err != nil {
return nil, err
}
if publicshare.IsExpired(&ps.PublicShare) {
return nil, errtypes.NotFound("public share has expired")
}
if ps.PublicShare.PasswordProtected {
if !publicshare.Authenticate(&ps.PublicShare, ps.Password, auth) {
return nil, errtypes.InvalidCredentials("access denied")
}
}
return &ps.PublicShare, nil
}
func indexOwnerFunc(v interface{}) (string, error) {
ps, ok := v.(*link.PublicShare)
if !ok {
return "", fmt.Errorf("given entity is not a public share")
}
return userIDToIndex(ps.Owner), nil
}
func indexCreatorFunc(v interface{}) (string, error) {
ps, ok := v.(*link.PublicShare)
if !ok {
return "", fmt.Errorf("given entity is not a public share")
}
return userIDToIndex(ps.Creator), nil
}
func indexResourceIDFunc(v interface{}) (string, error) {
ps, ok := v.(*link.PublicShare)
if !ok {
return "", fmt.Errorf("given entity is not a public share")
}
return resourceIDToIndex(ps.ResourceId), nil
}
func userIDToIndex(id *user.UserId) string {
return url.QueryEscape(id.Idp + ":" + id.OpaqueId)
}
func resourceIDToIndex(id *provider.ResourceId) string {
return strings.Join([]string{id.StorageId, id.OpaqueId}, "!")
}
func (m *Manager) persist(ctx context.Context, ps *publicshare.WithPassword) error {
data, err := json.Marshal(ps)
if err != nil {
return err
}
fn := path.Join("publicshares", ps.PublicShare.Token)
err = m.storage.SimpleUpload(ctx, fn, data)
if err != nil {
return err
}
_, err = m.indexer.Add(&ps.PublicShare)
if err != nil {
if _, ok := err.(*indexerErrors.AlreadyExistsErr); ok {
return nil
}
err = m.indexer.Delete(&ps.PublicShare)
if err != nil {
return err
}
_, err = m.indexer.Add(&ps.PublicShare)
return err
}
return nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/loader/loader.go
Generated Vendored
-1
View File
@@ -20,7 +20,6 @@ package loader
import (
// Load core share manager drivers.
_ "github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/cs3"
_ "github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/json"
_ "github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/memory"
_ "github.com/opencloud-eu/reva/v2/pkg/publicshare/manager/owncloudsql"
vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/cs3/cs3.go
Generated Vendored
-887
View File
@@ -1,887 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package cs3
import (
"context"
"encoding/json"
"fmt"
"net/url"
"path"
"strings"
"sync"
gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
groupv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/google/uuid"
"github.com/mitchellh/mapstructure"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/share"
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer"
indexerErrors "github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"google.golang.org/genproto/protobuf/field_mask"
)
// Manager implements a share manager using a cs3 storage backend
type Manager struct {
gatewayClient gatewayv1beta1.GatewayAPIClient
sync.RWMutex
storage metadata.Storage
indexer indexer.Indexer
initialized bool
}
// ReceivedShareMetadata hold the state information or a received share
type ReceivedShareMetadata struct {
State collaboration.ShareState `json:"state"`
MountPoint *provider.Reference `json:"mountpoint"`
}
func init() {
registry.Register("cs3", NewDefault)
}
type config struct {
GatewayAddr string `mapstructure:"gateway_addr"`
ProviderAddr string `mapstructure:"provider_addr"`
ServiceUserID string `mapstructure:"service_user_id"`
ServiceUserIdp string `mapstructure:"service_user_idp"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
}
// NewDefault returns a new manager instance with default dependencies
func NewDefault(m map[string]interface{}) (share.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error creating a new manager")
return nil, err
}
s, err := metadata.NewCS3Storage(c.GatewayAddr, c.ProviderAddr, c.ServiceUserID, c.ServiceUserIdp, c.MachineAuthAPIKey)
if err != nil {
return nil, err
}
indexer := indexer.CreateIndexer(s)
client, err := pool.GetGatewayServiceClient(c.GatewayAddr)
if err != nil {
return nil, err
}
return New(client, s, indexer)
}
// New returns a new manager instance
func New(gatewayClient gatewayv1beta1.GatewayAPIClient, s metadata.Storage, indexer indexer.Indexer) (*Manager, error) {
return &Manager{
gatewayClient: gatewayClient,
storage: s,
indexer: indexer,
initialized: false,
}, nil
}
func (m *Manager) initialize() error {
if m.initialized {
return nil
}
m.Lock()
defer m.Unlock()
if m.initialized { // check if initialization happened while grabbing the lock
return nil
}
err := m.storage.Init(context.Background(), "cs3-share-manager-metadata")
if err != nil {
return err
}
if err := m.storage.MakeDirIfNotExist(context.Background(), "shares"); err != nil {
return err
}
if err := m.storage.MakeDirIfNotExist(context.Background(), "metadata"); err != nil {
return err
}
err = m.indexer.AddIndex(&collaboration.Share{}, option.IndexByFunc{
Name: "OwnerId",
Func: indexOwnerFunc,
}, "Id.OpaqueId", "shares", "non_unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&collaboration.Share{}, option.IndexByFunc{
Name: "CreatorId",
Func: indexCreatorFunc,
}, "Id.OpaqueId", "shares", "non_unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&collaboration.Share{}, option.IndexByFunc{
Name: "GranteeId",
Func: indexGranteeFunc,
}, "Id.OpaqueId", "shares", "non_unique", nil, true)
if err != nil {
return err
}
err = m.indexer.AddIndex(&collaboration.Share{}, option.IndexByFunc{
Name: "ResourceId",
Func: indexResourceIDFunc,
}, "Id.OpaqueId", "shares", "non_unique", nil, true)
if err != nil {
return err
}
m.initialized = true
return nil
}
// Load imports shares and received shares from channels (e.g. during migration)
func (m *Manager) Load(ctx context.Context, shareChan <-chan *collaboration.Share, receivedShareChan <-chan share.ReceivedShareWithUser) error {
log := appctx.GetLogger(ctx)
if err := m.initialize(); err != nil {
return err
}
var mu sync.Mutex
var wg sync.WaitGroup
wg.Add(2)
go func() {
for s := range shareChan {
if s == nil {
continue
}
mu.Lock()
if err := m.persistShare(context.Background(), s); err != nil {
log.Error().Err(err).Interface("share", s).Msg("error persisting share")
}
mu.Unlock()
}
wg.Done()
}()
go func() {
for s := range receivedShareChan {
if s.ReceivedShare != nil && s.UserID != nil {
mu.Lock()
if err := m.persistReceivedShare(context.Background(), s.UserID, s.ReceivedShare); err != nil {
log.Error().Err(err).Interface("received share", s).Msg("error persisting received share")
}
mu.Unlock()
}
}
wg.Done()
}()
wg.Wait()
return nil
}
func (m *Manager) getMetadata(ctx context.Context, shareid, grantee string) ReceivedShareMetadata {
// use default values if the grantee didn't configure anything yet
metadata := ReceivedShareMetadata{
State: collaboration.ShareState_SHARE_STATE_PENDING,
}
data, err := m.storage.SimpleDownload(ctx, path.Join("metadata", shareid, grantee))
if err != nil {
return metadata
}
err = json.Unmarshal(data, &metadata)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("shareid", shareid).Msg("error fetching share")
}
return metadata
}
// Dump exports shares and received shares to channels (e.g. during migration)
func (m *Manager) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error {
log := appctx.GetLogger(ctx)
if err := m.initialize(); err != nil {
return err
}
shareids, err := m.storage.ReadDir(ctx, "shares")
if err != nil {
return err
}
for _, shareid := range shareids {
var s *collaboration.Share
if s, err = m.getShareByID(ctx, shareid); err != nil {
log.Error().Err(err).Str("shareid", shareid).Msg("error fetching share")
continue
}
// dump share data
shareChan <- s
// dump grantee metadata that includes share state and mount path
grantees, err := m.storage.ReadDir(ctx, path.Join("metadata", s.Id.OpaqueId))
if err != nil {
continue
}
for _, grantee := range grantees {
metadata := m.getMetadata(ctx, s.GetId().GetOpaqueId(), grantee)
g, err := indexToGrantee(grantee)
if err != nil || g.Type != provider.GranteeType_GRANTEE_TYPE_USER {
// ignore group grants, as every user has his own received state
continue
}
receivedShareChan <- share.ReceivedShareWithUser{
UserID: g.GetUserId(),
ReceivedShare: &collaboration.ReceivedShare{
Share: s,
State: metadata.State,
MountPoint: metadata.MountPoint,
},
}
}
}
return nil
}
// Share creates a new share
func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) {
if err := m.initialize(); err != nil {
return nil, err
}
user := ctxpkg.ContextMustGetUser(ctx)
// do not allow share to myself or the owner if share is for a user
if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER &&
(utils.UserEqual(g.Grantee.GetUserId(), user.Id) || utils.UserEqual(g.Grantee.GetUserId(), md.Owner)) {
return nil, errtypes.BadRequest("cs3: owner/creator and grantee are the same")
}
ts := utils.TSNow()
share := &collaboration.Share{
Id: &collaboration.ShareId{
OpaqueId: uuid.NewString(),
},
ResourceId: md.Id,
Permissions: g.Permissions,
Grantee: g.Grantee,
Owner: md.Owner,
Creator: user.Id,
Ctime: ts,
Mtime: ts,
}
err := m.persistShare(ctx, share)
return share, err
}
func (m *Manager) persistShare(ctx context.Context, share *collaboration.Share) error {
data, err := json.Marshal(share)
if err != nil {
return err
}
err = m.storage.SimpleUpload(ctx, shareFilename(share.Id.OpaqueId), data)
if err != nil {
return err
}
metadataPath := path.Join("metadata", share.Id.OpaqueId)
err = m.storage.MakeDirIfNotExist(ctx, metadataPath)
if err != nil {
return err
}
_, err = m.indexer.Add(share)
if _, ok := err.(*indexerErrors.AlreadyExistsErr); ok {
return nil
}
return err
}
// GetShare gets the information for a share by the given ref.
func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) {
err := m.initialize()
if err != nil {
return nil, err
}
var s *collaboration.Share
switch {
case ref.GetId() != nil:
s, err = m.getShareByID(ctx, ref.GetId().OpaqueId)
case ref.GetKey() != nil:
s, err = m.getShareByKey(ctx, ref.GetKey())
default:
return nil, errtypes.BadRequest("neither share id nor key was given")
}
if err != nil {
return nil, err
}
// check if we are the owner or the grantee
user := ctxpkg.ContextMustGetUser(ctx)
if user.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE || share.IsCreatedByUser(s, user) || share.IsGrantedToUser(s, user) {
return s, nil
}
return nil, errtypes.NotFound("not found")
}
// Unshare deletes the share pointed by ref.
func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error {
if err := m.initialize(); err != nil {
return err
}
share, err := m.GetShare(ctx, ref)
if err != nil {
return err
}
err = m.storage.Delete(ctx, shareFilename(ref.GetId().OpaqueId))
if err != nil {
if _, ok := err.(errtypes.NotFound); !ok {
return err
}
}
return m.indexer.Delete(share)
}
// ListShares returns the shares created by the user
func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) {
if err := m.initialize(); err != nil {
return nil, err
}
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return nil, errtypes.UserRequired("error getting user from context")
}
var rIDs []*provider.ResourceId
if len(filters) != 0 {
grouped := share.GroupFiltersByType(filters)
for _, g := range grouped {
for _, f := range g {
if f.GetResourceId() != nil {
rIDs = append(rIDs, f.GetResourceId())
}
}
}
}
var (
createdShareIds []string
err error
)
// in spaces, always use the resourceId
// We could have more than one resourceID
// which would form a logical OR
if len(rIDs) != 0 {
for _, rID := range rIDs {
shareIDs, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("ResourceId", resourceIDToIndex(rID)),
)
if err != nil {
return nil, err
}
createdShareIds = append(createdShareIds, shareIDs...)
}
} else {
createdShareIds, err = m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("OwnerId", userIDToIndex(user.Id)),
indexer.NewField("CreatorId", userIDToIndex(user.Id)),
)
if err != nil {
return nil, err
}
}
// We use shareMem as a temporary lookup store to check which shares were
// already added. This is to prevent duplicates.
shareMem := make(map[string]struct{})
result := []*collaboration.Share{}
for _, id := range createdShareIds {
s, err := m.getShareByID(ctx, id)
if err != nil {
return nil, err
}
if share.MatchesFilters(s, filters) {
result = append(result, s)
shareMem[s.Id.OpaqueId] = struct{}{}
}
}
// If a user requests to list shares which have not been created by them
// we have to explicitly fetch these shares and check if the user is
// allowed to list the shares.
// Only then can we add these shares to the result.
grouped := share.GroupFiltersByType(filters)
idFilter, ok := grouped[collaboration.Filter_TYPE_RESOURCE_ID]
if !ok {
return result, nil
}
shareIDsByResourceID := make(map[string]*provider.ResourceId)
for _, filter := range idFilter {
resourceID := filter.GetResourceId()
shareIDs, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("ResourceId", resourceIDToIndex(resourceID)),
)
if err != nil {
continue
}
for _, shareID := range shareIDs {
shareIDsByResourceID[shareID] = resourceID
}
}
// statMem is used as a local cache to prevent statting resources which
// already have been checked.
statMem := make(map[string]struct{})
for shareID, resourceID := range shareIDsByResourceID {
if _, handled := shareMem[shareID]; handled {
// We don't want to add a share multiple times when we added it
// already.
continue
}
if _, checked := statMem[resourceIDToIndex(resourceID)]; !checked {
sReq := &provider.StatRequest{
Ref: &provider.Reference{ResourceId: resourceID},
}
sRes, err := m.gatewayClient.Stat(ctx, sReq)
if err != nil {
continue
}
if sRes.Status.Code != rpcv1beta1.Code_CODE_OK {
continue
}
if !sRes.Info.PermissionSet.ListGrants {
continue
}
statMem[resourceIDToIndex(resourceID)] = struct{}{}
}
s, err := m.getShareByID(ctx, shareID)
if err != nil {
return nil, err
}
if share.MatchesFilters(s, filters) {
result = append(result, s)
shareMem[s.Id.OpaqueId] = struct{}{}
}
}
return result, nil
}
// UpdateShare updates the mode of the given share.
func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions, updated *collaboration.Share, fieldMask *field_mask.FieldMask) (*collaboration.Share, error) {
if err := m.initialize(); err != nil {
return nil, err
}
share, err := m.GetShare(ctx, ref)
if err != nil {
return nil, err
}
share.Permissions = p
data, err := json.Marshal(share)
if err != nil {
return nil, err
}
err = m.storage.SimpleUpload(ctx, shareFilename(share.Id.OpaqueId), data)
return share, err
}
// ListReceivedShares returns the list of shares the user has access to.
func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter, forUser *userpb.UserId) ([]*collaboration.ReceivedShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return nil, errtypes.UserRequired("error getting user from context")
}
uid, groups := user.GetId(), user.GetGroups()
if user.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE {
u, err := utils.GetUser(forUser, m.gatewayClient)
if err != nil {
return nil, errtypes.BadRequest("user not found")
}
uid = forUser
groups = u.GetGroups()
}
result := []*collaboration.ReceivedShare{}
ids, err := granteeToIndex(&provider.Grantee{
Type: provider.GranteeType_GRANTEE_TYPE_USER,
Id: &provider.Grantee_UserId{UserId: uid},
})
if err != nil {
return nil, err
}
receivedIds, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("GranteeId", ids),
)
if err != nil {
return nil, err
}
for _, group := range groups {
index, err := granteeToIndex(&provider.Grantee{
Type: provider.GranteeType_GRANTEE_TYPE_GROUP,
Id: &provider.Grantee_GroupId{GroupId: &groupv1beta1.GroupId{OpaqueId: group}},
})
if err != nil {
return nil, err
}
groupIds, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("GranteeId", index),
)
if err != nil {
return nil, err
}
receivedIds = append(receivedIds, groupIds...)
}
for _, id := range receivedIds {
s, err := m.getShareByID(ctx, id)
if err != nil {
return nil, err
}
if !share.MatchesFilters(s, filters) {
continue
}
metadata, err := m.downloadMetadata(ctx, s)
if err != nil {
if _, ok := err.(errtypes.NotFound); !ok {
return nil, err
}
// use default values if the grantee didn't configure anything yet
metadata = ReceivedShareMetadata{
State: collaboration.ShareState_SHARE_STATE_PENDING,
}
}
result = append(result, &collaboration.ReceivedShare{
Share: s,
State: metadata.State,
MountPoint: metadata.MountPoint,
})
}
return result, nil
}
// GetReceivedShare returns the information for a received share.
func (m *Manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
share, err := m.GetShare(ctx, ref)
if err != nil {
return nil, err
}
metadata, err := m.downloadMetadata(ctx, share)
if err != nil {
if _, ok := err.(errtypes.NotFound); !ok {
return nil, err
}
// use default values if the grantee didn't configure anything yet
metadata = ReceivedShareMetadata{
State: collaboration.ShareState_SHARE_STATE_PENDING,
}
}
return &collaboration.ReceivedShare{
Share: share,
State: metadata.State,
MountPoint: metadata.MountPoint,
}, nil
}
// UpdateReceivedShare updates the received share with share state.
func (m *Manager) UpdateReceivedShare(ctx context.Context, rshare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask, forUser *userpb.UserId) (*collaboration.ReceivedShare, error) {
if err := m.initialize(); err != nil {
return nil, err
}
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return nil, errtypes.UserRequired("error getting user from context")
}
rs, err := m.GetReceivedShare(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: rshare.Share.Id}})
if err != nil {
return nil, err
}
for i := range fieldMask.Paths {
switch fieldMask.Paths[i] {
case "state":
rs.State = rshare.State
case "mount_point":
rs.MountPoint = rshare.MountPoint
case "hidden":
continue
default:
return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported")
}
}
uid := user.GetId()
if user.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE {
uid = forUser
}
err = m.persistReceivedShare(ctx, uid, rs)
if err != nil {
return nil, err
}
return rs, nil
}
func (m *Manager) persistReceivedShare(ctx context.Context, userID *userpb.UserId, rs *collaboration.ReceivedShare) error {
err := m.persistShare(ctx, rs.Share)
if err != nil {
return err
}
meta := ReceivedShareMetadata{
State: rs.State,
MountPoint: rs.MountPoint,
}
data, err := json.Marshal(meta)
if err != nil {
return err
}
fn, err := metadataFilename(rs.Share, userID)
if err != nil {
return err
}
return m.storage.SimpleUpload(ctx, fn, data)
}
func (m *Manager) downloadMetadata(ctx context.Context, share *collaboration.Share) (ReceivedShareMetadata, error) {
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return ReceivedShareMetadata{}, errtypes.UserRequired("error getting user from context")
}
metadataFn, err := metadataFilename(share, user.Id)
if err != nil {
return ReceivedShareMetadata{}, err
}
data, err := m.storage.SimpleDownload(ctx, metadataFn)
if err != nil {
return ReceivedShareMetadata{}, err
}
metadata := ReceivedShareMetadata{}
err = json.Unmarshal(data, &metadata)
return metadata, err
}
func (m *Manager) getShareByID(ctx context.Context, id string) (*collaboration.Share, error) {
data, err := m.storage.SimpleDownload(ctx, shareFilename(id))
if err != nil {
return nil, err
}
userShare := &collaboration.Share{
Grantee: &provider.Grantee{Id: &provider.Grantee_UserId{}},
}
err = json.Unmarshal(data, userShare)
if err == nil && userShare.Grantee.GetUserId() != nil {
userShare.ResourceId = storagespace.UpdateLegacyResourceID(userShare.GetResourceId())
return userShare, nil
}
groupShare := &collaboration.Share{
Grantee: &provider.Grantee{Id: &provider.Grantee_GroupId{}},
}
err = json.Unmarshal(data, groupShare) // try to unmarshal to a group share if the user share unmarshalling failed
if err == nil && groupShare.Grantee.GetGroupId() != nil {
groupShare.ResourceId = storagespace.UpdateLegacyResourceID(groupShare.GetResourceId())
return groupShare, nil
}
return nil, errtypes.InternalError("failed to unmarshal share data")
}
func (m *Manager) getShareByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) {
ownerIds, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("OwnerId", userIDToIndex(key.Owner)),
)
if err != nil {
return nil, err
}
granteeIndex, err := granteeToIndex(key.Grantee)
if err != nil {
return nil, err
}
granteeIds, err := m.indexer.FindBy(&collaboration.Share{},
indexer.NewField("GranteeId", granteeIndex),
)
if err != nil {
return nil, err
}
ids := intersectSlices(ownerIds, granteeIds)
for _, id := range ids {
share, err := m.getShareByID(ctx, id)
if err != nil {
return nil, err
}
if utils.ResourceIDEqual(share.ResourceId, key.ResourceId) {
return share, nil
}
}
return nil, errtypes.NotFound("share not found")
}
func shareFilename(id string) string {
return path.Join("shares", id)
}
func metadataFilename(s *collaboration.Share, g interface{}) (string, error) {
var granteePart string
switch v := g.(type) {
case *userpb.UserId:
granteePart = url.QueryEscape("user:" + v.Idp + ":" + v.OpaqueId)
case *provider.Grantee:
var err error
granteePart, err = granteeToIndex(v)
if err != nil {
return "", err
}
}
return path.Join("metadata", s.Id.OpaqueId, granteePart), nil
}
func indexOwnerFunc(v interface{}) (string, error) {
share, ok := v.(*collaboration.Share)
if !ok {
return "", fmt.Errorf("given entity is not a share")
}
return userIDToIndex(share.Owner), nil
}
func indexCreatorFunc(v interface{}) (string, error) {
share, ok := v.(*collaboration.Share)
if !ok {
return "", fmt.Errorf("given entity is not a share")
}
return userIDToIndex(share.Creator), nil
}
func userIDToIndex(id *userpb.UserId) string {
return url.QueryEscape(id.Idp + ":" + id.OpaqueId)
}
func indexGranteeFunc(v interface{}) (string, error) {
share, ok := v.(*collaboration.Share)
if !ok {
return "", fmt.Errorf("given entity is not a share")
}
return granteeToIndex(share.Grantee)
}
func indexResourceIDFunc(v interface{}) (string, error) {
share, ok := v.(*collaboration.Share)
if !ok {
return "", fmt.Errorf("given entity is not a share")
}
return resourceIDToIndex(share.ResourceId), nil
}
func resourceIDToIndex(id *provider.ResourceId) string {
return strings.Join([]string{id.SpaceId, id.OpaqueId}, "!")
}
func granteeToIndex(grantee *provider.Grantee) (string, error) {
switch {
case grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER:
return url.QueryEscape("user:" + grantee.GetUserId().Idp + ":" + grantee.GetUserId().OpaqueId), nil
case grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP:
return url.QueryEscape("group:" + grantee.GetGroupId().OpaqueId), nil
default:
return "", fmt.Errorf("unknown grantee type")
}
}
// indexToGrantee tries to unparse a grantee in a metadata dir
// unfortunately, it is just concatenated by :, causing nasty corner cases
func indexToGrantee(name string) (*provider.Grantee, error) {
unescaped, err := url.QueryUnescape(name)
if err != nil {
return nil, err
}
parts := strings.SplitN(unescaped, ":", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid grantee %s", unescaped)
}
switch parts[0] {
case "user":
lastInd := strings.LastIndex(parts[1], ":")
return &provider.Grantee{
Type: provider.GranteeType_GRANTEE_TYPE_USER,
Id: &provider.Grantee_UserId{
UserId: &userpb.UserId{
Idp: parts[1][:lastInd],
OpaqueId: parts[1][lastInd+1:],
},
},
}, nil
case "group":
return &provider.Grantee{
Type: provider.GranteeType_GRANTEE_TYPE_GROUP,
Id: &provider.Grantee_GroupId{
GroupId: &groupv1beta1.GroupId{
OpaqueId: parts[1],
},
},
}, nil
default:
return nil, fmt.Errorf("invalid grantee %s", unescaped)
}
}
func intersectSlices(a, b []string) []string {
aMap := map[string]bool{}
for _, s := range a {
aMap[s] = true
}
result := []string{}
for _, s := range b {
if _, ok := aMap[s]; ok {
result = append(result, s)
}
}
return result
}
vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/loader/loader.go
Generated Vendored
-1
View File
@@ -20,7 +20,6 @@ package loader
import (
// Load core share manager drivers.
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/cs3"
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/json"
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3"
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/memory"
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/cephfs/upload.go
Generated Vendored
+1 -1
View File
@@ -323,7 +323,7 @@ func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.R
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
// However, for OwnCloudStore it's not important whether the stream has ended
// However, for the driver it's not important whether the stream has ended
// on purpose or accidentally.
if err != nil {
if err != io.ErrUnexpectedEOF {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go
Generated Vendored
+28 -6
View File
@@ -1,4 +1,5 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -22,6 +23,7 @@ import (
"bufio"
"io"
"os"
"path/filepath"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/pkg/errors"
@@ -40,7 +42,7 @@ func New(root string) (*Blobstore, error) {
}
// Upload stores some data in the blobstore under the given key
func (bs *Blobstore) Upload(node *node.Node, source string) error {
func (bs *Blobstore) Upload(node *node.Node, source, copyTarget string) error {
path := node.InternalPath()
// preserve the mtime of the file
@@ -48,7 +50,7 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error {
file, err := os.Open(source)
if err != nil {
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload")
return errors.Wrap(err, "Decomposedfs: posix blobstore: Can not open source file to upload")
}
defer file.Close()
@@ -63,15 +65,35 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error {
if err != nil {
return errors.Wrapf(err, "could not write blob '%s'", node.InternalPath())
}
err = w.Flush()
if err != nil {
return err
}
if fi != nil {
return os.Chtimes(path, fi.ModTime(), fi.ModTime())
err = os.Chtimes(path, fi.ModTime(), fi.ModTime())
if err != nil {
return err
}
if copyTarget != "" {
// also "upload" the file to a local path, e.g. for keeping the "current" version of the file
err := os.MkdirAll(filepath.Dir(copyTarget), 0700)
if err != nil {
return err
}
file.Seek(0, 0)
copyFile, err := os.OpenFile(copyTarget, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return errors.Wrapf(err, "could not open copy target '%s' for writing", copyTarget)
}
defer copyFile.Close()
_, err = copyFile.ReadFrom(file)
if err != nil {
return errors.Wrapf(err, "could not write blob copy of '%s' to '%s'", node.InternalPath(), copyTarget)
}
}
return nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go
Generated Vendored
+32
View File
@@ -1,4 +1,5 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -45,8 +46,11 @@ import (
var tracer trace.Tracer
const RevisionsDir = ".oc-nodes"
var _spaceTypePersonal = "personal"
var _spaceTypeProject = "project"
var _currentSuffix = ".current"
func init() {
tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/pkg/decomposedfs/lookup")
@@ -310,11 +314,39 @@ func (lu *Lookup) InternalRoot() string {
// InternalPath returns the internal path for a given ID
func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
if strings.Contains(nodeID, node.RevisionIDDelimiter) {
spaceRoot, _ := lu.IDCache.Get(context.Background(), spaceID, spaceID)
if len(spaceRoot) == 0 {
return ""
}
return filepath.Join(spaceRoot, RevisionsDir, Pathify(nodeID, 4, 2))
}
path, _ := lu.IDCache.Get(context.Background(), spaceID, nodeID)
return path
}
// VersionPath returns the path to the version of the node
func (lu *Lookup) VersionPath(spaceID, nodeID, version string) string {
spaceRoot, _ := lu.IDCache.Get(context.Background(), spaceID, spaceID)
if len(spaceRoot) == 0 {
return ""
}
return filepath.Join(spaceRoot, RevisionsDir, Pathify(nodeID, 4, 2)+node.RevisionIDDelimiter+version)
}
// VersionPath returns the "current" path of the node
func (lu *Lookup) CurrentPath(spaceID, nodeID string) string {
spaceRoot, _ := lu.IDCache.Get(context.Background(), spaceID, spaceID)
if len(spaceRoot) == 0 {
return ""
}
return filepath.Join(spaceRoot, RevisionsDir, Pathify(nodeID, 4, 2)+_currentSuffix)
}
// // ReferenceFromAttr returns a CS3 reference from xattr of a node.
// // Supported formats are: "cs3:storageid/nodeid"
// func ReferenceFromAttr(b []byte) (*provider.Reference, error) {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options/options.go
Generated Vendored
+6
View File
@@ -33,6 +33,12 @@ type Options struct {
ScanDebounceDelay time.Duration `mapstructure:"scan_debounce_delay"`
// Allows generating revisions from changes done to the local storage.
// Note: This basically doubles the number of bytes stored on disk because
// a copy of the current version of a file is kept available for generating
// a revision when the file is changed.
EnableFSRevisions bool `mapstructure:"enable_fs_revisions"`
WatchFS bool `mapstructure:"watch_fs"`
WatchType string `mapstructure:"watch_type"`
WatchPath string `mapstructure:"watch_path"`
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go
Generated Vendored
+11 -12
View File
@@ -103,7 +103,13 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s
return nil, fmt.Errorf("the posix driver requires a shared id cache, e.g. nats-js-kv or redis")
}
tp, err := tree.New(lu, bs, um, trashbin, o, stream, store.Create(
permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode))
if err != nil {
return nil, err
}
p := permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector)
tp, err := tree.New(lu, bs, um, trashbin, p, o, stream, store.Create(
store.Store(o.IDCache.Store),
store.TTL(o.IDCache.TTL),
store.Size(o.IDCache.Size),
@@ -117,20 +123,13 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s
return nil, err
}
permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode))
if err != nil {
return nil, err
}
p := permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector)
aspects := aspects.Aspects{
Lookup: lu,
Tree: tp,
Permissions: p,
EventStream: stream,
UserMapper: um,
DisableVersioning: true,
DisableVersioning: false,
Trashbin: trashbin,
}
@@ -203,19 +202,19 @@ func (fs *posixFS) GetUpload(ctx context.Context, id string) (upload tusd.Upload
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
// the storage needs to implement AsTerminatableUpload
func (fs *posixFS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
// the storage needs to implement AsLengthDeclarableUpload
func (fs *posixFS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
// AsConcatableUpload returns a ConcatableUpload
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
// the storage needs to implement AsConcatableUpload
func (fs *posixFS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager/timemanager.go
Generated Vendored
+2 -1
View File
@@ -33,7 +33,8 @@ type Manager struct {
}
// OverrideMtime overrides the modification time (mtime) of a node with the specified time.
func (m *Manager) OverrideMtime(ctx context.Context, n *node.Node, _ *node.Attributes, mtime time.Time) error {
func (m *Manager) OverrideMtime(ctx context.Context, n *node.Node, attrs *node.Attributes, mtime time.Time) error {
attrs.SetTime(prefixes.MTimeAttr, mtime)
return os.Chtimes(n.InternalPath(), mtime, mtime)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go
Generated Vendored
+73 -3
View File
@@ -1,4 +1,5 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -21,6 +22,7 @@ package tree
import (
"context"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
@@ -67,7 +69,7 @@ type queueItem struct {
timer *time.Timer
}
const dirtyFlag = "user.ocis.dirty"
const dirtyFlag = "user.oc.dirty"
// NewScanDebouncer returns a new SpaceDebouncer instance
func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer {
@@ -375,7 +377,24 @@ func (t *Tree) assimilate(item scanItem) error {
}
// check for the id attribute again after grabbing the lock, maybe the file was assimilated/created by us in the meantime
id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr)
md, err := t.lookup.MetadataBackend().All(context.Background(), item.Path)
if err != nil {
return err
}
attrs := node.Attributes(md)
// compare metadata mtime with actual mtime. if it matches we can skip the assimilation because the file was handled by us
mtime, err := attrs.Time(prefixes.MTimeAttr)
if err == nil {
fi, err := os.Stat(item.Path)
if err == nil {
if mtime.Equal(fi.ModTime()) {
return nil
}
}
}
id = attrs[prefixes.IDAttr]
if err == nil {
previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), spaceID, string(id))
previousParentID, _ := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
@@ -576,11 +595,62 @@ assimilate:
}
attributes[prefixes.PropagationAttr] = []byte("1")
} else {
attributes.SetString(prefixes.BlobIDAttr, uuid.NewString())
attributes.SetInt64(prefixes.BlobsizeAttr, fi.Size())
attributes.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE))
}
n := node.New(spaceID, id, parentID, filepath.Base(path), fi.Size(), "", provider.ResourceType_RESOURCE_TYPE_FILE, nil, t.lookup)
n.SpaceRoot = &node.Node{SpaceID: spaceID, ID: spaceID}
go func() {
// Copy the previous current version to a revision
currentPath := t.lookup.(*lookup.Lookup).CurrentPath(n.SpaceID, n.ID)
stat, err := os.Stat(currentPath)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("currentPath", currentPath).Msg("could not stat current path")
return
}
revisionPath := t.lookup.VersionPath(n.SpaceID, n.ID, stat.ModTime().UTC().Format(time.RFC3339Nano))
err = os.Rename(currentPath, revisionPath)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("revisionPath", revisionPath).Msg("could not create revision")
return
}
// Copy the new version to the current version
w, err := os.OpenFile(currentPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("currentPath", currentPath).Msg("could not open current path for writing")
return
}
defer w.Close()
r, err := os.OpenFile(n.InternalPath(), os.O_RDONLY, 0600)
if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("could not open file for reading")
return
}
defer r.Close()
_, err = io.Copy(w, r)
if err != nil {
t.log.Error().Err(err).Str("currentPath", currentPath).Str("path", path).Msg("could not copy new version to current version")
return
}
err = t.lookup.CopyMetadata(context.Background(), n.InternalPath(), currentPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
if err != nil {
t.log.Error().Err(err).Str("currentPath", currentPath).Str("path", path).Msg("failed to copy xattrs to 'current' file")
return
}
}()
err = t.Propagate(context.Background(), n, 0)
if err != nil {
return nil, errors.Wrap(err, "failed to propagate")
@@ -623,7 +693,7 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error {
sizes := make(map[string]int64)
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
// skip lock and upload files
if isLockFile(path) {
if isInternal(path) || isLockFile(path) {
return nil
}
if isTrash(path) || t.isUpload(path) {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go
Generated Vendored
+1 -1
View File
@@ -82,7 +82,7 @@ start:
w.log.Error().Err(err).Str("line", line).Msg("error unmarshalling line")
continue
}
if isLockFile(ev.Path) || isTrash(ev.Path) || w.tree.isUpload(ev.Path) {
if w.tree.isIgnored(ev.Path) {
continue
}
go func() {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go
Generated Vendored
+2 -2
View File
@@ -46,7 +46,7 @@ func NewGpfsWatchFolderWatcher(tree *Tree, kafkaBrokers []string, log *zerolog.L
func (w *GpfsWatchFolderWatcher) Watch(topic string) {
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: w.brokers,
GroupID: "ocis-posixfs",
GroupID: "opencloud-posixfs",
Topic: topic,
})
@@ -62,7 +62,7 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) {
continue
}
if isLockFile(lwev.Path) || isTrash(lwev.Path) || w.tree.isUpload(lwev.Path) {
if w.tree.isIgnored(lwev.Path) {
continue
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go
Generated Vendored
+1 -1
View File
@@ -63,7 +63,7 @@ func (iw *InotifyWatcher) Watch(path string) {
for {
select {
case event := <-events:
if isLockFile(event.Filename) || isTrash(event.Filename) || iw.tree.isUpload(event.Filename) {
if iw.tree.isIgnored(event.Filename) {
continue
}
for _, e := range event.Events {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/revisions.go
Generated Vendored
+338
View File
@@ -0,0 +1,338 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package tree
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// Revision entries are stored inside the revisions directory in .oc-nodes, sharded by the node ids.
// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions
// can be kept in the same location.
// CreateRevision creates a new version of the node
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
versionPath := tp.lookup.VersionPath(n.SpaceID, n.ID, version)
err := os.MkdirAll(filepath.Dir(versionPath), 0700)
if err != nil {
return "", err
}
// copy file content to version node
sf, err := os.OpenFile(n.InternalPath(), os.O_RDONLY, 0)
if err != nil {
return "", err
}
defer sf.Close()
vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return "", err
}
defer vf.Close()
if _, err := io.Copy(vf, sf); err != nil {
return "", err
}
// copy blob metadata to version node
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr
}, f, true); err != nil {
return "", err
}
return versionPath, nil
}
func (tp *Tree) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) {
_, span := tracer.Start(ctx, "ListRevisions")
defer span.End()
var n *node.Node
if n, err = tp.lookup.NodeFromResource(ctx, ref); err != nil {
return
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return
}
rp, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, err
case !rp.ListFileVersions:
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, errtypes.PermissionDenied(f)
}
return nil, errtypes.NotFound(f)
}
revisions = []*provider.FileVersion{}
versionGlob := tp.lookup.VersionPath(n.SpaceID, n.ID, "*")
if items, err := filepath.Glob(versionGlob); err == nil {
for i := range items {
if tp.lookup.MetadataBackend().IsMetaFile(items[i]) || strings.HasSuffix(items[i], ".mlock") {
continue
}
if fi, err := os.Stat(items[i]); err == nil {
parts := strings.SplitN(fi.Name(), node.RevisionIDDelimiter, 2)
if len(parts) != 2 {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("invalid revision name, skipping")
continue
}
mtime := fi.ModTime()
rev := &provider.FileVersion{
Key: n.ID + node.RevisionIDDelimiter + parts[1],
Mtime: uint64(mtime.Unix()),
}
_, blobSize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, items[i], nil)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0")
}
rev.Size = uint64(blobSize)
etag, err := node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, errors.Wrapf(err, "error calculating etag")
}
rev.Etag = etag
revisions = append(revisions, rev)
}
}
}
// maybe we need to sort the list by key
/*
sort.Slice(revisions, func(i, j int) bool {
return revisions[i].Key > revisions[j].Key
})
*/
return
}
// DownloadRevision returns a reader for the specified revision
// FIXME the CS3 api should explicitly allow initiating revision and trash download, a related issue is https://github.com/cs3org/reva/issues/1813
func (tp *Tree) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) {
_, span := tracer.Start(ctx, "DownloadRevision")
defer span.End()
log := appctx.GetLogger(ctx)
// verify revision key format
kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2)
if len(kp) != 2 {
log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey")
return nil, nil, errtypes.NotFound(revisionKey)
}
log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision")
spaceID := ref.ResourceId.SpaceId
// check if the node is available and has not been deleted
n, err := node.ReadNode(ctx, tp.lookup, spaceID, kp[0], false, nil, false)
if err != nil {
return nil, nil, err
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return nil, nil, err
}
rp, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, nil, err
case !rp.ListFileVersions || !rp.InitiateFileDownload: // TODO add explicit permission in the CS3 api?
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, nil, errtypes.PermissionDenied(f)
}
return nil, nil, errtypes.NotFound(f)
}
contentPath := tp.lookup.InternalPath(spaceID, revisionKey)
_, blobsize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, contentPath, nil)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID)
}
revisionNode := node.New(spaceID, revisionKey, n.ParentID, n.Name, blobsize, "", provider.ResourceType_RESOURCE_TYPE_FILE, n.Owner(), tp.lookup)
ri, err := n.AsResourceInfo(ctx, rp, nil, []string{"size", "mimetype", "etag"}, true)
if err != nil {
return nil, nil, err
}
// update resource info with revision data
mtime, err := time.Parse(time.RFC3339Nano, kp[1])
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not parse mtime for revision '%s' of node '%s'", kp[1], n.ID)
}
ri.Size = uint64(blobsize)
ri.Mtime = utils.TimeToTS(mtime)
ri.Etag, err = node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, nil, errors.Wrapf(err, "error calculating etag for revision '%s' of node '%s'", kp[1], n.ID)
}
var reader io.ReadCloser
if openReaderFunc(ri) {
reader, err = tp.ReadBlob(revisionNode)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", n.ID, revisionKey)
}
}
return ri, reader, nil
}
func (tp *Tree) getRevisionNode(ctx context.Context, ref *provider.Reference, revisionKey string, hasPermission func(*provider.ResourcePermissions) bool) (*node.Node, error) {
_, span := tracer.Start(ctx, "getRevisionNode")
defer span.End()
log := appctx.GetLogger(ctx)
// verify revision key format
kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2)
if len(kp) != 2 {
log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey")
return nil, errtypes.NotFound(revisionKey)
}
log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision")
spaceID := ref.ResourceId.SpaceId
// check if the node is available and has not been deleted
n, err := node.ReadNode(ctx, tp.lookup, spaceID, kp[0], false, nil, false)
if err != nil {
return nil, err
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return nil, err
}
p, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, err
case !hasPermission(p):
return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name))
}
// Set space owner in context
storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx))
return n, nil
}
func (tp *Tree) RestoreRevision(ctx context.Context, spaceID, nodeID, source string) error {
target := tp.lookup.InternalPath(spaceID, nodeID)
rf, err := os.Open(source)
if err != nil {
return err
}
defer rf.Close()
wf, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer wf.Close()
wf.Truncate(0)
if _, err := io.Copy(wf, rf); err != nil {
return err
}
err = tp.lookup.CopyMetadata(ctx, source, target, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
}
// always set the node mtime to the current time
mtime := time.Now()
os.Chtimes(target, mtime, mtime)
err = tp.lookup.MetadataBackend().SetMultiple(ctx, target,
map[string][]byte{
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
},
false)
if err != nil {
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
}
// update "current" revision
if tp.options.EnableFSRevisions {
currentPath := tp.lookup.(*lookup.Lookup).CurrentPath(spaceID, nodeID)
w, err := os.OpenFile(currentPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
tp.log.Error().Err(err).Str("currentPath", currentPath).Str("source", source).Msg("could not open current path for writing")
return err
}
defer w.Close()
r, err := os.OpenFile(source, os.O_RDONLY, 0600)
if err != nil {
tp.log.Error().Err(err).Str("currentPath", currentPath).Str("source", source).Msg("could not open file for reading")
return err
}
defer r.Close()
_, err = io.Copy(w, r)
if err != nil {
tp.log.Error().Err(err).Str("currentPath", currentPath).Str("source", source).Msg("could not copy new version to current version")
return err
}
err = tp.lookup.CopyMetadata(ctx, source, currentPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
if err != nil {
return errtypes.InternalError("failed to copy xattrs to 'current' file: " + err.Error())
}
}
return nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go
Generated Vendored
+58 -24
View File
@@ -1,4 +1,5 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -27,6 +28,7 @@ import (
"path/filepath"
"regexp"
"strings"
"time"
"github.com/google/uuid"
"github.com/pkg/errors"
@@ -48,6 +50,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -61,7 +64,7 @@ func init() {
// Blobstore defines an interface for storing blobs in a blobstore
type Blobstore interface {
Upload(node *node.Node, source string) error
Upload(node *node.Node, source, copyTarget string) error
Download(node *node.Node) (io.ReadCloser, error)
Delete(node *node.Node) error
}
@@ -78,10 +81,11 @@ type scanItem struct {
// Tree manages a hierarchical tree
type Tree struct {
lookup node.PathLookup
blobstore Blobstore
trashbin *trashbin.Trashbin
propagator propagator.Propagator
lookup node.PathLookup
blobstore Blobstore
trashbin *trashbin.Trashbin
propagator propagator.Propagator
permissions permissions.Permissions
options *options.Options
@@ -99,17 +103,18 @@ type Tree struct {
type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool
// New returns a new instance of Tree
func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, trashbin *trashbin.Trashbin, o *options.Options, es events.Stream, cache store.Store, log *zerolog.Logger) (*Tree, error) {
func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, trashbin *trashbin.Trashbin, permissions permissions.Permissions, o *options.Options, es events.Stream, cache store.Store, log *zerolog.Logger) (*Tree, error) {
scanQueue := make(chan scanItem)
t := &Tree{
lookup: lu,
blobstore: bs,
userMapper: um,
trashbin: trashbin,
options: o,
idCache: cache,
propagator: propagator.New(lu, &o.Options, log),
scanQueue: scanQueue,
lookup: lu,
blobstore: bs,
userMapper: um,
trashbin: trashbin,
permissions: permissions,
options: o,
idCache: cache,
propagator: propagator.New(lu, &o.Options, log),
scanQueue: scanQueue,
scanDebouncer: NewScanDebouncer(o.ScanDebounceDelay, func(item scanItem) {
scanQueue <- item
}),
@@ -220,7 +225,7 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
if err := os.MkdirAll(filepath.Dir(nodePath), 0700); err != nil {
return errors.Wrap(err, "Decomposedfs: error creating node")
}
_, err = os.Create(nodePath)
f, err := os.Create(nodePath)
if err != nil {
return errors.Wrap(err, "Decomposedfs: error creating node")
}
@@ -235,10 +240,14 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
if err != nil {
return err
}
err = os.Chtimes(nodePath, nodeMTime, nodeMTime)
t.lookup.TimeManager().OverrideMtime(ctx, n, &attributes, nodeMTime)
} else {
fi, err := f.Stat()
if err != nil {
return err
}
mtime := fi.ModTime()
attributes[prefixes.MTimeAttr] = []byte(mtime.UTC().Format(time.RFC3339Nano))
}
err = n.SetXattrsWithContext(ctx, attributes, false)
@@ -397,7 +406,7 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
g.Go(func() error {
defer close(work)
for _, name := range names {
if isLockFile(name) || isTrash(name) {
if isInternal(name) || isLockFile(name) || isTrash(name) {
continue
}
@@ -676,7 +685,24 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err
// WriteBlob writes a blob to the blobstore
func (t *Tree) WriteBlob(node *node.Node, source string) error {
return t.blobstore.Upload(node, source)
var currentPath string
var err error
if t.options.EnableFSRevisions {
currentPath = t.lookup.(*lookup.Lookup).CurrentPath(node.SpaceID, node.ID)
defer func() {
_ = t.lookup.CopyMetadata(context.Background(), node.InternalPath(), currentPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
}()
}
err = t.blobstore.Upload(node, source, currentPath)
return err
}
// ReadBlob reads a blob from the blobstore
@@ -845,14 +871,22 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) (
return
}
func isLockFile(path string) bool {
return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock")
}
func isTrash(path string) bool {
return strings.HasSuffix(path, ".trashinfo") || strings.HasSuffix(path, ".trashitem")
func (t *Tree) isIgnored(path string) bool {
return isLockFile(path) || isTrash(path) || t.isUpload(path) || isInternal(path)
}
func (t *Tree) isUpload(path string) bool {
return strings.HasPrefix(path, t.options.UploadDirectory)
}
func isInternal(path string) bool {
return strings.Contains(path, lookup.RevisionsDir)
}
func isLockFile(path string) bool {
return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock")
}
func isTrash(path string) bool {
return strings.HasSuffix(path, ".trashinfo") || strings.HasSuffix(path, ".trashitem") || strings.Contains(path, ".Trash")
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go
Generated Vendored
+12 -11
View File
@@ -90,7 +90,7 @@ func init() {
tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs")
}
// Session is the interface that OcisSession implements. By combining tus.Upload,
// Session is the interface that DecomposedfsSession implements. By combining tus.Upload,
// storage.UploadSession and custom functions we can reuse the same struct throughout
// the whole upload lifecycle.
//
@@ -104,9 +104,9 @@ type Session interface {
}
type SessionStore interface {
New(ctx context.Context) *upload.OcisSession
List(ctx context.Context) ([]*upload.OcisSession, error)
Get(ctx context.Context, id string) (*upload.OcisSession, error)
New(ctx context.Context) *upload.DecomposedFsSession
List(ctx context.Context) ([]*upload.DecomposedFsSession, error)
Get(ctx context.Context, id string) (*upload.DecomposedFsSession, error)
Cleanup(ctx context.Context, session upload.Session, revertNodeMetadata, keepUpload, unmarkPostprocessing bool)
}
@@ -151,7 +151,13 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream, l
return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend)
}
tp := tree.New(lu, bs, o, store.Create(
permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode))
if err != nil {
return nil, err
}
p := permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector)
tp := tree.New(lu, bs, o, p, store.Create(
store.Store(o.IDCache.Store),
store.TTL(o.IDCache.TTL),
store.Size(o.IDCache.Size),
@@ -162,15 +168,10 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream, l
store.Authentication(o.IDCache.AuthUsername, o.IDCache.AuthPassword),
), log)
permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode))
if err != nil {
return nil, err
}
aspects := aspects.Aspects{
Lookup: lu,
Tree: tp,
Permissions: permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector),
Permissions: p,
EventStream: es,
DisableVersioning: o.DisableVersioning,
Trashbin: &DecomposedfsTrashbin{},
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup/lookup.go
Generated Vendored
+5
View File
@@ -309,6 +309,11 @@ func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2))
}
// VersionPath returns the internal path for a version of a node
func (lu *Lookup) VersionPath(spaceID, nodeID, version string) string {
return lu.InternalPath(spaceID, nodeID) + node.RevisionIDDelimiter + version
}
// // ReferenceFromAttr returns a CS3 reference from xattr of a node.
// // Supported formats are: "cs3:storageid/nodeid"
// func ReferenceFromAttr(b []byte) (*provider.Reference, error) {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes/ocis_prefix.go → vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes/oc_prefix.go
Generated Vendored
+4 -4
View File
@@ -20,10 +20,10 @@
package prefixes
// The default namespace for ocis. As non root users can only manipulate
// the user. namespace, which is what is used to store ownCloud specific
// metadata. To prevent name collisions with other apps, we are going to
// introduce a sub namespace "user.ocis."
// The default namespace for decomposedfs. As non root users can only
// manipulate the user. namespace, which is what is used to store decomposedfs
// specific metadata. To prevent name collisions with other apps, we are going
// to introduce a sub namespace "user.oc."
const (
OcPrefix string = "user.oc."
)
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes/ocis_prefix_freebsd.go → vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes/oc_prefix_freebsd.go
Generated Vendored
+1 -1
View File
@@ -24,5 +24,5 @@ package prefixes
// and will fail with invalid argument when you try to start an xattr name with user. or system.
// For that reason we drop the superfluous user. prefix for FreeBSD specifically.
const (
OcisPrefix string = "ocis."
OcPrefix string = "oc."
)
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes/prefixes.go
Generated Vendored
+6 -6
View File
@@ -19,13 +19,13 @@
package prefixes
// Declare a list of xattr keys
// TODO the below comment is currently copied from the owncloud driver, revisit
// Currently,extended file attributes have four separated
// namespaces (user, trusted, security and system) followed by a dot.
// A non root user can only manipulate the user. namespace, which is what
// we will use to store ownCloud specific metadata. To prevent name
// we will use to store decomposedfs specific metadata. To prevent name
// collisions with other apps We are going to introduce a sub namespace
// "user.ocis." in the xattrs_prefix*.go files.
// "user.oc." in the xattrs_prefix*.go files.
const (
TypeAttr string = OcPrefix + "type"
IDAttr string = OcPrefix + "id"
@@ -59,7 +59,7 @@ const (
// a temporary etag for a folder that is removed when the mtime propagation happens
TmpEtagAttr string = OcPrefix + "tmp.etag"
ReferenceAttr string = OcPrefix + "cs3.ref" // arbitrary metadata
ChecksumPrefix string = OcPrefix + "cs." // followed by the algorithm, eg. ocis.cs.sha1
ChecksumPrefix string = OcPrefix + "cs." // followed by the algorithm, eg. oc.cs.sha1
TrashOriginAttr string = OcPrefix + "trash.origin" // trash origin
// we use a single attribute to enable or disable propagation of both: synctime and treesize
@@ -71,7 +71,7 @@ const (
MTimeAttr string = OcPrefix + "mtime"
// the tree modification time of the tree below this node,
// propagated when synctime_accounting is true and
// user.ocis.propagation=1 is set
// user.oc.propagation=1 is set
// stored as a readable time.RFC3339Nano
TreeMTimeAttr string = OcPrefix + "tmtime"
@@ -82,7 +82,7 @@ const (
// the size of the tree below this node,
// propagated when treesize_accounting is true and
// user.ocis.propagation=1 is set
// user.oc.propagation=1 is set
// stored as uint64, little endian
TreesizeAttr string = OcPrefix + "treesize"
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go
Generated Vendored
+11 -4
View File
@@ -124,6 +124,7 @@ type Tree interface {
PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*Node, func() error, error)
InitNewNode(ctx context.Context, n *Node, fsize uint64) (metadata.UnlockFunc, error)
RestoreRevision(ctx context.Context, spaceID, nodeID, sourcePath string) (err error)
WriteBlob(node *Node, source string) error
ReadBlob(node *Node) (io.ReadCloser, error)
@@ -132,6 +133,10 @@ type Tree interface {
BuildSpaceIDIndexEntry(spaceID, nodeID string) string
ResolveSpaceIDIndexEntry(spaceID, entry string) (string, string, error)
CreateRevision(ctx context.Context, n *Node, version string, f *lockedfile.File) (string, error)
ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)
DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error)
Propagate(ctx context.Context, node *Node, sizeDiff int64) (err error)
}
@@ -147,6 +152,7 @@ type PathLookup interface {
InternalRoot() string
InternalPath(spaceID, nodeID string) string
VersionPath(spaceID, nodeID, version string) string
Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error)
MetadataBackend() metadata.Backend
TimeManager() TimeManager
@@ -356,7 +362,7 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis
// use the actual node for the metadata lookup
nodeID = kp[0]
// remember revision for blob metadata
revisionSuffix = RevisionIDDelimiter + kp[1]
revisionSuffix = kp[1]
}
}
@@ -372,8 +378,8 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis
// append back revision to nodeid, even when returning a not existing node
defer func() {
// when returning errors n is nil
if n != nil {
n.ID += revisionSuffix
if n != nil && revisionSuffix != "" {
n.ID += RevisionIDDelimiter + revisionSuffix
}
}()
@@ -402,7 +408,8 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis
return nil, err
}
} else {
n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, nodePath+revisionSuffix, nil)
versionPath := lu.VersionPath(spaceID, nodeID, revisionSuffix)
n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil)
if err != nil {
return nil, err
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/xattrs.go
Generated Vendored
+5
View File
@@ -66,6 +66,11 @@ func (md Attributes) Time(key string) (time.Time, error) {
return time.Parse(time.RFC3339Nano, string(md[key]))
}
// SetTime sets a time value
func (md Attributes) SetTime(key string, t time.Time) {
md[key] = []byte(t.UTC().Format(time.RFC3339Nano))
}
// SetXattrs sets multiple extended attributes on the write-through cache/node
func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte, acquireLock bool) (err error) {
_, span := tracer.Start(ctx, "SetXattrsWithContext")
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go
Generated Vendored
+2 -2
View File
@@ -47,7 +47,7 @@ type Options struct {
// Options specific to the async propagator
AsyncPropagatorOptions AsyncPropagatorOptions `mapstructure:"async_propagator_options"`
// ocis fs works on top of a dir of uuid nodes
// decomposedfs fs works on top of a dir of uuid nodes
Root string `mapstructure:"root"`
// the upload directory where uploads in progress are stored
@@ -59,7 +59,7 @@ type Options struct {
// ProjectLayout describes the relative path from the storage's root node to the project spaces root directory.
ProjectLayout string `mapstructure:"project_layout"`
// propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node
// propagate mtime changes as tmtime (tree modification time) to the parent directory when user.oc.propagation=1 is set on a node
TreeTimeAccounting bool `mapstructure:"treetime_accounting"`
// propagate size changes as treesize
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/recycle.go
Generated Vendored
+1 -1
View File
@@ -58,7 +58,7 @@ func (tb *DecomposedfsTrashbin) Setup(fs storage.FS) error {
// The deleted file is kept in the same location/dir as the original node. This prevents deletes
// from triggering cross storage moves when the trash is accidentally stored on another partition,
// because the admin mounted a different partition there.
// For an efficient listing of deleted nodes the ocis storage driver maintains a 'trash' folder
// For an efficient listing of deleted nodes the decomposedfs storage driver maintains a 'trash' folder
// with symlinks to trash files for every storagespace.
// ListRecycle returns the list of available recycle items
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/revisions.go
Generated Vendored
+11 -193
View File
@@ -27,7 +27,6 @@ import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
@@ -35,7 +34,6 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// Revision entries are stored inside the node folder and start with the same uuid as the current version.
@@ -46,146 +44,14 @@ import (
// We can add a background process to move old revisions to a slower storage
// and replace the revision file with a symbolic link in the future, if necessary.
// ListRevisions lists the revisions of the given resource
func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) {
_, span := tracer.Start(ctx, "ListRevisions")
defer span.End()
var n *node.Node
if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil {
return
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return
}
rp, err := fs.p.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, err
case !rp.ListFileVersions:
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, errtypes.PermissionDenied(f)
}
return nil, errtypes.NotFound(f)
}
revisions = []*provider.FileVersion{}
np := n.InternalPath()
if items, err := filepath.Glob(np + node.RevisionIDDelimiter + "*"); err == nil {
for i := range items {
if fs.lu.MetadataBackend().IsMetaFile(items[i]) || strings.HasSuffix(items[i], ".mlock") {
continue
}
if fi, err := os.Stat(items[i]); err == nil {
parts := strings.SplitN(fi.Name(), node.RevisionIDDelimiter, 2)
if len(parts) != 2 {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("invalid revision name, skipping")
continue
}
mtime := fi.ModTime()
rev := &provider.FileVersion{
Key: n.ID + node.RevisionIDDelimiter + parts[1],
Mtime: uint64(mtime.Unix()),
}
_, blobSize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, items[i], nil)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0")
}
rev.Size = uint64(blobSize)
etag, err := node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, errors.Wrapf(err, "error calculating etag")
}
rev.Etag = etag
revisions = append(revisions, rev)
}
}
}
// maybe we need to sort the list by key
/*
sort.Slice(revisions, func(i, j int) bool {
return revisions[i].Key > revisions[j].Key
})
*/
return
return fs.tp.ListRevisions(ctx, ref)
}
// DownloadRevision returns a reader for the specified revision
// FIXME the CS3 api should explicitly allow initiating revision and trash download, a related issue is https://github.com/cs3org/reva/issues/1813
func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) {
_, span := tracer.Start(ctx, "DownloadRevision")
defer span.End()
log := appctx.GetLogger(ctx)
// verify revision key format
kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2)
if len(kp) != 2 {
log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey")
return nil, nil, errtypes.NotFound(revisionKey)
}
log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision")
spaceID := ref.ResourceId.SpaceId
// check if the node is available and has not been deleted
n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false)
if err != nil {
return nil, nil, err
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return nil, nil, err
}
rp, err := fs.p.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, nil, err
case !rp.ListFileVersions || !rp.InitiateFileDownload: // TODO add explicit permission in the CS3 api?
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, nil, errtypes.PermissionDenied(f)
}
return nil, nil, errtypes.NotFound(f)
}
contentPath := fs.lu.InternalPath(spaceID, revisionKey)
blobid, blobsize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, contentPath, nil)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID)
}
revisionNode := node.Node{SpaceID: spaceID, BlobID: blobid, Blobsize: blobsize} // blobsize is needed for the s3ng blobstore
ri, err := n.AsResourceInfo(ctx, rp, nil, []string{"size", "mimetype", "etag"}, true)
if err != nil {
return nil, nil, err
}
// update resource info with revision data
mtime, err := time.Parse(time.RFC3339Nano, kp[1])
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not parse mtime for revision '%s' of node '%s'", kp[1], n.ID)
}
ri.Size = uint64(blobsize)
ri.Mtime = utils.TimeToTS(mtime)
ri.Etag, err = node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, nil, errors.Wrapf(err, "error calculating etag for revision '%s' of node '%s'", kp[1], n.ID)
}
var reader io.ReadCloser
if openReaderFunc(ri) {
reader, err = fs.tp.ReadBlob(&revisionNode)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", n.ID, revisionKey)
}
}
return ri, reader, nil
return fs.tp.DownloadRevision(ctx, ref, revisionKey, openReaderFunc)
}
// RestoreRevision restores the specified revision of the resource
@@ -250,67 +116,15 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
return err
}
// revisions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries
newRevisionPath := fs.lu.InternalPath(spaceID, kp[0]+node.RevisionIDDelimiter+mtime.UTC().Format(time.RFC3339Nano))
// touch new revision
if _, err := os.Create(newRevisionPath); err != nil {
// create a revision of the current node
if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano), f); err != nil {
return err
}
defer func() {
if returnErr != nil {
if err := os.Remove(newRevisionPath); err != nil {
log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node")
}
if err := fs.lu.MetadataBackend().Purge(ctx, newRevisionPath); err != nil {
log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node")
}
}
}()
// copy blob metadata from node to new revision node
err = fs.lu.CopyMetadataWithSourceLock(ctx, nodePath, newRevisionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || // for checksums
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr // FIXME somewhere I mix up the revision time and the mtime, causing the restore to overwrite the other existing revisien
}, f, true)
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to version node: " + err.Error())
}
// remember mtime from node as new revision mtime
if err = os.Chtimes(newRevisionPath, mtime, mtime); err != nil {
return errtypes.InternalError("failed to change mtime of version node")
}
// update blob id in node
// copy blob metadata from restored revision to node
// restore revision
restoredRevisionPath := fs.lu.InternalPath(spaceID, revisionKey)
err = fs.lu.CopyMetadata(ctx, restoredRevisionPath, nodePath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
}
// always set the node mtime to the current time
err = fs.lu.MetadataBackend().SetMultiple(ctx, nodePath,
map[string][]byte{
prefixes.MTimeAttr: []byte(time.Now().UTC().Format(time.RFC3339Nano)),
},
false)
if err != nil {
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
}
revisionSize, err := fs.lu.MetadataBackend().GetInt64(ctx, restoredRevisionPath, prefixes.BlobsizeAttr)
if err != nil {
return errtypes.InternalError("failed to read blob size xattr from old revision")
if err := fs.tp.RestoreRevision(ctx, spaceID, kp[0], restoredRevisionPath); err != nil {
return err
}
// drop old revision
@@ -329,6 +143,10 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
// revision 5, current 10 (restore a smaller blob) -> 5-10 = -5
// revision 10, current 5 (restore a bigger blob) -> 10-5 = +5
revisionSize, err := fs.lu.MetadataBackend().GetInt64(ctx, nodePath, prefixes.BlobsizeAttr)
if err != nil {
return errtypes.InternalError("failed to read blob size xattr from old revision")
}
sizeDiff := revisionSize - n.Blobsize
return fs.tp.Propagate(ctx, n, sizeDiff)
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/spaces.go
Generated Vendored
+2 -2
View File
@@ -767,7 +767,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De
return err
}
bid := m["user.ocis.blobid"]
bid := m["user.oc.blobid"]
if string(bid) == "" {
return nil
}
@@ -807,7 +807,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De
}
// the value of `target` depends on the implementation:
// - for ocis/s3ng it is the relative link to the space root
// - for decomposedfs/s3ng it is the relative link to the space root
// - for the posixfs it is the node id
func (fs *Decomposedfs) updateIndexes(ctx context.Context, grantee *provider.Grantee, spaceType, spaceID, nodeID string) error {
target := fs.tp.BuildSpaceIDIndexEntry(spaceID, nodeID)
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/revisions.go
Generated Vendored
+299
View File
@@ -0,0 +1,299 @@
// Copyright 2018-2021 CERN
// Copyright 2025 OpenCloud GmbH <mail@opencloud.eu>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package tree
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// Revision entries are stored inside the node folder and start with the same uuid as the current version.
// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions
// can be kept in the same location as the current file content. This prevents new fileuploads
// to trigger cross storage moves when revisions accidentally are stored on another partition,
// because the admin mounted a different partition there.
// We can add a background process to move old revisions to a slower storage
// and replace the revision file with a symbolic link in the future, if necessary.
// CreateVersion creates a new version of the node
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
versionPath := tp.lookup.VersionPath(n.SpaceID, n.ID, version)
err := os.MkdirAll(filepath.Dir(versionPath), 0700)
if err != nil {
return "", err
}
// create version node
vf, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return "", err
}
defer vf.Close()
// copy blob metadata to version node
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr
}, f, true); err != nil {
return "", err
}
return versionPath, nil
}
func (tp *Tree) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) {
_, span := tracer.Start(ctx, "ListRevisions")
defer span.End()
var n *node.Node
if n, err = tp.lookup.NodeFromResource(ctx, ref); err != nil {
return
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return
}
rp, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, err
case !rp.ListFileVersions:
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, errtypes.PermissionDenied(f)
}
return nil, errtypes.NotFound(f)
}
revisions = []*provider.FileVersion{}
np := n.InternalPath()
if items, err := filepath.Glob(np + node.RevisionIDDelimiter + "*"); err == nil {
for i := range items {
if tp.lookup.MetadataBackend().IsMetaFile(items[i]) || strings.HasSuffix(items[i], ".mlock") {
continue
}
if fi, err := os.Stat(items[i]); err == nil {
parts := strings.SplitN(fi.Name(), node.RevisionIDDelimiter, 2)
if len(parts) != 2 {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("invalid revision name, skipping")
continue
}
mtime := fi.ModTime()
rev := &provider.FileVersion{
Key: n.ID + node.RevisionIDDelimiter + parts[1],
Mtime: uint64(mtime.Unix()),
}
_, blobSize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, items[i], nil)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0")
}
rev.Size = uint64(blobSize)
etag, err := node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, errors.Wrapf(err, "error calculating etag")
}
rev.Etag = etag
revisions = append(revisions, rev)
}
}
}
// maybe we need to sort the list by key
/*
sort.Slice(revisions, func(i, j int) bool {
return revisions[i].Key > revisions[j].Key
})
*/
return
}
// DownloadRevision returns a reader for the specified revision
// FIXME the CS3 api should explicitly allow initiating revision and trash download, a related issue is https://github.com/cs3org/reva/issues/1813
func (tp *Tree) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) {
_, span := tracer.Start(ctx, "DownloadRevision")
defer span.End()
log := appctx.GetLogger(ctx)
// verify revision key format
kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2)
if len(kp) != 2 {
log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey")
return nil, nil, errtypes.NotFound(revisionKey)
}
log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision")
spaceID := ref.ResourceId.SpaceId
// check if the node is available and has not been deleted
n, err := node.ReadNode(ctx, tp.lookup, spaceID, kp[0], false, nil, false)
if err != nil {
return nil, nil, err
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return nil, nil, err
}
rp, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, nil, err
case !rp.ListFileVersions || !rp.InitiateFileDownload: // TODO add explicit permission in the CS3 api?
f, _ := storagespace.FormatReference(ref)
if rp.Stat {
return nil, nil, errtypes.PermissionDenied(f)
}
return nil, nil, errtypes.NotFound(f)
}
contentPath := tp.lookup.InternalPath(spaceID, revisionKey)
blobid, blobsize, err := tp.lookup.ReadBlobIDAndSizeAttr(ctx, contentPath, nil)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID)
}
revisionNode := node.Node{SpaceID: spaceID, BlobID: blobid, Blobsize: blobsize} // blobsize is needed for the s3ng blobstore
ri, err := n.AsResourceInfo(ctx, rp, nil, []string{"size", "mimetype", "etag"}, true)
if err != nil {
return nil, nil, err
}
// update resource info with revision data
mtime, err := time.Parse(time.RFC3339Nano, kp[1])
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not parse mtime for revision '%s' of node '%s'", kp[1], n.ID)
}
ri.Size = uint64(blobsize)
ri.Mtime = utils.TimeToTS(mtime)
ri.Etag, err = node.CalculateEtag(n.ID, mtime)
if err != nil {
return nil, nil, errors.Wrapf(err, "error calculating etag for revision '%s' of node '%s'", kp[1], n.ID)
}
var reader io.ReadCloser
if openReaderFunc(ri) {
reader, err = tp.ReadBlob(&revisionNode)
if err != nil {
return nil, nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", n.ID, revisionKey)
}
}
return ri, reader, nil
}
// DeleteRevision deletes the specified revision of the resource
func (tp *Tree) DeleteRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error {
_, span := tracer.Start(ctx, "DeleteRevision")
defer span.End()
n, err := tp.getRevisionNode(ctx, ref, revisionKey, func(rp *provider.ResourcePermissions) bool {
return rp.RestoreFileVersion
})
if err != nil {
return err
}
if err := os.RemoveAll(tp.lookup.InternalPath(n.SpaceID, revisionKey)); err != nil {
return err
}
return tp.DeleteBlob(n)
}
func (tp *Tree) getRevisionNode(ctx context.Context, ref *provider.Reference, revisionKey string, hasPermission func(*provider.ResourcePermissions) bool) (*node.Node, error) {
_, span := tracer.Start(ctx, "getRevisionNode")
defer span.End()
log := appctx.GetLogger(ctx)
// verify revision key format
kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2)
if len(kp) != 2 {
log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey")
return nil, errtypes.NotFound(revisionKey)
}
log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision")
spaceID := ref.ResourceId.SpaceId
// check if the node is available and has not been deleted
n, err := node.ReadNode(ctx, tp.lookup, spaceID, kp[0], false, nil, false)
if err != nil {
return nil, err
}
if !n.Exists {
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
return nil, err
}
p, err := tp.permissions.AssemblePermissions(ctx, n)
switch {
case err != nil:
return nil, err
case !hasPermission(p):
return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name))
}
// Set space owner in context
storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx))
return n, nil
}
func (tp *Tree) RestoreRevision(ctx context.Context, spaceID, nodeID, source string) error {
target := tp.lookup.InternalPath(spaceID, nodeID)
err := tp.lookup.CopyMetadata(ctx, source, target, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr
}, false)
if err != nil {
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
}
// always set the node mtime to the current time
err = tp.lookup.MetadataBackend().SetMultiple(ctx, target,
map[string][]byte{
prefixes.MTimeAttr: []byte(time.Now().UTC().Format(time.RFC3339Nano)),
},
false)
if err != nil {
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
}
return nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go
Generated Vendored
+12 -9
View File
@@ -39,6 +39,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
@@ -64,9 +65,10 @@ type Blobstore interface {
// Tree manages a hierarchical tree
type Tree struct {
lookup node.PathLookup
blobstore Blobstore
propagator propagator.Propagator
lookup node.PathLookup
blobstore Blobstore
propagator propagator.Propagator
permissions permissions.Permissions
options *options.Options
@@ -77,13 +79,14 @@ type Tree struct {
type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool
// New returns a new instance of Tree
func New(lu node.PathLookup, bs Blobstore, o *options.Options, cache store.Store, log *zerolog.Logger) *Tree {
func New(lu node.PathLookup, bs Blobstore, o *options.Options, p permissions.Permissions, cache store.Store, log *zerolog.Logger) *Tree {
return &Tree{
lookup: lu,
blobstore: bs,
options: o,
idCache: cache,
propagator: propagator.New(lu, o, log),
lookup: lu,
blobstore: bs,
options: o,
permissions: p,
idCache: cache,
propagator: propagator.New(lu, o, log),
}
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go
Generated Vendored
+6 -6
View File
@@ -54,7 +54,7 @@ func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, u
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload")
}
session := up.(*upload.OcisSession)
session := up.(*upload.DecomposedFsSession)
ctx = session.Context(ctx)
@@ -371,13 +371,13 @@ func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload,
// ListUploadSessions returns the upload sessions for the given filter
func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
var sessions []*upload.OcisSession
var sessions []*upload.DecomposedFsSession
if filter.ID != nil && *filter.ID != "" {
session, err := fs.sessionStore.Get(ctx, *filter.ID)
if err != nil {
return nil, err
}
sessions = []*upload.OcisSession{session}
sessions = []*upload.DecomposedFsSession{session}
} else {
var err error
sessions, err = fs.sessionStore.List(ctx)
@@ -418,19 +418,19 @@ func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.U
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
// the storage needs to implement AsTerminatableUpload
func (fs *Decomposedfs) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
// the storage needs to implement AsLengthDeclarableUpload
func (fs *Decomposedfs) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
// AsConcatableUpload returns a ConcatableUpload
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
// the storage needs to implement AsConcatableUpload
func (fs *Decomposedfs) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
return up.(*upload.OcisSession)
return up.(*upload.DecomposedFsSession)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/session.go
Generated Vendored
+41 -41
View File
@@ -38,15 +38,15 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// OcisSession extends tus upload lifecycle with postprocessing steps.
type OcisSession struct {
store OcisStore
// DecomposedFsSession extends tus upload lifecycle with postprocessing steps.
type DecomposedFsSession struct {
store DecomposedFsStore
// for now, we keep the json files in the uploads folder
info tusd.FileInfo
}
// Context returns a context with the user, logger and lockid used when initiating the upload session
func (s *OcisSession) Context(ctx context.Context) context.Context { // restore logger from file info
func (s *DecomposedFsSession) Context(ctx context.Context) context.Context { // restore logger from file info
sub := s.store.log.With().Int("pid", os.Getpid()).Logger()
ctx = appctx.WithLogger(ctx, &sub)
ctx = ctxpkg.ContextSetLockID(ctx, s.lockID())
@@ -54,10 +54,10 @@ func (s *OcisSession) Context(ctx context.Context) context.Context { // restore
return ctxpkg.ContextSetInitiator(ctx, s.InitiatorID())
}
func (s *OcisSession) lockID() string {
func (s *DecomposedFsSession) lockID() string {
return s.info.MetaData["lockid"]
}
func (s *OcisSession) executantUser() *userpb.User {
func (s *DecomposedFsSession) executantUser() *userpb.User {
var o *typespb.Opaque
_ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o)
return &userpb.User{
@@ -73,7 +73,7 @@ func (s *OcisSession) executantUser() *userpb.User {
}
// Purge deletes the upload session metadata and written binary data
func (s *OcisSession) Purge(ctx context.Context) error {
func (s *DecomposedFsSession) Purge(ctx context.Context) error {
_, span := tracer.Start(ctx, "Purge")
defer span.End()
sessionPath := sessionPath(s.store.root, s.info.ID)
@@ -87,7 +87,7 @@ func (s *OcisSession) Purge(ctx context.Context) error {
}
// TouchBin creates a file to contain the binary data. It's size will be used to keep track of the tus upload offset.
func (s *OcisSession) TouchBin() error {
func (s *DecomposedFsSession) TouchBin() error {
file, err := os.OpenFile(s.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
if err != nil {
return err
@@ -98,7 +98,7 @@ func (s *OcisSession) TouchBin() error {
// Persist writes the upload session metadata to disk
// events can update the scan outcome and the finished event might read an empty file because of race conditions
// so we need to lock the file while writing and use atomic writes
func (s *OcisSession) Persist(ctx context.Context) error {
func (s *DecomposedFsSession) Persist(ctx context.Context) error {
_, span := tracer.Start(ctx, "Persist")
defer span.End()
sessionPath := sessionPath(s.store.root, s.info.ID)
@@ -116,27 +116,27 @@ func (s *OcisSession) Persist(ctx context.Context) error {
}
// ToFileInfo returns tus compatible FileInfo so the tus handler can access the upload offset
func (s *OcisSession) ToFileInfo() tusd.FileInfo {
func (s *DecomposedFsSession) ToFileInfo() tusd.FileInfo {
return s.info
}
// ProviderID returns the provider id
func (s *OcisSession) ProviderID() string {
func (s *DecomposedFsSession) ProviderID() string {
return s.info.MetaData["providerID"]
}
// SpaceID returns the space id
func (s *OcisSession) SpaceID() string {
func (s *DecomposedFsSession) SpaceID() string {
return s.info.Storage["SpaceRoot"]
}
// NodeID returns the node id
func (s *OcisSession) NodeID() string {
func (s *DecomposedFsSession) NodeID() string {
return s.info.Storage["NodeId"]
}
// NodeParentID returns the nodes parent id
func (s *OcisSession) NodeParentID() string {
func (s *DecomposedFsSession) NodeParentID() string {
return s.info.Storage["NodeParentId"]
}
@@ -148,62 +148,62 @@ func (s *OcisSession) NodeParentID() string {
// A node should be created as part of InitiateUpload. When listing a directory
// we can decide if we want to skip the entry, or expose uploed progress
// information. But that is a bigger change and might involve client work.
func (s *OcisSession) NodeExists() bool {
func (s *DecomposedFsSession) NodeExists() bool {
return s.info.Storage["NodeExists"] == "true"
}
// HeaderIfMatch returns the if-match header for the upload session
func (s *OcisSession) HeaderIfMatch() string {
func (s *DecomposedFsSession) HeaderIfMatch() string {
return s.info.MetaData["if-match"]
}
// HeaderIfNoneMatch returns the if-none-match header for the upload session
func (s *OcisSession) HeaderIfNoneMatch() string {
func (s *DecomposedFsSession) HeaderIfNoneMatch() string {
return s.info.MetaData["if-none-match"]
}
// HeaderIfUnmodifiedSince returns the if-unmodified-since header for the upload session
func (s *OcisSession) HeaderIfUnmodifiedSince() string {
func (s *DecomposedFsSession) HeaderIfUnmodifiedSince() string {
return s.info.MetaData["if-unmodified-since"]
}
// Node returns the node for the session
func (s *OcisSession) Node(ctx context.Context) (*node.Node, error) {
func (s *DecomposedFsSession) Node(ctx context.Context) (*node.Node, error) {
return node.ReadNode(ctx, s.store.lu, s.SpaceID(), s.info.Storage["NodeId"], false, nil, true)
}
// ID returns the upload session id
func (s *OcisSession) ID() string {
func (s *DecomposedFsSession) ID() string {
return s.info.ID
}
// Filename returns the name of the node which is not the same as the name af the file being uploaded for legacy chunked uploads
func (s *OcisSession) Filename() string {
func (s *DecomposedFsSession) Filename() string {
return s.info.Storage["NodeName"]
}
// Chunk returns the chunk name when a legacy chunked upload was started
func (s *OcisSession) Chunk() string {
func (s *DecomposedFsSession) Chunk() string {
return s.info.Storage["Chunk"]
}
// SetMetadata is used to fill the upload metadata that will be exposed to the end user
func (s *OcisSession) SetMetadata(key, value string) {
func (s *DecomposedFsSession) SetMetadata(key, value string) {
s.info.MetaData[key] = value
}
// SetStorageValue is used to set metadata only relevant for the upload session implementation
func (s *OcisSession) SetStorageValue(key, value string) {
func (s *DecomposedFsSession) SetStorageValue(key, value string) {
s.info.Storage[key] = value
}
// SetSize will set the upload size of the underlying tus info.
func (s *OcisSession) SetSize(size int64) {
func (s *DecomposedFsSession) SetSize(size int64) {
s.info.Size = size
}
// SetSizeIsDeferred is uset to change the SizeIsDeferred property of the underlying tus info.
func (s *OcisSession) SetSizeIsDeferred(value bool) {
func (s *DecomposedFsSession) SetSizeIsDeferred(value bool) {
s.info.SizeIsDeferred = value
}
@@ -227,23 +227,23 @@ func (s *OcisSession) SetSizeIsDeferred(value bool) {
//
// I think we can safely determine the path later, right before emitting the
// event. And maybe make it configurable, because only audit needs it, anyway.
func (s *OcisSession) Dir() string {
func (s *DecomposedFsSession) Dir() string {
return s.info.Storage["Dir"]
}
// Size returns the upload size
func (s *OcisSession) Size() int64 {
func (s *DecomposedFsSession) Size() int64 {
return s.info.Size
}
// SizeDiff returns the size diff that was calculated after postprocessing
func (s *OcisSession) SizeDiff() int64 {
func (s *DecomposedFsSession) SizeDiff() int64 {
sizeDiff, _ := strconv.ParseInt(s.info.MetaData["sizeDiff"], 10, 64)
return sizeDiff
}
// Reference returns a reference that can be used to access the uploaded resource
func (s *OcisSession) Reference() provider.Reference {
func (s *DecomposedFsSession) Reference() provider.Reference {
return provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: s.info.MetaData["providerID"],
@@ -255,7 +255,7 @@ func (s *OcisSession) Reference() provider.Reference {
}
// Executant returns the id of the user that initiated the upload session
func (s *OcisSession) Executant() userpb.UserId {
func (s *DecomposedFsSession) Executant() userpb.UserId {
return userpb.UserId{
Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]),
Idp: s.info.Storage["Idp"],
@@ -264,7 +264,7 @@ func (s *OcisSession) Executant() userpb.UserId {
}
// SetExecutant is used to remember the user that initiated the upload session
func (s *OcisSession) SetExecutant(u *userpb.User) {
func (s *DecomposedFsSession) SetExecutant(u *userpb.User) {
s.info.Storage["Idp"] = u.GetId().GetIdp()
s.info.Storage["UserId"] = u.GetId().GetOpaqueId()
s.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type)
@@ -276,12 +276,12 @@ func (s *OcisSession) SetExecutant(u *userpb.User) {
}
// Offset returns the current upload offset
func (s *OcisSession) Offset() int64 {
func (s *DecomposedFsSession) Offset() int64 {
return s.info.Offset
}
// SpaceOwner returns the id of the space owner
func (s *OcisSession) SpaceOwner() *userpb.UserId {
func (s *DecomposedFsSession) SpaceOwner() *userpb.UserId {
return &userpb.UserId{
// idp and type do not seem to be consumed and the node currently only stores the user id anyway
OpaqueId: s.info.Storage["SpaceOwnerOrManager"],
@@ -289,7 +289,7 @@ func (s *OcisSession) SpaceOwner() *userpb.UserId {
}
// Expires returns the time the upload session expires
func (s *OcisSession) Expires() time.Time {
func (s *DecomposedFsSession) Expires() time.Time {
var t time.Time
if value, ok := s.info.MetaData["expires"]; ok {
t, _ = utils.MTimeToTime(value)
@@ -298,7 +298,7 @@ func (s *OcisSession) Expires() time.Time {
}
// MTime returns the mtime to use for the uploaded file
func (s *OcisSession) MTime() time.Time {
func (s *DecomposedFsSession) MTime() time.Time {
var t time.Time
if value, ok := s.info.MetaData["mtime"]; ok {
t, _ = utils.MTimeToTime(value)
@@ -307,29 +307,29 @@ func (s *OcisSession) MTime() time.Time {
}
// IsProcessing returns true if all bytes have been received. The session then has entered postprocessing state.
func (s *OcisSession) IsProcessing() bool {
func (s *DecomposedFsSession) IsProcessing() bool {
// We might need a more sophisticated way to determine processing status soon
return s.info.Size == s.info.Offset && s.info.MetaData["scanResult"] == ""
}
// binPath returns the path to the file storing the binary data.
func (s *OcisSession) binPath() string {
func (s *DecomposedFsSession) binPath() string {
return filepath.Join(s.store.root, "uploads", s.info.ID)
}
// InitiatorID returns the id of the initiating client
func (s *OcisSession) InitiatorID() string {
func (s *DecomposedFsSession) InitiatorID() string {
return s.info.MetaData["initiatorid"]
}
// SetScanData sets virus scan data to the upload session
func (s *OcisSession) SetScanData(result string, date time.Time) {
func (s *DecomposedFsSession) SetScanData(result string, date time.Time) {
s.info.MetaData["scanResult"] = result
s.info.MetaData["scanDate"] = date.Format(time.RFC3339)
}
// ScanData returns the virus scan data
func (s *OcisSession) ScanData() (string, time.Time) {
func (s *DecomposedFsSession) ScanData() (string, time.Time) {
date := s.info.MetaData["scanDate"]
if date == "" {
return "", time.Time{}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go
Generated Vendored
+19 -32
View File
@@ -56,8 +56,8 @@ type PermissionsChecker interface {
AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error)
}
// OcisStore manages upload sessions
type OcisStore struct {
// DecomposedFsStore manages upload sessions
type DecomposedFsStore struct {
fs storage.FS
lu node.PathLookup
tp node.Tree
@@ -70,9 +70,9 @@ type OcisStore struct {
log *zerolog.Logger
}
// NewSessionStore returns a new OcisStore
func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async bool, tknopts options.TokenOptions, log *zerolog.Logger) *OcisStore {
return &OcisStore{
// NewSessionStore returns a new DecomposedFsStore
func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async bool, tknopts options.TokenOptions, log *zerolog.Logger) *DecomposedFsStore {
return &DecomposedFsStore{
fs: fs,
lu: aspects.Lookup,
tp: aspects.Tree,
@@ -87,13 +87,13 @@ func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async
}
// New returns a new upload session
func (store OcisStore) New(ctx context.Context) *OcisSession {
return &OcisSession{
func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession {
return &DecomposedFsSession{
store: store,
info: tusd.FileInfo{
ID: uuid.New().String(),
Storage: map[string]string{
"Type": "OCISStore",
"Type": "DecomposedFsStore",
},
MetaData: tusd.MetaData{},
},
@@ -101,8 +101,8 @@ func (store OcisStore) New(ctx context.Context) *OcisSession {
}
// List lists all upload sessions
func (store OcisStore) List(ctx context.Context) ([]*OcisSession, error) {
uploads := []*OcisSession{}
func (store DecomposedFsStore) List(ctx context.Context) ([]*DecomposedFsSession, error) {
uploads := []*DecomposedFsSession{}
infoFiles, err := filepath.Glob(filepath.Join(store.root, "uploads", "*.info"))
if err != nil {
return nil, err
@@ -122,14 +122,14 @@ func (store OcisStore) List(ctx context.Context) ([]*OcisSession, error) {
}
// Get returns the upload session for the given upload id
func (store OcisStore) Get(ctx context.Context, id string) (*OcisSession, error) {
func (store DecomposedFsStore) Get(ctx context.Context, id string) (*DecomposedFsSession, error) {
sessionPath := sessionPath(store.root, id)
match := _idRegexp.FindStringSubmatch(sessionPath)
if match == nil || len(match) < 2 {
return nil, fmt.Errorf("invalid upload path")
}
session := OcisSession{
session := DecomposedFsSession{
store: store,
info: tusd.FileInfo{},
}
@@ -174,7 +174,7 @@ type Session interface {
}
// Cleanup cleans upload metadata, binary data and processing status as necessary
func (store OcisStore) Cleanup(ctx context.Context, session Session, revertNodeMetadata, keepUpload, unmarkPostprocessing bool) {
func (store DecomposedFsStore) Cleanup(ctx context.Context, session Session, revertNodeMetadata, keepUpload, unmarkPostprocessing bool) {
ctx, span := tracer.Start(session.Context(ctx), "Cleanup")
defer span.End()
session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload)
@@ -198,7 +198,7 @@ func (store OcisStore) Cleanup(ctx context.Context, session Session, revertNodeM
// CreateNodeForUpload will create the target node for the Upload
// TODO move this to the node package as NodeFromUpload?
// should we in InitiateUpload create the node first? and then the upload?
func (store OcisStore) CreateNodeForUpload(ctx context.Context, session *OcisSession, initAttrs node.Attributes) (*node.Node, error) {
func (store DecomposedFsStore) CreateNodeForUpload(ctx context.Context, session *DecomposedFsSession, initAttrs node.Attributes) (*node.Node, error) {
ctx, span := tracer.Start(session.Context(ctx), "CreateNodeForUpload")
defer span.End()
n := node.New(
@@ -301,7 +301,7 @@ func (store OcisStore) CreateNodeForUpload(ctx context.Context, session *OcisSes
return n, nil
}
func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) {
func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *DecomposedFsSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) {
_, span := tracer.Start(ctx, "updateExistingNode")
defer span.End()
targetPath := n.InternalPath()
@@ -364,10 +364,8 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess
}
if !store.disableVersioning {
versionPath := session.store.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano))
// create version node
_, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600)
span.AddEvent("CreateVersion")
versionPath, err := session.store.tp.CreateRevision(ctx, n, oldNodeMtime.UTC().Format(time.RFC3339Nano), f)
if err != nil {
if !errors.Is(err, os.ErrExist) {
return unlock, err
@@ -389,22 +387,11 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess
}
// clean revision file
span.AddEvent("os.Create")
if _, err := os.Create(versionPath); err != nil {
if versionPath, err = session.store.tp.CreateRevision(ctx, n, oldNodeMtime.UTC().Format(time.RFC3339Nano), f); err != nil {
return unlock, err
}
}
// copy blob metadata to version node
if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, versionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) {
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
attributeName == prefixes.TypeAttr ||
attributeName == prefixes.BlobIDAttr ||
attributeName == prefixes.BlobsizeAttr ||
attributeName == prefixes.MTimeAttr
}, f, true); err != nil {
return unlock, err
}
session.info.MetaData["versionsPath"] = versionPath
// keep mtime from previous version
span.AddEvent("os.Chtimes")
@@ -418,7 +405,7 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess
return unlock, nil
}
func validateChecksums(ctx context.Context, n *node.Node, session *OcisSession, versionPath string) error {
func validateChecksums(ctx context.Context, n *node.Node, session *DecomposedFsSession, versionPath string) error {
for _, t := range []string{"md5", "sha1", "adler32"} {
key := prefixes.ChecksumPrefix + t
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go
Generated Vendored
+14 -14
View File
@@ -60,7 +60,7 @@ func init() {
}
// WriteChunk writes the stream from the reader to the given offset of the upload
func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
func (session *DecomposedFsSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
ctx, span := tracer.Start(session.Context(ctx), "WriteChunk")
defer span.End()
_, subspan := tracer.Start(ctx, "os.OpenFile")
@@ -81,7 +81,7 @@ func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
// However, for the ocis driver it's not important whether the stream has ended
// However, for the decompsedfs driver it's not important whether the stream has ended
// on purpose or accidentally.
if err != nil && err != io.ErrUnexpectedEOF {
return n, err
@@ -95,12 +95,12 @@ func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io
}
// GetInfo returns the FileInfo
func (session *OcisSession) GetInfo(_ context.Context) (tusd.FileInfo, error) {
func (session *DecomposedFsSession) GetInfo(_ context.Context) (tusd.FileInfo, error) {
return session.ToFileInfo(), nil
}
// GetReader returns an io.Reader for the upload
func (session *OcisSession) GetReader(ctx context.Context) (io.ReadCloser, error) {
func (session *DecomposedFsSession) GetReader(ctx context.Context) (io.ReadCloser, error) {
_, span := tracer.Start(session.Context(ctx), "GetReader")
defer span.End()
return os.Open(session.binPath())
@@ -109,7 +109,7 @@ func (session *OcisSession) GetReader(ctx context.Context) (io.ReadCloser, error
// FinishUpload finishes an upload and moves the file to the internal destination
// implements tusd.DataStore interface
// returns tusd errors
func (session *OcisSession) FinishUpload(ctx context.Context) error {
func (session *DecomposedFsSession) FinishUpload(ctx context.Context) error {
err := session.FinishUploadDecomposed(ctx)
// we need to return a tusd error here to make the tusd handler return the correct status code
@@ -125,7 +125,7 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error {
// FinishUploadDecomposed finishes an upload and moves the file to the internal destination
// retures errtypes errors
func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error {
func (session *DecomposedFsSession) FinishUploadDecomposed(ctx context.Context) error {
ctx, span := tracer.Start(session.Context(ctx), "FinishUpload")
defer span.End()
log := appctx.GetLogger(ctx)
@@ -239,13 +239,13 @@ func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error {
}
// Terminate terminates the upload
func (session *OcisSession) Terminate(_ context.Context) error {
func (session *DecomposedFsSession) Terminate(_ context.Context) error {
session.Cleanup(true, true, true)
return nil
}
// DeclareLength updates the upload length information
func (session *OcisSession) DeclareLength(ctx context.Context, length int64) error {
func (session *DecomposedFsSession) DeclareLength(ctx context.Context, length int64) error {
session.info.Size = length
session.info.SizeIsDeferred = false
return session.store.um.RunInBaseScope(func() error {
@@ -254,7 +254,7 @@ func (session *OcisSession) DeclareLength(ctx context.Context, length int64) err
}
// ConcatUploads concatenates multiple uploads
func (session *OcisSession) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) {
func (session *DecomposedFsSession) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) {
file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
if err != nil {
return err
@@ -262,7 +262,7 @@ func (session *OcisSession) ConcatUploads(_ context.Context, uploads []tusd.Uplo
defer file.Close()
for _, partialUpload := range uploads {
fileUpload := partialUpload.(*OcisSession)
fileUpload := partialUpload.(*DecomposedFsSession)
src, err := os.Open(fileUpload.binPath())
if err != nil {
@@ -279,7 +279,7 @@ func (session *OcisSession) ConcatUploads(_ context.Context, uploads []tusd.Uplo
}
// Finalize finalizes the upload (eg moves the file to the internal destination)
func (session *OcisSession) Finalize(ctx context.Context) (err error) {
func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
ctx, span := tracer.Start(session.Context(ctx), "Finalize")
defer span.End()
@@ -305,7 +305,7 @@ func checkHash(expected string, h hash.Hash) error {
return nil
}
func (session *OcisSession) removeNode(ctx context.Context) {
func (session *DecomposedFsSession) removeNode(ctx context.Context) {
n, err := session.Node(ctx)
if err != nil {
appctx.GetLogger(ctx).Error().Str("session", session.ID()).Err(err).Msg("getting node from session failed")
@@ -317,7 +317,7 @@ func (session *OcisSession) removeNode(ctx context.Context) {
}
// cleanup cleans up after the upload is finished
func (session *OcisSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool) {
func (session *DecomposedFsSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool) {
ctx := session.Context(context.Background())
if revertNodeMetadata {
@@ -370,7 +370,7 @@ func (session *OcisSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool
}
// URL returns a url to download an upload
func (session *OcisSession) URL(_ context.Context) (string, error) {
func (session *DecomposedFsSession) URL(_ context.Context) (string, error) {
type transferClaims struct {
jwt.RegisteredClaims
Target string `json:"target"`
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/registry/spaces/Readme.md
Generated Vendored
+1 -1
View File
@@ -41,7 +41,7 @@ It also carries filters that are sent with a ListStorageSpaces call to a storage
* a display name, that is assigned by the owner or managers, eg. project names or 'Phils Home' for personal spaces. They are not unique
* an alias that is human readable and unique per user. It is used when listing paths on the CS3 global names as well as oc10 `/webdav` and `/dav/files/{username}` endpoints
5. on the ocis `/dav/spaces/{spaceid}/` endpoint the alias is actually not used because navigation happens by `{spaceid}`
5. on the OpenCloud `/dav/spaces/{spaceid}/` endpoint the alias is actually not used because navigation happens by `{spaceid}`
6. Every user has their own list of path to spaceid mappings, like one config file per user.
## consequences for storage providers
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking/chunking.go
Generated Vendored
+1 -1
View File
@@ -160,7 +160,7 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (bool, string, er
// there are still some chunks to be uploaded.
// we return CodeUploadIsPartial to notify upper layers that the upload is still
// not complete and requires more actions.
// This code is needed to notify the owncloud webservice that the upload has not yet been
// This code is needed to notify the OpenCloud webservice that the upload has not yet been
// completed and needs to continue uploading chunks.
if len(chunks) < chunkInfo.TotalChunks {
return false, "", nil
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors/errors.go
Generated Vendored
-51
View File
@@ -1,51 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package errors
import (
"fmt"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
)
// AlreadyExistsErr implements the Error interface.
type AlreadyExistsErr struct {
TypeName, Value string
IndexBy option.IndexBy
}
func (e *AlreadyExistsErr) Error() string {
return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.IndexBy.String(), e.Value)
}
// IsAlreadyExists implements the IsAlreadyExists interface.
func (e *AlreadyExistsErr) IsAlreadyExists() {}
// NotFoundErr implements the Error interface.
type NotFoundErr struct {
TypeName, Value string
IndexBy option.IndexBy
}
func (e *NotFoundErr) Error() string {
return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.IndexBy.String(), e.Value)
}
// IsNotFound implements the IsNotFound interface.
func (e *NotFoundErr) IsNotFound() {}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/helper.go
Generated Vendored
-32
View File
@@ -1,32 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package indexer
// dedup removes duplicate values in given slice
func dedup(s []string) []string {
var out []string
exists := make(map[string]bool)
for _, ss := range s {
if _, ok := exists[ss]; !ok {
out = append(out, ss)
exists[ss] = true
}
}
return out
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index/autoincrement.go
Generated Vendored
-260
View File
@@ -1,260 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package index
import (
"context"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
idxerrs "github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
metadata "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
)
// Autoincrement are fields for an index of type autoincrement.
type Autoincrement struct {
indexBy option.IndexBy
typeName string
filesDir string
indexBaseDir string
indexRootDir string
bound *option.Bound
storage metadata.Storage
}
// NewAutoincrementIndex instantiates a new AutoincrementIndex instance.
func NewAutoincrementIndex(storage metadata.Storage, o ...option.Option) Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Autoincrement{
storage: storage,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
bound: opts.Bound,
indexBaseDir: path.Join(opts.Prefix, "index."+storage.Backend()),
indexRootDir: path.Join(opts.Prefix, "index."+storage.Backend(), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy.String()}, ".")),
}
return u
}
// Init initializes an autoincrement index.
func (idx *Autoincrement) Init() error {
if err := idx.storage.MakeDirIfNotExist(context.Background(), idx.indexBaseDir); err != nil {
return err
}
return idx.storage.MakeDirIfNotExist(context.Background(), idx.indexRootDir)
}
// Lookup exact lookup by value.
func (idx *Autoincrement) Lookup(v string) ([]string, error) {
return idx.LookupCtx(context.Background(), v)
}
// LookupCtx retieves multiple exact values and allows passing in a context
func (idx *Autoincrement) LookupCtx(ctx context.Context, values ...string) ([]string, error) {
var allValues map[string]struct{}
if len(values) != 1 {
// prefetch all values with one request
entries, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
// convert known values to set
allValues = make(map[string]struct{}, len(entries))
for _, e := range entries {
allValues[path.Base(e)] = struct{}{}
}
}
// convert requested values to set
valueSet := make(map[string]struct{}, len(values))
for _, v := range values {
valueSet[v] = struct{}{}
}
var matches = []string{}
for v := range valueSet {
if _, ok := allValues[v]; ok || len(allValues) == 0 {
oldname, err := idx.storage.ResolveSymlink(context.Background(), path.Join(idx.indexRootDir, v))
if err != nil {
continue
}
matches = append(matches, oldname)
}
}
if len(matches) == 0 {
var v string
switch len(values) {
case 0:
v = "none"
case 1:
v = values[0]
default:
v = "multiple"
}
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return matches, nil
}
// Add a new value to the index.
func (idx *Autoincrement) Add(id, v string) (string, error) {
var newName string
if v == "" {
next, err := idx.next()
if err != nil {
return "", err
}
newName = path.Join(idx.indexRootDir, strconv.Itoa(next))
} else {
newName = path.Join(idx.indexRootDir, v)
}
if err := idx.storage.CreateSymlink(context.Background(), id, newName); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *Autoincrement) Remove(_ string, v string) error {
if v == "" {
return nil
}
searchPath := path.Join(idx.indexRootDir, v)
_, err := idx.storage.ResolveSymlink(context.Background(), searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return err
}
deletePath := path.Join(idx.indexRootDir, v)
return idx.storage.Delete(context.Background(), deletePath)
}
// Update index from <oldV> to <newV>.
func (idx *Autoincrement) Update(id, oldV, newV string) error {
if err := idx.Remove(id, oldV); err != nil {
return err
}
_, err := idx.Add(id, newV)
return err
}
// Search allows for glob search on the index.
func (idx *Autoincrement) Search(pattern string) ([]string, error) {
paths, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
searchPath := idx.indexRootDir
matches := make([]string, 0)
for _, p := range paths {
if found, err := filepath.Match(pattern, path.Base(p)); found {
if err != nil {
return nil, err
}
oldPath, err := idx.storage.ResolveSymlink(context.Background(), path.Join(searchPath, path.Base(p)))
if err != nil {
return nil, err
}
matches = append(matches, oldPath)
}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *Autoincrement) CaseInsensitive() bool {
return false
}
// IndexBy undocumented.
func (idx *Autoincrement) IndexBy() option.IndexBy {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Autoincrement) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Autoincrement) FilesDir() string {
return idx.filesDir
}
func (idx *Autoincrement) next() (int, error) {
paths, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return -1, err
}
if len(paths) == 0 {
return int(idx.bound.Lower), nil
}
sort.Slice(paths, func(i, j int) bool {
a, _ := strconv.Atoi(path.Base(paths[i]))
b, _ := strconv.Atoi(path.Base(paths[j]))
return a < b
})
latest, err := strconv.Atoi(path.Base(paths[len(paths)-1])) // would returning a string be a better interface?
if err != nil {
return -1, err
}
if int64(latest) < idx.bound.Lower {
return int(idx.bound.Lower), nil
}
return latest + 1, nil
}
// Delete deletes the index folder from its storage.
func (idx *Autoincrement) Delete() error {
return idx.storage.Delete(context.Background(), idx.indexRootDir)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index/index.go
Generated Vendored
-42
View File
@@ -1,42 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package index
import (
"context"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
)
// Index can be implemented to create new indexer-strategies. See Unique for example.
// Each indexer implementation is bound to one data-column (IndexBy) and a data-type (TypeName)
type Index interface {
Init() error
Lookup(v string) ([]string, error)
LookupCtx(ctx context.Context, v ...string) ([]string, error)
Add(id, v string) (string, error)
Remove(id string, v string) error
Update(id, oldV, newV string) error
Search(pattern string) ([]string, error)
CaseInsensitive() bool
IndexBy() option.IndexBy
TypeName() string
FilesDir() string
Delete() error // Delete deletes the index folder from its storage.
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index/non_unique.go
Generated Vendored
-280
View File
@@ -1,280 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package index
import (
"context"
"os"
"path"
"path/filepath"
"strings"
idxerrs "github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
metadata "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
)
// NonUnique are fields for an index of type non_unique.
type NonUnique struct {
caseInsensitive bool
indexBy option.IndexBy
typeName string
filesDir string
indexBaseDir string
indexRootDir string
storage metadata.Storage
}
// NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance.
// /tmp/ocis/accounts/index.cs3/Pets/Bro*
// ├── Brown/
// │ └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123
// ├── Green/
// │ ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789
// │ └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189
// └── White/
// | └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456
func NewNonUniqueIndexWithOptions(storage metadata.Storage, o ...option.Option) Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
return &NonUnique{
storage: storage,
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.Prefix, "index."+storage.Backend()),
indexRootDir: path.Join(opts.Prefix, "index."+storage.Backend(), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy.String()}, ".")),
}
}
// Init initializes a non_unique index.
func (idx *NonUnique) Init() error {
if err := idx.storage.MakeDirIfNotExist(context.Background(), idx.indexBaseDir); err != nil {
return err
}
return idx.storage.MakeDirIfNotExist(context.Background(), idx.indexRootDir)
}
// Lookup exact lookup by value.
func (idx *NonUnique) Lookup(v string) ([]string, error) {
return idx.LookupCtx(context.Background(), v)
}
// LookupCtx retieves multiple exact values and allows passing in a context
func (idx *NonUnique) LookupCtx(ctx context.Context, values ...string) ([]string, error) {
// prefetch all values with one request
entries, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
// convert known values to set
allValues := make(map[string]struct{}, len(entries))
for _, e := range entries {
allValues[path.Base(e)] = struct{}{}
}
// convert requested values to set
valueSet := make(map[string]struct{}, len(values))
if idx.caseInsensitive {
for _, v := range values {
valueSet[strings.ToLower(v)] = struct{}{}
}
} else {
for _, v := range values {
valueSet[v] = struct{}{}
}
}
var matches = map[string]struct{}{}
for v := range valueSet {
if _, ok := allValues[v]; ok {
children, err := idx.storage.ReadDir(context.Background(), filepath.Join(idx.indexRootDir, v))
if err != nil {
continue
}
for _, c := range children {
matches[path.Base(c)] = struct{}{}
}
}
}
if len(matches) == 0 {
var v string
switch len(values) {
case 0:
v = "none"
case 1:
v = values[0]
default:
v = "multiple"
}
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
ret := make([]string, 0, len(matches))
for m := range matches {
ret = append(ret, m)
}
return ret, nil
}
// Add a new value to the index.
func (idx *NonUnique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
newName := path.Join(idx.indexRootDir, v)
if err := idx.storage.MakeDirIfNotExist(context.Background(), newName); err != nil {
return "", err
}
if err := idx.storage.CreateSymlink(context.Background(), id, path.Join(newName, id)); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *NonUnique) Remove(id string, v string) error {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
deletePath := path.Join(idx.indexRootDir, v, id)
err := idx.storage.Delete(context.Background(), deletePath)
if err != nil {
return err
}
toStat := path.Join(idx.indexRootDir, v)
infos, err := idx.storage.ReadDir(context.Background(), toStat)
if err != nil {
return err
}
if len(infos) == 0 {
deletePath = path.Join(idx.indexRootDir, v)
err := idx.storage.Delete(context.Background(), deletePath)
if err != nil {
return err
}
}
return nil
}
// Update index from <oldV> to <newV>.
func (idx *NonUnique) Update(id, oldV, newV string) error {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
if err := idx.Remove(id, oldV); err != nil {
return err
}
if _, err := idx.Add(id, newV); err != nil {
return err
}
return nil
}
// Search allows for glob search on the index.
func (idx *NonUnique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
foldersMatched := make([]string, 0)
matches := make([]string, 0)
paths, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
for _, p := range paths {
if found, err := filepath.Match(pattern, path.Base(p)); found {
if err != nil {
return nil, err
}
foldersMatched = append(foldersMatched, p)
}
}
for i := range foldersMatched {
paths, _ := idx.storage.ReadDir(context.Background(), foldersMatched[i])
for _, p := range paths {
matches = append(matches, path.Base(p))
}
}
if len(matches) == 0 {
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: pattern}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *NonUnique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *NonUnique) IndexBy() option.IndexBy {
return idx.indexBy
}
// TypeName undocumented.
func (idx *NonUnique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *NonUnique) FilesDir() string {
return idx.filesDir
}
// Delete deletes the index folder from its storage.
func (idx *NonUnique) Delete() error {
return idx.storage.Delete(context.Background(), idx.indexRootDir)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index/unique.go
Generated Vendored
-253
View File
@@ -1,253 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package index
import (
"context"
"os"
"path"
"path/filepath"
"strings"
idxerrs "github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/errors"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
metadata "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
)
// Unique are fields for an index of type unique.
type Unique struct {
caseInsensitive bool
indexBy option.IndexBy
typeName string
filesDir string
indexBaseDir string
indexRootDir string
storage metadata.Storage
}
// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndexWithOptions(storage metadata.Storage, o ...option.Option) Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Unique{
storage: storage,
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.Prefix, "index."+storage.Backend()),
indexRootDir: path.Join(opts.Prefix, "index."+storage.Backend(), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy.String()}, ".")),
}
return u
}
// Init initializes a unique index.
func (idx *Unique) Init() error {
if err := idx.storage.MakeDirIfNotExist(context.Background(), idx.indexBaseDir); err != nil {
return err
}
return idx.storage.MakeDirIfNotExist(context.Background(), idx.indexRootDir)
}
// Lookup exact lookup by value.
func (idx *Unique) Lookup(v string) ([]string, error) {
return idx.LookupCtx(context.Background(), v)
}
// LookupCtx retieves multiple exact values and allows passing in a context
func (idx *Unique) LookupCtx(ctx context.Context, values ...string) ([]string, error) {
var allValues map[string]struct{}
if len(values) != 1 {
// prefetch all values with one request
entries, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
// convert known values to set
allValues = make(map[string]struct{}, len(entries))
for _, e := range entries {
allValues[path.Base(e)] = struct{}{}
}
}
// convert requested values to set
valueSet := make(map[string]struct{}, len(values))
if idx.caseInsensitive {
for _, v := range values {
valueSet[strings.ToLower(v)] = struct{}{}
}
} else {
for _, v := range values {
valueSet[v] = struct{}{}
}
}
var matches = make([]string, 0)
for v := range valueSet {
if _, ok := allValues[v]; ok || len(allValues) == 0 {
oldname, err := idx.storage.ResolveSymlink(context.Background(), path.Join(idx.indexRootDir, v))
if err != nil {
continue
}
matches = append(matches, oldname)
}
}
if len(matches) == 0 {
var v string
switch len(values) {
case 0:
v = "none"
case 1:
v = values[0]
default:
v = "multiple"
}
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return matches, nil
}
// Add adds a value to the index, returns the path to the root-document
func (idx *Unique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
target := path.Join(idx.filesDir, id)
newName := path.Join(idx.indexRootDir, v)
if err := idx.storage.CreateSymlink(context.Background(), target, newName); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *Unique) Remove(_ string, v string) error {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
_, err := idx.storage.ResolveSymlink(context.Background(), searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: v}
}
return err
}
deletePath := path.Join(idx.indexRootDir, v)
return idx.storage.Delete(context.Background(), deletePath)
}
// Update index from <oldV> to <newV>.
func (idx *Unique) Update(id, oldV, newV string) error {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
if err := idx.Remove(id, oldV); err != nil {
return err
}
if _, err := idx.Add(id, newV); err != nil {
return err
}
return nil
}
// Search allows for glob search on the index.
func (idx *Unique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
paths, err := idx.storage.ReadDir(context.Background(), idx.indexRootDir)
if err != nil {
return nil, err
}
searchPath := idx.indexRootDir
matches := make([]string, 0)
for _, p := range paths {
if found, err := filepath.Match(pattern, path.Base(p)); found {
if err != nil {
return nil, err
}
oldPath, err := idx.storage.ResolveSymlink(context.Background(), path.Join(searchPath, path.Base(p)))
if err != nil {
return nil, err
}
matches = append(matches, oldPath)
}
}
if len(matches) == 0 {
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, IndexBy: idx.indexBy, Value: pattern}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *Unique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *Unique) IndexBy() option.IndexBy {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Unique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Unique) FilesDir() string {
return idx.filesDir
}
// Delete deletes the index folder from its storage.
func (idx *Unique) Delete() error {
return idx.storage.Delete(context.Background(), idx.indexRootDir)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/indexer.go
Generated Vendored
-469
View File
@@ -1,469 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package indexer provides symlink-based indexer for on-disk document-directories.
package indexer
import (
"context"
"errors"
"fmt"
"path"
"strings"
"github.com/CiscoM31/godata"
"github.com/iancoleman/strcase"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/sync"
)
// Indexer is a facade to configure and query over multiple indices.
type Indexer interface {
AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error
Add(t interface{}) ([]IdxAddResult, error)
FindBy(t interface{}, fields ...Field) ([]string, error)
Delete(t interface{}) error
}
// Field combines the name and value of an indexed field.
type Field struct {
Name string
Value string
}
// NewField is a utility function to create a new Field.
func NewField(name, value string) Field {
return Field{Name: name, Value: value}
}
// StorageIndexer is the indexer implementation using metadata storage
type StorageIndexer struct {
storage metadata.Storage
indices typeMap
mu sync.NamedRWMutex
}
// IdxAddResult represents the result of an Add call on an index
type IdxAddResult struct {
Field, Value string
}
// CreateIndexer creates a new Indexer.
func CreateIndexer(storage metadata.Storage) Indexer {
return &StorageIndexer{
storage: storage,
indices: typeMap{},
mu: sync.NewNamedRWMutex(),
}
}
// Reset takes care of deleting all indices from storage and from the internal map of indices
func (i *StorageIndexer) Reset() error {
for j := range i.indices {
for _, indices := range i.indices[j].IndicesByField {
for _, idx := range indices {
err := idx.Delete()
if err != nil {
return err
}
}
}
delete(i.indices, j)
}
return nil
}
// AddIndex adds a new index to the indexer receiver.
func (i *StorageIndexer) AddIndex(t interface{}, indexBy option.IndexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error {
var idx index.Index
var f func(metadata.Storage, ...option.Option) index.Index
switch indexType {
case "unique":
f = index.NewUniqueIndexWithOptions
case "non_unique":
f = index.NewNonUniqueIndexWithOptions
case "autoincrement":
f = index.NewAutoincrementIndex
default:
return fmt.Errorf("invalid index type: %s", indexType)
}
idx = f(
i.storage,
option.CaseInsensitive(caseInsensitive),
option.WithBounds(bound),
option.WithIndexBy(indexBy),
option.WithTypeName(getTypeFQN(t)),
)
i.indices.addIndex(getTypeFQN(t), pkName, idx)
return idx.Init()
}
// Add a new entry to the indexer
func (i *StorageIndexer) Add(t interface{}) ([]IdxAddResult, error) {
typeName := getTypeFQN(t)
i.mu.Lock(typeName)
defer i.mu.Unlock(typeName)
var results []IdxAddResult
if fields, ok := i.indices[typeName]; ok {
for _, indices := range fields.IndicesByField {
for _, idx := range indices {
pkVal, err := valueOf(t, option.IndexByField(fields.PKFieldName))
if err != nil {
return []IdxAddResult{}, err
}
idxByVal, err := valueOf(t, idx.IndexBy())
if err != nil {
return []IdxAddResult{}, err
}
value, err := idx.Add(pkVal, idxByVal)
if err != nil {
return []IdxAddResult{}, err
}
if value == "" {
continue
}
results = append(results, IdxAddResult{Field: idx.IndexBy().String(), Value: value})
}
}
}
return results, nil
}
// FindBy finds a value on an index by fields.
// If multiple fields are given then they are handled like an or condition.
func (i *StorageIndexer) FindBy(t interface{}, queryFields ...Field) ([]string, error) {
typeName := getTypeFQN(t)
i.mu.RLock(typeName)
defer i.mu.RUnlock(typeName)
resultPaths := make(map[string]struct{})
if fields, ok := i.indices[typeName]; ok {
for fieldName, queryFields := range groupFieldsByName(queryFields) {
idxes := fields.IndicesByField[strcase.ToCamel(fieldName)]
values := make([]string, 0, len(queryFields))
for _, f := range queryFields {
values = append(values, f.Value)
}
for _, idx := range idxes {
res, err := idx.LookupCtx(context.Background(), values...)
if err != nil {
if _, ok := err.(errtypes.IsNotFound); ok {
continue
}
if err != nil {
return nil, err
}
}
for _, r := range res {
resultPaths[path.Base(r)] = struct{}{}
}
}
}
}
result := make([]string, 0, len(resultPaths))
for p := range resultPaths {
result = append(result, path.Base(p))
}
return result, nil
}
// groupFieldsByName groups the given filters and returns a map using the filter type as the key.
func groupFieldsByName(queryFields []Field) map[string][]Field {
grouped := make(map[string][]Field)
for _, f := range queryFields {
grouped[f.Name] = append(grouped[f.Name], f)
}
return grouped
}
// Delete deletes all indexed fields of a given type t on the Indexer.
func (i *StorageIndexer) Delete(t interface{}) error {
typeName := getTypeFQN(t)
i.mu.Lock(typeName)
defer i.mu.Unlock(typeName)
if fields, ok := i.indices[typeName]; ok {
for _, indices := range fields.IndicesByField {
for _, idx := range indices {
pkVal, err := valueOf(t, option.IndexByField(fields.PKFieldName))
if err != nil {
return err
}
idxByVal, err := valueOf(t, idx.IndexBy())
if err != nil {
return err
}
if err := idx.Remove(pkVal, idxByVal); err != nil {
return err
}
}
}
}
return nil
}
// FindByPartial allows for glob search across all indexes.
func (i *StorageIndexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) {
typeName := getTypeFQN(t)
i.mu.RLock(typeName)
defer i.mu.RUnlock(typeName)
resultPaths := make([]string, 0)
if fields, ok := i.indices[typeName]; ok {
for _, idx := range fields.IndicesByField[strcase.ToCamel(field)] {
res, err := idx.Search(pattern)
if err != nil {
if _, ok := err.(errtypes.IsNotFound); ok {
continue
}
if err != nil {
return nil, err
}
}
resultPaths = append(resultPaths, res...)
}
}
result := make([]string, 0, len(resultPaths))
for _, v := range resultPaths {
result = append(result, path.Base(v))
}
return result, nil
}
// Update updates all indexes on a value <from> to a value <to>.
func (i *StorageIndexer) Update(from, to interface{}) error {
typeNameFrom := getTypeFQN(from)
i.mu.Lock(typeNameFrom)
defer i.mu.Unlock(typeNameFrom)
if typeNameTo := getTypeFQN(to); typeNameFrom != typeNameTo {
return fmt.Errorf("update types do not match: from %v to %v", typeNameFrom, typeNameTo)
}
if fields, ok := i.indices[typeNameFrom]; ok {
for fName, indices := range fields.IndicesByField {
oldV, err := valueOf(from, option.IndexByField(fName))
if err != nil {
return err
}
newV, err := valueOf(to, option.IndexByField(fName))
if err != nil {
return err
}
pkVal, err := valueOf(from, option.IndexByField(fields.PKFieldName))
if err != nil {
return err
}
for _, idx := range indices {
if oldV == newV {
continue
}
if oldV == "" {
if _, err := idx.Add(pkVal, newV); err != nil {
return err
}
continue
}
if newV == "" {
if err := idx.Remove(pkVal, oldV); err != nil {
return err
}
continue
}
if err := idx.Update(pkVal, oldV, newV); err != nil {
return err
}
}
}
}
return nil
}
// Query parses an OData query into something our indexer.Index understands and resolves it.
func (i *StorageIndexer) Query(ctx context.Context, t interface{}, q string) ([]string, error) {
query, err := godata.ParseFilterString(ctx, q)
if err != nil {
return nil, err
}
tree := newQueryTree()
if err := buildTreeFromOdataQuery(query.Tree, &tree); err != nil {
return nil, err
}
results := make([]string, 0)
if err := i.resolveTree(t, &tree, &results); err != nil {
return nil, err
}
return results, nil
}
// t is used to infer the indexed field names. When building an index search query, field names have to respect Golang
// conventions and be in PascalCase. For a better overview on this contemplate reading the reflection package under the
// indexer directory. Traversal of the tree happens in a pre-order fashion.
// TODO implement logic for `and` operators.
func (i *StorageIndexer) resolveTree(t interface{}, tree *queryTree, partials *[]string) error {
if partials == nil {
return errors.New("return value cannot be nil: partials")
}
if tree.left != nil {
_ = i.resolveTree(t, tree.left, partials)
}
if tree.right != nil {
_ = i.resolveTree(t, tree.right, partials)
}
// by the time we're here we reached a leaf node.
if tree.token != nil {
switch tree.token.filterType {
case "FindBy":
operand, err := sanitizeInput(tree.token.operands)
if err != nil {
return err
}
field := Field{Name: operand.field, Value: operand.value}
r, err := i.FindBy(t, field)
if err != nil {
return err
}
*partials = append(*partials, r...)
case "FindByPartial":
operand, err := sanitizeInput(tree.token.operands)
if err != nil {
return err
}
r, err := i.FindByPartial(t, operand.field, fmt.Sprintf("%v*", operand.value))
if err != nil {
return err
}
*partials = append(*partials, r...)
default:
return fmt.Errorf("unsupported filter: %v", tree.token.filterType)
}
}
*partials = dedup(*partials)
return nil
}
type indexerTuple struct {
field, value string
}
// sanitizeInput returns a tuple of fieldName + value to be applied on indexer.Index filters.
func sanitizeInput(operands []string) (*indexerTuple, error) {
if len(operands) != 2 {
return nil, fmt.Errorf("invalid number of operands for filter function: got %v expected 2", len(operands))
}
// field names are Go public types and by design they are in PascalCase, therefore we need to adhere to this rules.
// for further information on this have a look at the reflection package.
f := strcase.ToCamel(operands[0])
// remove single quotes from value.
v := strings.ReplaceAll(operands[1], "'", "")
return &indexerTuple{
field: f,
value: v,
}, nil
}
// buildTreeFromOdataQuery builds an indexer.queryTree out of a GOData ParseNode. The purpose of this intermediate tree
// is to transform godata operators and functions into supported operations on our index. At the time of this writing
// we only support `FindBy` and `FindByPartial` queries as these are the only implemented filters on indexer.Index(es).
func buildTreeFromOdataQuery(root *godata.ParseNode, tree *queryTree) error {
if root.Token.Type == godata.ExpressionTokenFunc { // i.e "startswith", "contains"
switch root.Token.Value {
case "startswith":
token := token{
operator: root.Token.Value,
filterType: "FindByPartial",
// TODO sanitize the number of operands it the expected one.
operands: []string{
root.Children[0].Token.Value, // field name, i.e: Name
root.Children[1].Token.Value, // field value, i.e: Jac
},
}
tree.insert(&token)
default:
return errors.New("operation not supported")
}
}
if root.Token.Type == godata.ExpressionTokenLogical {
switch root.Token.Value {
case "or":
tree.insert(&token{operator: root.Token.Value})
for _, child := range root.Children {
if err := buildTreeFromOdataQuery(child, tree.left); err != nil {
return err
}
}
case "eq":
tree.insert(&token{
operator: root.Token.Value,
filterType: "FindBy",
operands: []string{
root.Children[0].Token.Value,
root.Children[1].Token.Value,
},
})
for _, child := range root.Children {
if err := buildTreeFromOdataQuery(child, tree.left); err != nil {
return err
}
}
default:
return errors.New("operator not supported")
}
}
return nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/map.go
Generated Vendored
-48
View File
@@ -1,48 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package indexer
import (
"github.com/iancoleman/strcase"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/index"
)
// typeMap stores the indexer layout at runtime.
type typeMap map[tName]typeMapping
type tName = string
type fieldName = string
type typeMapping struct {
PKFieldName string
IndicesByField map[fieldName][]index.Index
}
func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) {
if val, ok := m[typeName]; ok {
val.IndicesByField[strcase.ToCamel(idx.IndexBy().String())] = append(val.IndicesByField[strcase.ToCamel(idx.IndexBy().String())], idx)
return
}
m[typeName] = typeMapping{
PKFieldName: pkName,
IndicesByField: map[string][]index.Index{
strcase.ToCamel(idx.IndexBy().String()): {idx},
},
}
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option/option.go
Generated Vendored
-105
View File
@@ -1,105 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package option
import (
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
)
// Option defines a single option function.
type Option func(o *Options)
// IndexBy defines how the data is being indexed
type IndexBy interface {
String() string
}
// IndexByField represents the field that's being used to index the data by
type IndexByField string
// String returns a string representation
func (ibf IndexByField) String() string {
return string(ibf)
}
// IndexByFunc represents a function that's being used to index the data by
type IndexByFunc struct {
Name string
Func func(v interface{}) (string, error)
}
// String returns a string representation
func (ibf IndexByFunc) String() string {
return ibf.Name
}
// Bound represents a lower and upper bound range for an index.
// todo: if we would like to provide an upper bound then we would need to deal with ranges, in which case this is why the
// upper bound attribute is here.
type Bound struct {
Lower, Upper int64
}
// Options defines the available options for this package.
type Options struct {
CaseInsensitive bool
Bound *Bound
TypeName string
IndexBy IndexBy
FilesDir string
Prefix string
Storage metadata.Storage
}
// CaseInsensitive sets the CaseInsensitive field.
func CaseInsensitive(val bool) Option {
return func(o *Options) {
o.CaseInsensitive = val
}
}
// WithBounds sets the Bounds field.
func WithBounds(val *Bound) Option {
return func(o *Options) {
o.Bound = val
}
}
// WithTypeName sets the TypeName option.
func WithTypeName(val string) Option {
return func(o *Options) {
o.TypeName = val
}
}
// WithIndexBy sets the option IndexBy.
func WithIndexBy(val IndexBy) Option {
return func(o *Options) {
o.IndexBy = val
}
}
// WithFilesDir sets the option FilesDir.
func WithFilesDir(val string) Option {
return func(o *Options) {
o.FilesDir = val
}
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/query_tree.go
Generated Vendored
-58
View File
@@ -1,58 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package indexer
type queryTree struct {
token *token
root bool
left *queryTree
right *queryTree
}
// token to be resolved by the index
type token struct {
operator string // original OData operator. i.e: 'startswith', `or`, `and`.
filterType string // equivalent operator from OData -> indexer i.e FindByPartial or FindBy.
operands []string
}
// newQueryTree constructs a new tree with a root node.
func newQueryTree() queryTree {
return queryTree{
root: true,
}
}
// insert populates first the LHS of the tree first, if this is not possible it fills the RHS.
func (t *queryTree) insert(tkn *token) {
if t != nil && t.root {
t.left = &queryTree{token: tkn}
return
}
if t.left == nil {
t.left = &queryTree{token: tkn}
return
}
if t.left != nil && t.right == nil {
t.right = &queryTree{token: tkn}
return
}
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/reflect.go
Generated Vendored
-86
View File
@@ -1,86 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package indexer
import (
"errors"
"fmt"
"path"
"reflect"
"strconv"
"strings"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/indexer/option"
)
func getType(v interface{}) (reflect.Value, error) {
rv := reflect.ValueOf(v)
for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface {
rv = rv.Elem()
}
if !rv.IsValid() {
return reflect.Value{}, errors.New("failed to read value via reflection")
}
return rv, nil
}
func getTypeFQN(t interface{}) string {
typ, _ := getType(t)
typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name())
typeName = strings.ReplaceAll(typeName, "/", ".")
return typeName
}
func valueOf(v interface{}, indexBy option.IndexBy) (string, error) {
switch idxBy := indexBy.(type) {
case option.IndexByField:
return valueOfField(v, string(idxBy))
case option.IndexByFunc:
return idxBy.Func(v)
default:
return "", fmt.Errorf("unknown indexBy type")
}
}
func valueOfField(v interface{}, field string) (string, error) {
parts := strings.Split(field, ".")
for i, part := range parts {
r := reflect.ValueOf(v)
if r.Kind() == reflect.Ptr {
r = r.Elem()
}
f := reflect.Indirect(r).FieldByName(part)
if f.Kind() == reflect.Ptr {
f = f.Elem()
}
switch {
case f.Kind() == reflect.Struct && i != len(parts)-1:
v = f.Interface()
case f.Kind() == reflect.String:
return f.String(), nil
case f.IsZero():
return "", nil
default:
return strconv.Itoa(int(f.Int())), nil
}
}
return "", nil
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/localfs/upload.go
Generated Vendored
+1 -1
View File
@@ -312,7 +312,7 @@ func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.R
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
// However, for OwnCloudStore it's not important whether the stream has ended
// However, for the driver it's not important whether the stream has ended
// on purpose or accidentally.
if err != nil {
if err != io.ErrUnexpectedEOF {
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/sync/mutex.go
Generated Vendored
-67
View File
@@ -1,67 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package sync
import (
"sync"
)
// NamedRWMutex works the same as RWMutex, the only difference is that it stores mutexes in a map and reuses them.
// It's handy if you want to write-lock, write-unlock, read-lock and read-unlock for specific names only.
type NamedRWMutex struct {
pool sync.Pool
mus sync.Map
}
// NewNamedRWMutex returns a new instance of NamedRWMutex.
func NewNamedRWMutex() NamedRWMutex {
return NamedRWMutex{pool: sync.Pool{New: func() interface{} {
return new(sync.RWMutex)
}}}
}
// Lock locks rw for writing.
func (m *NamedRWMutex) Lock(name string) {
m.loadOrStore(name).Lock()
}
// Unlock unlocks rw for writing.
func (m *NamedRWMutex) Unlock(name string) {
m.loadOrStore(name).Unlock()
}
// RLock locks rw for reading.
func (m *NamedRWMutex) RLock(name string) {
m.loadOrStore(name).RLock()
}
// RUnlock undoes a single RLock call.
func (m *NamedRWMutex) RUnlock(name string) {
m.loadOrStore(name).RUnlock()
}
func (m *NamedRWMutex) loadOrStore(name string) *sync.RWMutex {
pmu := m.pool.Get()
mmu, loaded := m.mus.LoadOrStore(name, pmu)
if loaded {
m.pool.Put(pmu)
}
return mmu.(*sync.RWMutex)
}
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/sync/sync.go
Generated Vendored
-26
View File
@@ -1,26 +0,0 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package sync
import "sync"
var (
// ParsingViperConfig addresses the fact that config parsing using Viper is not thread safe.
ParsingViperConfig sync.Mutex
)
vendor/github.com/opencloud-eu/reva/v2/pkg/store/store.go
Generated Vendored
+1 -1
View File
@@ -135,7 +135,7 @@ func Create(opts ...microstore.Option) microstore.Store {
append(opts,
natsjs.NatsOptions(natsOptions), // always pass in properly initialized default nats options
natsjs.DefaultTTL(ttl))...,
) // TODO test with ocis nats
) // TODO test with OpenCloud nats
case TypeNatsJSKV:
// NOTE: nats needs a DefaultTTL option as it does not support per Write TTL ...
ttl, _ := options.Context.Value(ttlContextKey{}).(time.Duration)