Initial QSfera import
This commit is contained in:
+28
@@ -0,0 +1,28 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
)
|
||||
|
||||
// Warmup is the interface to implement cache warmup strategies.
|
||||
type Warmup interface {
|
||||
GetResourceInfos() ([]*provider.ResourceInfo, error)
|
||||
}
|
||||
Generated
Vendored
+22
@@ -0,0 +1,22 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package loader
|
||||
|
||||
// Load share cache drivers.
|
||||
// Add your own here
|
||||
Generated
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package registry
|
||||
|
||||
import "github.com/opencloud-eu/reva/v2/pkg/share/cache"
|
||||
|
||||
// NewFunc is the function that cache warmup implementations
|
||||
// should register at init time.
|
||||
type NewFunc func(map[string]interface{}) (cache.Warmup, error)
|
||||
|
||||
// NewFuncs is a map containing all the registered cache warmup implementations.
|
||||
var NewFuncs = map[string]NewFunc{}
|
||||
|
||||
// Register registers a new cache warmup function.
|
||||
// Not safe for concurrent use. Safe for use from package init.
|
||||
func Register(name string, f NewFunc) {
|
||||
NewFuncs[name] = f
|
||||
}
|
||||
Generated
Vendored
+1376
File diff suppressed because it is too large
Load Diff
Generated
Vendored
+435
@@ -0,0 +1,435 @@
|
||||
// Copyright 2026 OpenCloud GmbH
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff"
|
||||
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1"
|
||||
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/rs/zerolog"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// storageProvider is the narrow subset of provider.ProviderAPIClient that the
|
||||
// migration actually uses. Keeping it narrow makes test stubs trivial to write.
|
||||
type storageProvider interface {
|
||||
ListGrants(ctx context.Context, in *provider.ListGrantsRequest, opts ...grpc.CallOption) (*provider.ListGrantsResponse, error)
|
||||
}
|
||||
|
||||
type ImportSpaceMembersMigration struct {
|
||||
cfg config
|
||||
sharesChan chan *collaboration.Share
|
||||
receivedChan chan share.ReceivedShareWithUser
|
||||
userCache map[string]*userpb.UserId
|
||||
groupCache map[string]*grouppb.GroupId
|
||||
providerResolver func(context.Context, *provider.StorageSpace) (storageProvider, error)
|
||||
}
|
||||
|
||||
func init() {
|
||||
registerMigration(&ImportSpaceMembersMigration{})
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) Initialize(cfg config) {
|
||||
m.cfg = cfg
|
||||
m.sharesChan = make(chan *collaboration.Share)
|
||||
m.receivedChan = make(chan share.ReceivedShareWithUser)
|
||||
m.userCache = make(map[string]*userpb.UserId)
|
||||
m.groupCache = make(map[string]*grouppb.GroupId)
|
||||
m.providerResolver = func(ctx context.Context, space *provider.StorageSpace) (storageProvider, error) {
|
||||
return m.storageProviderForSpace(ctx, space)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) Name() string {
|
||||
return "import_space_members"
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) Version() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) Migrate() error {
|
||||
gwc, err := m.cfg.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
svcCtx, err := utils.GetServiceUserContextWithContext(context.Background(), gwc, m.cfg.serviceAccountID, m.cfg.serviceAccountSecret)
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).Msg("failed to get service user context for migration")
|
||||
return err
|
||||
}
|
||||
// List all project spaces.
|
||||
listRes, err := gwc.ListStorageSpaces(svcCtx, &provider.ListStorageSpacesRequest{
|
||||
Opaque: utils.AppendPlainToOpaque(nil, "unrestricted", "true"),
|
||||
Filters: []*provider.ListStorageSpacesRequest_Filter{
|
||||
{
|
||||
Type: provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE,
|
||||
Term: &provider.ListStorageSpacesRequest_Filter_SpaceType{SpaceType: "project"},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).Msg("space-membership migration: failed to list storage spaces")
|
||||
return err
|
||||
}
|
||||
|
||||
if listRes.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
m.cfg.logger.Error().Str("status", listRes.GetStatus().GetMessage()).Msg("space-membership migration: ListStorageSpaces returned non-OK status")
|
||||
return errtypes.InternalError("ListStorageSpaces")
|
||||
}
|
||||
|
||||
spaces := listRes.GetStorageSpaces()
|
||||
m.cfg.logger.Info().Int("spaces", len(spaces)).Msg("Starting migration")
|
||||
|
||||
// loadCtx is cancelled when the producer finishes (or fails) so that the
|
||||
// Load goroutine — which blocks reading from the channels — is not left
|
||||
// waiting forever if we return early from an error.
|
||||
loadCtx, cancelLoad := context.WithCancel(svcCtx)
|
||||
defer cancelLoad()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var loaderError error
|
||||
wg.Go(func() {
|
||||
loaderError = m.cfg.loader.Load(loadCtx, m.sharesChan, m.receivedChan)
|
||||
})
|
||||
|
||||
migrated := 0
|
||||
for _, space := range spaces {
|
||||
sharesCreated, err := m.migrateSpace(loadCtx, space)
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).Str("space", space.GetId().GetOpaqueId()).Msg("failed to migrate space; continuing with remaining spaces")
|
||||
continue
|
||||
}
|
||||
migrated++
|
||||
m.cfg.logger.Debug().
|
||||
Str("space", space.GetId().GetOpaqueId()).
|
||||
Int("shares_created", sharesCreated).
|
||||
Msg("space migrated")
|
||||
if migrated%10 == 0 {
|
||||
m.cfg.logger.Info().
|
||||
Int("migrated", migrated).
|
||||
Int("total", len(spaces)).
|
||||
Msg("migration progress")
|
||||
}
|
||||
}
|
||||
close(m.receivedChan)
|
||||
close(m.sharesChan)
|
||||
|
||||
wg.Wait()
|
||||
m.cfg.logger.Info().Err(loaderError).Int("migrated", migrated).Int("total", len(spaces)).Msg("Migration finished")
|
||||
return loaderError
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) migrateSpace(ctx context.Context, space *provider.StorageSpace) (int, error) {
|
||||
spClient, err := m.providerResolver(ctx, space)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ref := &provider.Reference{ResourceId: space.GetRoot()}
|
||||
grantsRes, err := spClient.ListGrants(ctx, &provider.ListGrantsRequest{Ref: ref})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if grantsRes.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
return 0, errtypes.NewErrtypeFromStatus(grantsRes.GetStatus())
|
||||
}
|
||||
|
||||
sharesCreated := 0
|
||||
for _, grant := range grantsRes.GetGrants() {
|
||||
share, receivedShares, err := m.spaceGrantToShares(ctx, grant, space)
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).
|
||||
Interface("grant", grant).
|
||||
Msg("Failed to convert grant to shares")
|
||||
continue
|
||||
}
|
||||
if share == nil {
|
||||
// share already existed; nothing to import for this grant
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case m.sharesChan <- share:
|
||||
case <-ctx.Done():
|
||||
return sharesCreated, ctx.Err()
|
||||
}
|
||||
for _, rs := range receivedShares {
|
||||
select {
|
||||
case m.receivedChan <- rs:
|
||||
case <-ctx.Done():
|
||||
return sharesCreated, ctx.Err()
|
||||
}
|
||||
}
|
||||
sharesCreated++
|
||||
}
|
||||
return sharesCreated, nil
|
||||
}
|
||||
|
||||
// resolveRetries is the maximum number of times resolveUserID / resolveGroupID
|
||||
// will retry after receiving an errtypes.Unavailable response (LDAP down).
|
||||
const resolveRetries = 10
|
||||
|
||||
// retryOnUnavailable calls op, retrying with exponential backoff whenever op
|
||||
// returns errtypes.Unavailable. Any other error (including context
|
||||
// cancellation) stops the loop immediately and is returned as-is.
|
||||
// Retries are capped at resolveRetries attempts and respect ctx cancellation.
|
||||
func retryOnUnavailable(ctx context.Context, log zerolog.Logger, op func() error) error {
|
||||
b := backoff.WithContext(
|
||||
backoff.WithMaxRetries(backoff.NewExponentialBackOff(), resolveRetries),
|
||||
ctx,
|
||||
)
|
||||
notify := func(err error, d time.Duration) {
|
||||
log.Warn().Err(err).Dur("retry_in", d).Msg("identity provider temporarily unavailable, retrying")
|
||||
}
|
||||
return backoff.RetryNotify(func() error {
|
||||
err := op()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if _, ok := err.(errtypes.Unavailable); ok {
|
||||
return err // transient — keep retrying
|
||||
}
|
||||
return backoff.Permanent(err) // permanent — stop immediately
|
||||
}, b, notify)
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) resolveUserID(ctx context.Context, opaqueID string) (*userpb.UserId, error) {
|
||||
if id, ok := m.userCache[opaqueID]; ok {
|
||||
return id, nil
|
||||
}
|
||||
var id *userpb.UserId
|
||||
err := retryOnUnavailable(ctx, m.cfg.logger, func() error {
|
||||
gwc, err := m.cfg.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := gwc.GetUser(ctx, &userpb.GetUserRequest{
|
||||
UserId: &userpb.UserId{OpaqueId: opaqueID},
|
||||
SkipFetchingUserGroups: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
// errtypes.NewErrtypeFromStatus maps CODE_UNAVAILABLE → errtypes.Unavailable,
|
||||
// which retryOnUnavailable will retry; all other codes are treated as permanent.
|
||||
return errtypes.NewErrtypeFromStatus(res.GetStatus())
|
||||
}
|
||||
id = res.GetUser().GetId()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.userCache[opaqueID] = id
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) resolveGroupID(ctx context.Context, opaqueID string) (*grouppb.GroupId, error) {
|
||||
if id, ok := m.groupCache[opaqueID]; ok {
|
||||
return id, nil
|
||||
}
|
||||
var id *grouppb.GroupId
|
||||
err := retryOnUnavailable(ctx, m.cfg.logger, func() error {
|
||||
gwc, err := m.cfg.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err := gwc.GetGroup(ctx, &grouppb.GetGroupRequest{
|
||||
GroupId: &grouppb.GroupId{OpaqueId: opaqueID},
|
||||
SkipFetchingMembers: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
return errtypes.NewErrtypeFromStatus(res.GetStatus())
|
||||
}
|
||||
id = res.GetGroup().GetId()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.groupCache[opaqueID] = id
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (m *ImportSpaceMembersMigration) spaceGrantToShares(ctx context.Context, grant *provider.Grant, space *provider.StorageSpace) (*collaboration.Share, []share.ReceivedShareWithUser, error) {
|
||||
// The grantee ids as persisted on disk do not have an IDP or type stored as
|
||||
// part of the userid/groupid. Resolve them via the gateway so we get the
|
||||
// full userid
|
||||
switch grant.GetGrantee().GetType() {
|
||||
case provider.GranteeType_GRANTEE_TYPE_GROUP:
|
||||
groupID, err := m.resolveGroupID(ctx, grant.GetGrantee().GetGroupId().GetOpaqueId())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve group %s: %w", grant.GetGrantee().GetGroupId().GetOpaqueId(), err)
|
||||
}
|
||||
grant.Grantee.Id = &provider.Grantee_GroupId{GroupId: groupID}
|
||||
case provider.GranteeType_GRANTEE_TYPE_USER:
|
||||
userID, err := m.resolveUserID(ctx, grant.GetGrantee().GetUserId().GetOpaqueId())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve user %s: %w", grant.GetGrantee().GetUserId().GetOpaqueId(), err)
|
||||
}
|
||||
grant.Grantee.Id = &provider.Grantee_UserId{UserId: userID}
|
||||
}
|
||||
|
||||
ref := &collaboration.ShareReference{
|
||||
Spec: &collaboration.ShareReference_Key{
|
||||
Key: &collaboration.ShareKey{
|
||||
ResourceId: space.GetRoot(),
|
||||
Grantee: grant.GetGrantee(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx = ctxpkg.ContextSetUser(ctx, &userpb.User{Id: grant.Creator})
|
||||
if s, err := m.cfg.manager.GetShare(ctx, ref); err == nil {
|
||||
// FIXME: Verify the actual grants?
|
||||
m.cfg.logger.Debug().Interface("share", s).Msg("share already exists")
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
ts := utils.TSNow()
|
||||
shareID := shareid.Encode(space.GetRoot().GetStorageId(), space.GetRoot().GetSpaceId(), uuid.NewString())
|
||||
|
||||
creator := grant.GetCreator()
|
||||
if creator.Type == userpb.UserType_USER_TYPE_INVALID {
|
||||
creator = nil
|
||||
}
|
||||
newShare := &collaboration.Share{
|
||||
Id: &collaboration.ShareId{OpaqueId: shareID},
|
||||
ResourceId: space.GetRoot(),
|
||||
Permissions: &collaboration.SharePermissions{Permissions: grant.GetPermissions()},
|
||||
Grantee: grant.GetGrantee(),
|
||||
Expiration: grant.GetExpiration(),
|
||||
Owner: creator,
|
||||
Creator: creator,
|
||||
Ctime: ts,
|
||||
Mtime: ts,
|
||||
}
|
||||
|
||||
var newReceivedShares []share.ReceivedShareWithUser
|
||||
switch grant.GetGrantee().GetType() {
|
||||
case provider.GranteeType_GRANTEE_TYPE_GROUP:
|
||||
gwc, err := m.cfg.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).Msg("Failed to get gateway client")
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
gr, err := gwc.GetMembers(ctx, &grouppb.GetMembersRequest{
|
||||
GroupId: grant.GetGrantee().GetGroupId(),
|
||||
})
|
||||
if err != nil {
|
||||
m.cfg.logger.Error().Err(err).Msg("Failed to expand group membership")
|
||||
return nil, nil, err
|
||||
}
|
||||
if gr.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
m.cfg.logger.Error().Str("Status", gr.GetStatus().GetMessage()).Msg("Failed to expand group membership")
|
||||
return nil, nil, errtypes.NewErrtypeFromStatus(gr.GetStatus())
|
||||
}
|
||||
for _, u := range gr.GetMembers() {
|
||||
newReceivedShares = append(newReceivedShares, share.ReceivedShareWithUser{
|
||||
UserID: u,
|
||||
ReceivedShare: &collaboration.ReceivedShare{
|
||||
Share: newShare,
|
||||
State: collaboration.ShareState_SHARE_STATE_ACCEPTED,
|
||||
},
|
||||
})
|
||||
}
|
||||
// Also add a group-level entry (UserID == nil) so the group cache is populated.
|
||||
newReceivedShares = append(newReceivedShares, share.ReceivedShareWithUser{
|
||||
UserID: nil,
|
||||
ReceivedShare: &collaboration.ReceivedShare{
|
||||
Share: newShare,
|
||||
State: collaboration.ShareState_SHARE_STATE_ACCEPTED,
|
||||
},
|
||||
})
|
||||
case provider.GranteeType_GRANTEE_TYPE_USER:
|
||||
newReceivedShares = append(newReceivedShares, share.ReceivedShareWithUser{
|
||||
UserID: grant.GetGrantee().GetUserId(),
|
||||
ReceivedShare: &collaboration.ReceivedShare{
|
||||
Share: newShare,
|
||||
State: collaboration.ShareState_SHARE_STATE_ACCEPTED,
|
||||
},
|
||||
})
|
||||
}
|
||||
return newShare, newReceivedShares, nil
|
||||
}
|
||||
|
||||
// storageProviderForSpace resolves the storageprovider responsible for the
|
||||
// given storage space and returns a dialled client. In the default opencloud
|
||||
// deployment the storage registry is co-located with the gateway, so
|
||||
// the GatewayAddr is used as the registry address.
|
||||
func (m *ImportSpaceMembersMigration) storageProviderForSpace(ctx context.Context, space *provider.StorageSpace) (provider.ProviderAPIClient, error) {
|
||||
|
||||
srClient, err := pool.GetStorageRegistryClient(m.cfg.providerRegistryAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get storage registry client: %w", err)
|
||||
}
|
||||
|
||||
spaceJSON, err := json.Marshal(space)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal space: %w", err)
|
||||
}
|
||||
|
||||
res, err := srClient.GetStorageProviders(ctx, ®istry.GetStorageProvidersRequest{
|
||||
Opaque: &typesv1beta1.Opaque{
|
||||
Map: map[string]*typesv1beta1.OpaqueEntry{
|
||||
"space": {
|
||||
Decoder: "json",
|
||||
Value: spaceJSON,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetStorageProviders: %w", err)
|
||||
}
|
||||
if len(res.GetProviders()) == 0 {
|
||||
return nil, fmt.Errorf("no storage provider found for space %s", space.GetId().GetOpaqueId())
|
||||
}
|
||||
|
||||
c, err := pool.GetStorageProviderServiceClient(res.GetProviders()[0].GetAddress())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dial storage provider: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
Generated
Vendored
+353
@@ -0,0 +1,353 @@
|
||||
// Copyright 2026 OpenCloud GmbH
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
const stateFile = "migrations/state.json"
|
||||
|
||||
const (
|
||||
lockFile = "migrations/lock.json"
|
||||
lockTTL = time.Minute
|
||||
lockHeartbeatInterval = 20 * time.Second
|
||||
)
|
||||
|
||||
// lockPollInterval is how long acquireLock sleeps between retries when the
|
||||
// lock is held by another instance. Declared as a variable so tests can
|
||||
// shorten it without rebuilding.
|
||||
var lockPollInterval = 5 * time.Second
|
||||
|
||||
// lockData is the content written to the lock file.
|
||||
type lockData struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
}
|
||||
|
||||
type migration interface {
|
||||
Name() string
|
||||
Version() int
|
||||
Initialize(config)
|
||||
Migrate() error
|
||||
}
|
||||
|
||||
// persistedState is the on-disk representation of the migration state.
|
||||
type persistedState struct {
|
||||
Version int `json:"version"`
|
||||
}
|
||||
|
||||
type state struct {
|
||||
version int
|
||||
}
|
||||
|
||||
// MigrationConfig holds all caller-supplied options for a migration run.
|
||||
// It is intentionally a plain struct so that new fields can be added without
|
||||
// changing function signatures throughout the call chain.
|
||||
type MigrationConfig struct {
|
||||
ServiceAccountID string
|
||||
ServiceAccountSecret string
|
||||
ProviderRegistryAddr string
|
||||
}
|
||||
|
||||
type config struct {
|
||||
logger zerolog.Logger
|
||||
gatewaySelector pool.Selectable[gatewayv1beta1.GatewayAPIClient]
|
||||
storage metadata.Storage
|
||||
serviceAccountID string
|
||||
serviceAccountSecret string
|
||||
providerRegistryAddr string
|
||||
manager share.Manager
|
||||
loader share.LoadableManager
|
||||
}
|
||||
|
||||
type Migrations struct {
|
||||
config
|
||||
state state
|
||||
instanceID string
|
||||
}
|
||||
|
||||
var migrations []migration
|
||||
|
||||
// registerMigration is only supposed to be call from init(), which runs sequentially
|
||||
// so we don't need ot protect migrations with a lock
|
||||
func registerMigration(m migration) {
|
||||
migrations = append(migrations, m)
|
||||
}
|
||||
|
||||
func New(logger zerolog.Logger,
|
||||
gatewaySelector pool.Selectable[gatewayv1beta1.GatewayAPIClient],
|
||||
storage metadata.Storage,
|
||||
cfg MigrationConfig,
|
||||
manager share.Manager,
|
||||
loader share.LoadableManager,
|
||||
) Migrations {
|
||||
|
||||
slices.SortFunc(migrations, func(a, b migration) int {
|
||||
return cmp.Compare(a.Version(), b.Version())
|
||||
})
|
||||
|
||||
b := make([]byte, 8)
|
||||
_, _ = rand.Read(b)
|
||||
instanceID := fmt.Sprintf("%x", b)
|
||||
|
||||
return Migrations{
|
||||
config{
|
||||
logger: logger.With().Str("jsoncs3", "migrations").Logger(),
|
||||
gatewaySelector: gatewaySelector,
|
||||
storage: storage,
|
||||
serviceAccountID: cfg.ServiceAccountID,
|
||||
serviceAccountSecret: cfg.ServiceAccountSecret,
|
||||
providerRegistryAddr: cfg.ProviderRegistryAddr,
|
||||
manager: manager,
|
||||
loader: loader,
|
||||
},
|
||||
state{},
|
||||
instanceID,
|
||||
}
|
||||
}
|
||||
|
||||
// acquireLock tries to atomically create the lock file, blocking until the lock
|
||||
// is obtained. It returns the etag of the lock file on success. It retries
|
||||
// indefinitely until ctx is cancelled. A lock whose timestamp is older than
|
||||
// lockTTL is considered stale and will be taken over.
|
||||
func (m *Migrations) acquireLock(ctx context.Context) (string, error) {
|
||||
m.logger.Debug().Str("instance", m.instanceID).Msg("acquiring migration lock")
|
||||
for {
|
||||
// Fast path: create the lock file only if it does not exist yet.
|
||||
data, err := json.Marshal(lockData{Timestamp: time.Now(), InstanceID: m.instanceID})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
res, err := m.storage.Upload(ctx, metadata.UploadRequest{
|
||||
Path: lockFile,
|
||||
Content: data,
|
||||
IfNoneMatch: []string{"*"},
|
||||
})
|
||||
if err == nil {
|
||||
m.logger.Debug().Str("instance", m.instanceID).Msg("migration lock acquired")
|
||||
return res.Etag, nil
|
||||
}
|
||||
|
||||
// Propagate context cancellation immediately.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Any error other than a conflict means something unexpected happened.
|
||||
if !isConflict(err) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Lock file already exists — read it to decide whether it is stale.
|
||||
dl, err := m.storage.Download(ctx, metadata.DownloadRequest{Path: lockFile})
|
||||
if err != nil {
|
||||
if _, ok := err.(errtypes.IsNotFound); ok {
|
||||
// Lock was released between our upload attempt and the download;
|
||||
// retry acquiring it immediately.
|
||||
m.logger.Debug().Str("instance", m.instanceID).Msg("migration lock vanished during read; retrying")
|
||||
continue
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
var existing lockData
|
||||
stale := true
|
||||
if err := json.Unmarshal(dl.Content, &existing); err == nil {
|
||||
stale = time.Since(existing.Timestamp) > lockTTL
|
||||
}
|
||||
|
||||
if stale {
|
||||
m.logger.Debug().
|
||||
Str("instance", m.instanceID).
|
||||
Str("held_by", existing.InstanceID).
|
||||
Time("lock_timestamp", existing.Timestamp).
|
||||
Msg("migration lock is stale; attempting takeover")
|
||||
|
||||
// Atomically take over the stale lock using the etag we just read.
|
||||
newData, err := json.Marshal(lockData{Timestamp: time.Now(), InstanceID: m.instanceID})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
res, err := m.storage.Upload(ctx, metadata.UploadRequest{
|
||||
Path: lockFile,
|
||||
Content: newData,
|
||||
IfMatchEtag: dl.Etag,
|
||||
})
|
||||
if err == nil {
|
||||
m.logger.Debug().Str("instance", m.instanceID).Msg("migration lock acquired via stale takeover")
|
||||
return res.Etag, nil
|
||||
}
|
||||
// Another instance took the stale lock before us; loop and retry.
|
||||
m.logger.Debug().Str("instance", m.instanceID).Err(err).Msg("stale lock takeover lost race; retrying")
|
||||
continue
|
||||
}
|
||||
|
||||
m.logger.Debug().
|
||||
Str("instance", m.instanceID).
|
||||
Str("held_by", existing.InstanceID).
|
||||
Time("lock_timestamp", existing.Timestamp).
|
||||
Dur("poll_interval", lockPollInterval).
|
||||
Msg("migration lock held by another instance; waiting")
|
||||
|
||||
// Lock is fresh and held by another instance; wait before retrying.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(lockPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startHeartbeat spawns a goroutine that periodically renews the lock file so
|
||||
// that it is not considered stale while a long migration is running. Call the
|
||||
// returned cancel function to stop the heartbeat.
|
||||
func (m *Migrations) startHeartbeat(ctx context.Context, etag string) context.CancelFunc {
|
||||
hbCtx, cancel := context.WithCancel(ctx)
|
||||
go func() {
|
||||
ticker := time.NewTicker(lockHeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
data, err := json.Marshal(lockData{Timestamp: time.Now(), InstanceID: m.instanceID})
|
||||
if err != nil {
|
||||
m.logger.Warn().Err(err).Msg("failed to marshal heartbeat data for migration lock")
|
||||
return
|
||||
}
|
||||
res, err := m.storage.Upload(hbCtx, metadata.UploadRequest{
|
||||
Path: lockFile,
|
||||
Content: data,
|
||||
IfMatchEtag: etag,
|
||||
})
|
||||
if err != nil {
|
||||
m.logger.Warn().Err(err).Msg("failed to renew migration lock; another instance may take over")
|
||||
return
|
||||
}
|
||||
etag = res.Etag
|
||||
}
|
||||
}
|
||||
}()
|
||||
return cancel
|
||||
}
|
||||
|
||||
// releaseLock deletes the lock file unconditionally.
|
||||
func (m *Migrations) releaseLock(ctx context.Context) {
|
||||
if err := m.storage.Delete(ctx, lockFile); err != nil {
|
||||
m.logger.Warn().Err(err).Msg("failed to release migration lock")
|
||||
}
|
||||
}
|
||||
|
||||
// isConflict returns true for errors that signal a conditional-upload conflict,
|
||||
// i.e. the lock file already exists or the etag did not match.
|
||||
func isConflict(err error) bool {
|
||||
switch err.(type) {
|
||||
case errtypes.IsAlreadyExists, errtypes.IsAborted, errtypes.IsPreconditionFailed:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// loadState reads the persisted migration version from storage. If no state
|
||||
// file exists yet (fresh deployment) it returns version 0 without error.
|
||||
func (m *Migrations) loadState(ctx context.Context) error {
|
||||
data, err := m.storage.SimpleDownload(ctx, stateFile)
|
||||
if err != nil {
|
||||
if _, ok := err.(errtypes.IsNotFound); ok {
|
||||
m.state = state{version: 0}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
var ps persistedState
|
||||
if err := json.Unmarshal(data, &ps); err != nil {
|
||||
return err
|
||||
}
|
||||
m.state = state{version: ps.Version}
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveState writes the current migration version to storage so that already-
|
||||
// applied migrations are not re-run on the next server start.
|
||||
func (m *Migrations) saveState(ctx context.Context) error {
|
||||
data, err := json.Marshal(persistedState{Version: m.state.version})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return m.storage.SimpleUpload(ctx, stateFile, data)
|
||||
}
|
||||
|
||||
func (m *Migrations) RunMigrations() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
etag, err := m.acquireLock(ctx)
|
||||
if err != nil {
|
||||
m.logger.Error().Err(err).Msg("failed to acquire migration lock; skipping migrations")
|
||||
return
|
||||
}
|
||||
cancelHB := m.startHeartbeat(ctx, etag)
|
||||
defer cancelHB()
|
||||
defer m.releaseLock(ctx)
|
||||
|
||||
if err := m.loadState(ctx); err != nil {
|
||||
m.logger.Error().Err(err).Msg("failed to load migration state; skipping migrations")
|
||||
return
|
||||
}
|
||||
|
||||
m.logger.Info().Int("current state", m.state.version).Msg("checking migrations")
|
||||
|
||||
for _, mig := range migrations {
|
||||
if mig.Version() > m.state.version {
|
||||
m.logger.Info().Str("migration", mig.Name()).Int("version", mig.Version()).Msg("running migration")
|
||||
mig.Initialize(m.config)
|
||||
if err := mig.Migrate(); err != nil {
|
||||
m.logger.Error().Err(err).Str("migration", mig.Name()).Msg("migration failed; stopping")
|
||||
return
|
||||
}
|
||||
m.state.version = mig.Version()
|
||||
if err := m.saveState(ctx); err != nil {
|
||||
m.logger.Error().Err(err).Msg("failed to save migration state; stopping")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
m.logger.Info().Str("migration", mig.Name()).Int("version", mig.Version()).Msg("skipping migration")
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+543
@@ -0,0 +1,543 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package providercache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/providercache")
|
||||
}
|
||||
|
||||
// Cache holds share information structured by provider and space
|
||||
type Cache struct {
|
||||
lockMap sync.Map
|
||||
|
||||
Providers mtimesyncedcache.Map[string, *Spaces]
|
||||
|
||||
storage metadata.Storage
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// Spaces holds the share information for provider
|
||||
type Spaces struct {
|
||||
Spaces mtimesyncedcache.Map[string, *Shares]
|
||||
}
|
||||
|
||||
// Shares holds the share information of one space
|
||||
type Shares struct {
|
||||
Shares map[string]*collaboration.Share
|
||||
|
||||
Etag string
|
||||
}
|
||||
|
||||
// UnmarshalJSON overrides the default unmarshaling
|
||||
// Shares are tricky to unmarshal because they contain an interface (Grantee) which makes the json Unmarshal bail out
|
||||
// To work around that problem we unmarshal into json.RawMessage in a first step and then try to manually unmarshal
|
||||
// into the specific types in a second step.
|
||||
func (s *Shares) UnmarshalJSON(data []byte) error {
|
||||
tmp := struct {
|
||||
Shares map[string]json.RawMessage
|
||||
}{}
|
||||
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.Shares = make(map[string]*collaboration.Share, len(tmp.Shares))
|
||||
for id, genericShare := range tmp.Shares {
|
||||
userShare := &collaboration.Share{
|
||||
Grantee: &provider.Grantee{Id: &provider.Grantee_UserId{}},
|
||||
}
|
||||
err = json.Unmarshal(genericShare, userShare) // is this a user share?
|
||||
if err == nil && userShare.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER {
|
||||
s.Shares[id] = userShare
|
||||
continue
|
||||
}
|
||||
|
||||
groupShare := &collaboration.Share{
|
||||
Grantee: &provider.Grantee{Id: &provider.Grantee_GroupId{}},
|
||||
}
|
||||
err = json.Unmarshal(genericShare, groupShare) // is this a group share?
|
||||
if err == nil && groupShare.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
|
||||
s.Shares[id] = groupShare
|
||||
continue
|
||||
}
|
||||
|
||||
invalidShare := &collaboration.Share{}
|
||||
err = json.Unmarshal(genericShare, invalidShare) // invalid
|
||||
if err == nil {
|
||||
s.Shares[id] = invalidShare
|
||||
continue
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LockSpace locks the cache for a given space and returns an unlock function
|
||||
func (c *Cache) LockSpace(spaceID string) func() {
|
||||
v, _ := c.lockMap.LoadOrStore(spaceID, &sync.Mutex{})
|
||||
lock := v.(*sync.Mutex)
|
||||
|
||||
lock.Lock()
|
||||
return func() { lock.Unlock() }
|
||||
}
|
||||
|
||||
// New returns a new Cache instance
|
||||
func New(s metadata.Storage, ttl time.Duration) Cache {
|
||||
return Cache{
|
||||
Providers: mtimesyncedcache.Map[string, *Spaces]{},
|
||||
storage: s,
|
||||
ttl: ttl,
|
||||
lockMap: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) isSpaceCached(storageID, spaceID string) bool {
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, ok = spaces.Spaces.Load(spaceID)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Add adds a share to the cache
|
||||
func (c *Cache) Add(ctx context.Context, storageID, spaceID, shareID string, share *collaboration.Share) error {
|
||||
ctx, span := tracer.Start(ctx, "Add")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID), attribute.String("cs3.shareid", shareID))
|
||||
|
||||
switch {
|
||||
case storageID == "":
|
||||
return fmt.Errorf("missing storage id")
|
||||
case spaceID == "":
|
||||
return fmt.Errorf("missing space id")
|
||||
case shareID == "":
|
||||
return fmt.Errorf("missing share id")
|
||||
}
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
defer unlock()
|
||||
span.AddEvent("got lock")
|
||||
|
||||
var err error
|
||||
if !c.isSpaceCached(storageID, spaceID) {
|
||||
err = c.syncWithLock(ctx, storageID, spaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("storageID", storageID).
|
||||
Str("spaceID", spaceID).
|
||||
Str("shareID", shareID).Logger()
|
||||
|
||||
persistFunc := func() error {
|
||||
|
||||
spaces, _ := c.Providers.Load(storageID)
|
||||
space, _ := spaces.Spaces.Load(spaceID)
|
||||
|
||||
log.Info().Interface("shares", maps.Keys(space.Shares)).Str("New share", shareID).Msg("Adding share to space")
|
||||
space.Shares[shareID] = share
|
||||
|
||||
return c.Persist(ctx, storageID, spaceID)
|
||||
}
|
||||
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting added provider share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting added provider share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
case errtypes.AlreadyExists:
|
||||
log.Debug().Msg("already exists when persisting added provider share. retrying...")
|
||||
// CS3 uses an already exists error instead of precondition failed when using an If-None-Match=* header / IfExists flag in the InitiateFileUpload call.
|
||||
// Thas happens when the cache thinks there is no file.
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting added provider share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting added provider share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, storageID, spaceID); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Error().Err(err).Msg("persisting added provider share failed. giving up.")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove removes a share from the cache
|
||||
func (c *Cache) Remove(ctx context.Context, storageID, spaceID, shareID string) error {
|
||||
ctx, span := tracer.Start(ctx, "Remove")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID), attribute.String("cs3.shareid", shareID))
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
defer unlock()
|
||||
span.AddEvent("got lock")
|
||||
|
||||
if !c.isSpaceCached(storageID, spaceID) {
|
||||
err := c.syncWithLock(ctx, storageID, spaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
persistFunc := func() error {
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
space, _ := spaces.Spaces.Load(spaceID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
delete(space.Shares, shareID)
|
||||
|
||||
return c.Persist(ctx, storageID, spaceID)
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("storageID", storageID).
|
||||
Str("spaceID", spaceID).
|
||||
Str("shareID", shareID).Logger()
|
||||
|
||||
var err error
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting removed provider share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting removed provider share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting removed provider share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting removed provider share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, storageID, spaceID); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Error().Err(err).Msg("persisting removed provider share failed. giving up.")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns one entry from the cache
|
||||
func (c *Cache) Get(ctx context.Context, storageID, spaceID, shareID string, skipSync bool) (*collaboration.Share, error) {
|
||||
ctx, span := tracer.Start(ctx, "Get")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID), attribute.String("cs3.shareid", shareID))
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
defer unlock()
|
||||
span.AddEvent("got lock")
|
||||
|
||||
if !skipSync {
|
||||
// sync cache, maybe our data is outdated
|
||||
err := c.syncWithLock(ctx, storageID, spaceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
space, ok := spaces.Spaces.Load(spaceID)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return space.Shares[shareID], nil
|
||||
}
|
||||
|
||||
// All returns all entries in the storage
|
||||
func (c *Cache) All(ctx context.Context) (*mtimesyncedcache.Map[string, *Spaces], error) {
|
||||
ctx, span := tracer.Start(ctx, "All")
|
||||
defer span.End()
|
||||
|
||||
providers, err := c.storage.ListDir(ctx, "/storages")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, provider := range providers {
|
||||
storageID := provider.Name
|
||||
spaces, err := c.storage.ListDir(ctx, path.Join("/storages", storageID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, space := range spaces {
|
||||
spaceID := strings.TrimSuffix(space.Name, ".json")
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
span.AddEvent("got lock for space " + spaceID)
|
||||
if err := c.syncWithLock(ctx, storageID, spaceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unlock()
|
||||
}
|
||||
}
|
||||
|
||||
return &c.Providers, nil
|
||||
}
|
||||
|
||||
// ListSpace returns the list of shares in a given space
|
||||
func (c *Cache) ListSpace(ctx context.Context, storageID, spaceID string) (*Shares, error) {
|
||||
ctx, span := tracer.Start(ctx, "ListSpace")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID))
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
defer unlock()
|
||||
span.AddEvent("got lock")
|
||||
|
||||
// sync cache, maybe our data is outdated
|
||||
err := c.syncWithLock(ctx, storageID, spaceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
return &Shares{}, nil
|
||||
}
|
||||
|
||||
space, ok := spaces.Spaces.Load(spaceID)
|
||||
if !ok {
|
||||
return &Shares{}, nil
|
||||
}
|
||||
|
||||
shares := &Shares{
|
||||
Shares: maps.Clone(space.Shares),
|
||||
Etag: space.Etag,
|
||||
}
|
||||
return shares, nil
|
||||
}
|
||||
|
||||
// Persist persists the data of one space
|
||||
func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error {
|
||||
ctx, span := tracer.Start(ctx, "Persist")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID))
|
||||
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
span.AddEvent("nothing to persist")
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
space, ok := spaces.Spaces.Load(spaceID)
|
||||
if !ok {
|
||||
span.AddEvent("nothing to persist")
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
span.SetAttributes(attribute.String("BeforeEtag", space.Etag))
|
||||
log := appctx.GetLogger(ctx).With().Str("storageID", storageID).Str("spaceID", spaceID).Logger()
|
||||
log = log.With().Str("BeforeEtag", space.Etag).Logger()
|
||||
|
||||
createdBytes, err := json.Marshal(space)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
jsonPath := spaceJSONPath(storageID, spaceID)
|
||||
if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
span.SetAttributes(attribute.String("etag", space.Etag))
|
||||
|
||||
ur := metadata.UploadRequest{
|
||||
Path: jsonPath,
|
||||
Content: createdBytes,
|
||||
IfMatchEtag: space.Etag,
|
||||
}
|
||||
// when there is no etag in memory make sure the file has not been created on the server, see https://www.rfc-editor.org/rfc/rfc9110#field.if-match
|
||||
// > If the field value is "*", the condition is false if the origin server has a current representation for the target resource.
|
||||
if space.Etag == "" {
|
||||
ur.IfNoneMatch = []string{"*"}
|
||||
}
|
||||
|
||||
res, err := c.storage.Upload(ctx, ur)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Debug().Err(err).Msg("persisting provider cache failed")
|
||||
return err
|
||||
}
|
||||
space.Etag = res.Etag
|
||||
|
||||
span.SetStatus(codes.Ok, "")
|
||||
shares := []string{}
|
||||
for _, s := range space.Shares {
|
||||
shares = append(shares, s.GetId().GetOpaqueId())
|
||||
}
|
||||
log.Debug().Str("AfterEtag", space.Etag).Interface("Shares", shares).Msg("persisted provider cache")
|
||||
return nil
|
||||
}
|
||||
|
||||
// PurgeSpace removes a space from the cache
|
||||
func (c *Cache) PurgeSpace(ctx context.Context, storageID, spaceID string) error {
|
||||
ctx, span := tracer.Start(ctx, "PurgeSpace")
|
||||
defer span.End()
|
||||
|
||||
unlock := c.LockSpace(spaceID)
|
||||
defer unlock()
|
||||
span.AddEvent("got lock")
|
||||
|
||||
if !c.isSpaceCached(storageID, spaceID) {
|
||||
err := c.syncWithLock(ctx, storageID, spaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
spaces, ok := c.Providers.Load(storageID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
newShares := &Shares{}
|
||||
if space, ok := spaces.Spaces.Load(spaceID); ok {
|
||||
newShares.Etag = space.Etag // keep the etag to allow overwriting the state on the server
|
||||
}
|
||||
spaces.Spaces.Store(spaceID, newShares)
|
||||
|
||||
return c.Persist(ctx, storageID, spaceID)
|
||||
}
|
||||
|
||||
func (c *Cache) syncWithLock(ctx context.Context, storageID, spaceID string) error {
|
||||
ctx, span := tracer.Start(ctx, "syncWithLock")
|
||||
defer span.End()
|
||||
|
||||
c.initializeIfNeeded(storageID, spaceID)
|
||||
|
||||
spaces, _ := c.Providers.Load(storageID)
|
||||
space, _ := spaces.Spaces.Load(spaceID)
|
||||
span.SetAttributes(attribute.String("cs3.storageid", storageID), attribute.String("cs3.spaceid", spaceID), attribute.String("etag", space.Etag))
|
||||
log := appctx.GetLogger(ctx).With().Str("storageID", storageID).Str("spaceID", spaceID).Str("etag", space.Etag).Str("hostname", os.Getenv("HOSTNAME")).Logger()
|
||||
|
||||
dlreq := metadata.DownloadRequest{
|
||||
Path: spaceJSONPath(storageID, spaceID),
|
||||
}
|
||||
// when we know an etag, only download if it changed remotely
|
||||
if space.Etag != "" {
|
||||
dlreq.IfNoneMatch = []string{space.Etag}
|
||||
}
|
||||
|
||||
dlres, err := c.storage.Download(ctx, dlreq)
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.AddEvent("updating local cache")
|
||||
case errtypes.NotFound:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.NotModified:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
default:
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "downloading provider cache failed")
|
||||
return err
|
||||
}
|
||||
|
||||
span.AddEvent("updating local cache")
|
||||
newShares := &Shares{}
|
||||
err = json.Unmarshal(dlres.Content, newShares)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "unmarshaling provider cache failed")
|
||||
log.Error().Err(err).Msg("unmarshaling provider cache failed")
|
||||
return err
|
||||
}
|
||||
newShares.Etag = dlres.Etag
|
||||
|
||||
spaces.Spaces.Store(spaceID, newShares)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cache) initializeIfNeeded(storageID, spaceID string) {
|
||||
spaces, _ := c.Providers.LoadOrStore(storageID, &Spaces{
|
||||
Spaces: mtimesyncedcache.Map[string, *Shares]{},
|
||||
})
|
||||
_, _ = spaces.Spaces.LoadOrStore(spaceID, &Shares{
|
||||
Shares: map[string]*collaboration.Share{},
|
||||
})
|
||||
}
|
||||
|
||||
func spaceJSONPath(storageID, spaceID string) string {
|
||||
return filepath.Join("/storages", storageID, spaceID+".json")
|
||||
}
|
||||
Generated
Vendored
+392
@@ -0,0 +1,392 @@
|
||||
// Copyright 2018-2022 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package receivedsharecache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
)
|
||||
|
||||
// name is the Tracer name used to identify this instrumentation library.
|
||||
const tracerName = "receivedsharecache"
|
||||
|
||||
// Cache stores the list of received shares and their states
|
||||
// It functions as an in-memory cache with a persistence layer
|
||||
// The storage is sharded by user
|
||||
type Cache struct {
|
||||
lockMap sync.Map
|
||||
|
||||
ReceivedSpaces mtimesyncedcache.Map[string, *Spaces]
|
||||
|
||||
storage metadata.Storage
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// Spaces holds the received shares of one user per space
|
||||
type Spaces struct {
|
||||
Spaces map[string]*Space
|
||||
|
||||
etag string
|
||||
}
|
||||
|
||||
// Space holds the received shares of one user in one space
|
||||
type Space struct {
|
||||
States map[string]*State
|
||||
}
|
||||
|
||||
// State holds the state information of a received share
|
||||
type State struct {
|
||||
State collaboration.ShareState
|
||||
MountPoint *provider.Reference
|
||||
Hidden bool
|
||||
}
|
||||
|
||||
// New returns a new Cache instance
|
||||
func New(s metadata.Storage, ttl time.Duration) Cache {
|
||||
return Cache{
|
||||
ReceivedSpaces: mtimesyncedcache.Map[string, *Spaces]{},
|
||||
storage: s,
|
||||
ttl: ttl,
|
||||
lockMap: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) lockUser(userID string) func() {
|
||||
v, _ := c.lockMap.LoadOrStore(userID, &sync.Mutex{})
|
||||
lock := v.(*sync.Mutex)
|
||||
|
||||
lock.Lock()
|
||||
return func() { lock.Unlock() }
|
||||
}
|
||||
|
||||
// Add adds a new entry to the cache
|
||||
func (c *Cache) Add(ctx context.Context, userID, spaceID string, rs *collaboration.ReceivedShare) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userID)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
defer unlock()
|
||||
|
||||
if _, ok := c.ReceivedSpaces.Load(userID); !ok {
|
||||
err := c.syncWithLock(ctx, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, span = appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Add")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID), attribute.String("cs3.spaceid", spaceID))
|
||||
|
||||
persistFunc := func() error {
|
||||
c.initializeIfNeeded(userID, spaceID)
|
||||
|
||||
rss, _ := c.ReceivedSpaces.Load(userID)
|
||||
receivedSpace := rss.Spaces[spaceID]
|
||||
if receivedSpace.States == nil {
|
||||
receivedSpace.States = map[string]*State{}
|
||||
}
|
||||
receivedSpace.States[rs.Share.Id.GetOpaqueId()] = &State{
|
||||
State: rs.State,
|
||||
MountPoint: rs.MountPoint,
|
||||
Hidden: rs.Hidden,
|
||||
}
|
||||
|
||||
return c.persist(ctx, userID)
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("userID", userID).
|
||||
Str("spaceID", spaceID).Logger()
|
||||
|
||||
var err error
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting added received share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting added received share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
case errtypes.AlreadyExists:
|
||||
log.Debug().Msg("already exists when persisting added received share. retrying...")
|
||||
// CS3 uses an already exists error instead of precondition failed when using an If-None-Match=* header / IfExists flag in the InitiateFileUpload call.
|
||||
// Thas happens when the cache thinks there is no file.
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting added received share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting added received share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, userID); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Error().Err(err).Msg("persisting added received share failed. giving up.")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns one entry from the cache
|
||||
func (c *Cache) Get(ctx context.Context, userID, spaceID, shareID string) (*State, error) {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userID)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
defer unlock()
|
||||
|
||||
err := c.syncWithLock(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rss, ok := c.ReceivedSpaces.Load(userID)
|
||||
if !ok || rss.Spaces[spaceID] == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return rss.Spaces[spaceID].States[shareID], nil
|
||||
}
|
||||
|
||||
// Remove removes an entry from the cache
|
||||
func (c *Cache) Remove(ctx context.Context, userID, spaceID, shareID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userID)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
defer unlock()
|
||||
|
||||
ctx, span = appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Add")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID), attribute.String("cs3.spaceid", spaceID))
|
||||
|
||||
persistFunc := func() error {
|
||||
c.initializeIfNeeded(userID, spaceID)
|
||||
|
||||
rss, _ := c.ReceivedSpaces.Load(userID)
|
||||
receivedSpace := rss.Spaces[spaceID]
|
||||
if receivedSpace.States == nil {
|
||||
receivedSpace.States = map[string]*State{}
|
||||
}
|
||||
delete(receivedSpace.States, shareID)
|
||||
if len(receivedSpace.States) == 0 {
|
||||
delete(rss.Spaces, spaceID)
|
||||
}
|
||||
|
||||
return c.persist(ctx, userID)
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("userID", userID).
|
||||
Str("spaceID", spaceID).Logger()
|
||||
|
||||
var err error
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting added received share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting added received share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
case errtypes.AlreadyExists:
|
||||
log.Debug().Msg("already exists when persisting added received share. retrying...")
|
||||
// CS3 uses an already exists error instead of precondition failed when using an If-None-Match=* header / IfExists flag in the InitiateFileUpload call.
|
||||
// Thas happens when the cache thinks there is no file.
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting added received share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting added received share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, userID); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Error().Err(err).Msg("persisting added received share failed. giving up.")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// List returns a list of received shares for a given user
|
||||
// The return list is guaranteed to be thread-safe
|
||||
func (c *Cache) List(ctx context.Context, userID string) (map[string]*Space, error) {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userID)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
defer unlock()
|
||||
|
||||
err := c.syncWithLock(ctx, userID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spaces := map[string]*Space{}
|
||||
rss, _ := c.ReceivedSpaces.Load(userID)
|
||||
for spaceID, space := range rss.Spaces {
|
||||
spaceCopy := &Space{
|
||||
States: map[string]*State{},
|
||||
}
|
||||
for shareID, state := range space.States {
|
||||
spaceCopy.States[shareID] = &State{
|
||||
State: state.State,
|
||||
MountPoint: state.MountPoint,
|
||||
Hidden: state.Hidden,
|
||||
}
|
||||
}
|
||||
spaces[spaceID] = spaceCopy
|
||||
}
|
||||
return spaces, nil
|
||||
}
|
||||
|
||||
func (c *Cache) syncWithLock(ctx context.Context, userID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Sync")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
|
||||
log := appctx.GetLogger(ctx).With().Str("userID", userID).Logger()
|
||||
|
||||
c.initializeIfNeeded(userID, "")
|
||||
|
||||
jsonPath := userJSONPath(userID)
|
||||
span.AddEvent("updating cache")
|
||||
// - update cached list of created shares for the user in memory if changed
|
||||
rss, _ := c.ReceivedSpaces.Load(userID)
|
||||
dlres, err := c.storage.Download(ctx, metadata.DownloadRequest{
|
||||
Path: jsonPath,
|
||||
IfNoneMatch: []string{rss.etag},
|
||||
})
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.AddEvent("updating local cache")
|
||||
case errtypes.NotFound:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.NotModified:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("Failed to download the received share: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("Failed to download the received share")
|
||||
return err
|
||||
}
|
||||
|
||||
newSpaces := &Spaces{}
|
||||
err = json.Unmarshal(dlres.Content, newSpaces)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("Failed to unmarshal the received share: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("Failed to unmarshal the received share")
|
||||
return err
|
||||
}
|
||||
newSpaces.etag = dlres.Etag
|
||||
|
||||
c.ReceivedSpaces.Store(userID, newSpaces)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// persist persists the data for one user to the storage
|
||||
func (c *Cache) persist(ctx context.Context, userID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Persist")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
|
||||
rss, ok := c.ReceivedSpaces.Load(userID)
|
||||
if !ok {
|
||||
span.SetStatus(codes.Ok, "no received shares")
|
||||
return nil
|
||||
}
|
||||
|
||||
createdBytes, err := json.Marshal(rss)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
jsonPath := userJSONPath(userID)
|
||||
if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
ur := metadata.UploadRequest{
|
||||
Path: jsonPath,
|
||||
Content: createdBytes,
|
||||
IfMatchEtag: rss.etag,
|
||||
}
|
||||
// when there is no etag in memory make sure the file has not been created on the server, see https://www.rfc-editor.org/rfc/rfc9110#field.if-match
|
||||
// > If the field value is "*", the condition is false if the origin server has a current representation for the target resource.
|
||||
if rss.etag == "" {
|
||||
ur.IfNoneMatch = []string{"*"}
|
||||
}
|
||||
|
||||
res, err := c.storage.Upload(ctx, ur)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
rss.etag = res.Etag
|
||||
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
func userJSONPath(userID string) string {
|
||||
return filepath.Join("/users", userID, "received.json")
|
||||
}
|
||||
|
||||
func (c *Cache) initializeIfNeeded(userID, spaceID string) {
|
||||
rss, _ := c.ReceivedSpaces.LoadOrStore(userID, &Spaces{Spaces: map[string]*Space{}})
|
||||
if spaceID != "" && rss.Spaces[spaceID] == nil {
|
||||
rss.Spaces[spaceID] = &Space{}
|
||||
c.ReceivedSpaces.Store(userID, rss)
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+371
@@ -0,0 +1,371 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package sharecache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
)
|
||||
|
||||
// name is the Tracer name used to identify this instrumentation library.
|
||||
const tracerName = "sharecache"
|
||||
|
||||
// Cache caches the list of share ids for users/groups
|
||||
// It functions as an in-memory cache with a persistence layer
|
||||
// The storage is sharded by user/group
|
||||
type Cache struct {
|
||||
lockMap sync.Map
|
||||
|
||||
UserShares mtimesyncedcache.Map[string, *UserShareCache]
|
||||
|
||||
storage metadata.Storage
|
||||
namespace string
|
||||
filename string
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// UserShareCache holds the space/share map for one user
|
||||
type UserShareCache struct {
|
||||
UserShares map[string]*SpaceShareIDs
|
||||
|
||||
Etag string
|
||||
}
|
||||
|
||||
// SpaceShareIDs holds the unique list of share ids for a space
|
||||
type SpaceShareIDs struct {
|
||||
IDs map[string]struct{}
|
||||
}
|
||||
|
||||
func (c *Cache) lockUser(userID string) func() {
|
||||
v, _ := c.lockMap.LoadOrStore(userID, &sync.Mutex{})
|
||||
lock := v.(*sync.Mutex)
|
||||
|
||||
lock.Lock()
|
||||
return func() { lock.Unlock() }
|
||||
}
|
||||
|
||||
// New returns a new Cache instance
|
||||
func New(s metadata.Storage, namespace, filename string, ttl time.Duration) Cache {
|
||||
return Cache{
|
||||
UserShares: mtimesyncedcache.Map[string, *UserShareCache]{},
|
||||
storage: s,
|
||||
namespace: namespace,
|
||||
filename: filename,
|
||||
ttl: ttl,
|
||||
lockMap: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds a share to the cache
|
||||
func (c *Cache) Add(ctx context.Context, userid, shareID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userid)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid))
|
||||
defer unlock()
|
||||
|
||||
if _, ok := c.UserShares.Load(userid); !ok {
|
||||
err := c.syncWithLock(ctx, userid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, span = appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Add")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid), attribute.String("cs3.shareid", shareID))
|
||||
|
||||
storageid, spaceid, _ := shareid.Decode(shareID)
|
||||
ssid := storageid + shareid.IDDelimiter + spaceid
|
||||
|
||||
persistFunc := func() error {
|
||||
c.initializeIfNeeded(userid, ssid)
|
||||
|
||||
// add share id
|
||||
us, _ := c.UserShares.Load(userid)
|
||||
us.UserShares[ssid].IDs[shareID] = struct{}{}
|
||||
return c.Persist(ctx, userid)
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("userID", userid).
|
||||
Str("shareID", shareID).Logger()
|
||||
|
||||
var err error
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting added share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting added share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
case errtypes.AlreadyExists:
|
||||
log.Debug().Msg("already exists when persisting added share. retrying...")
|
||||
// CS3 uses an already exists error instead of precondition failed when using an If-None-Match=* header / IfExists flag in the InitiateFileUpload call.
|
||||
// Thas happens when the cache thinks there is no file.
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting added share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting added share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, userid); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Error().Err(err).Msg("persisting added share failed. giving up.")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove removes a share for the given user
|
||||
func (c *Cache) Remove(ctx context.Context, userid, shareID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userid)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid))
|
||||
defer unlock()
|
||||
|
||||
if _, ok := c.UserShares.Load(userid); ok {
|
||||
err := c.syncWithLock(ctx, userid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ctx, span = appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Remove")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid), attribute.String("cs3.shareid", shareID))
|
||||
|
||||
storageid, spaceid, _ := shareid.Decode(shareID)
|
||||
ssid := storageid + shareid.IDDelimiter + spaceid
|
||||
|
||||
persistFunc := func() error {
|
||||
us, loaded := c.UserShares.LoadOrStore(userid, &UserShareCache{
|
||||
UserShares: map[string]*SpaceShareIDs{},
|
||||
})
|
||||
|
||||
if loaded {
|
||||
// remove share id
|
||||
delete(us.UserShares[ssid].IDs, shareID)
|
||||
}
|
||||
|
||||
return c.Persist(ctx, userid)
|
||||
}
|
||||
|
||||
log := appctx.GetLogger(ctx).With().
|
||||
Str("hostname", os.Getenv("HOSTNAME")).
|
||||
Str("userID", userid).
|
||||
Str("shareID", shareID).Logger()
|
||||
|
||||
var err error
|
||||
for retries := 100; retries > 0; retries-- {
|
||||
err = persistFunc()
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Msg("aborted when persisting removed share: etag changed. retrying...")
|
||||
// this is the expected status code from the server when the if-match etag check fails
|
||||
// continue with sync below
|
||||
case errtypes.PreconditionFailed:
|
||||
log.Debug().Msg("precondition failed when persisting removed share: etag changed. retrying...")
|
||||
// actually, this is the wrong status code and we treat it like errtypes.Aborted because of inconsistencies on the server side
|
||||
// continue with sync below
|
||||
case errtypes.AlreadyExists:
|
||||
log.Debug().Msg("file already existed when persisting removed share. retrying...")
|
||||
// CS3 uses an already exists error instead of precondition failed when using an If-None-Match=* header / IfExists flag in the InitiateFileUpload call.
|
||||
// Thas happens when the cache thinks there is no file.
|
||||
// continue with sync below
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("persisting removed share failed. giving up: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("persisting removed share failed")
|
||||
return err
|
||||
}
|
||||
if err := c.syncWithLock(ctx, userid); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// List return the list of spaces/shares for the given user/group
|
||||
func (c *Cache) List(ctx context.Context, userid string) (map[string]SpaceShareIDs, error) {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Grab lock")
|
||||
unlock := c.lockUser(userid)
|
||||
span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid))
|
||||
defer unlock()
|
||||
if err := c.syncWithLock(ctx, userid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r := map[string]SpaceShareIDs{}
|
||||
us, ok := c.UserShares.Load(userid)
|
||||
if !ok {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
for ssid, cached := range us.UserShares {
|
||||
r[ssid] = SpaceShareIDs{
|
||||
IDs: maps.Clone(cached.IDs),
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (c *Cache) syncWithLock(ctx context.Context, userID string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Sync")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userID))
|
||||
|
||||
log := appctx.GetLogger(ctx).With().Str("userID", userID).Logger()
|
||||
|
||||
c.initializeIfNeeded(userID, "")
|
||||
|
||||
userCreatedPath := c.userCreatedPath(userID)
|
||||
span.AddEvent("updating cache")
|
||||
// - update cached list of created shares for the user in memory if changed
|
||||
dlreq := metadata.DownloadRequest{
|
||||
Path: userCreatedPath,
|
||||
}
|
||||
if us, ok := c.UserShares.Load(userID); ok && us.Etag != "" {
|
||||
dlreq.IfNoneMatch = []string{us.Etag}
|
||||
}
|
||||
|
||||
dlres, err := c.storage.Download(ctx, dlreq)
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
span.AddEvent("updating local cache")
|
||||
case errtypes.NotFound:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
case errtypes.NotModified:
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
default:
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("Failed to download the share cache: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("Failed to download the share cache")
|
||||
return err
|
||||
}
|
||||
|
||||
newShareCache := &UserShareCache{}
|
||||
err = json.Unmarshal(dlres.Content, newShareCache)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, fmt.Sprintf("Failed to unmarshal the share cache: %s", err.Error()))
|
||||
log.Error().Err(err).Msg("Failed to unmarshal the share cache")
|
||||
return err
|
||||
}
|
||||
newShareCache.Etag = dlres.Etag
|
||||
|
||||
c.UserShares.Store(userID, newShareCache)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Persist persists the data for one user/group to the storage
|
||||
func (c *Cache) Persist(ctx context.Context, userid string) error {
|
||||
ctx, span := appctx.GetTracerProvider(ctx).Tracer(tracerName).Start(ctx, "Persist")
|
||||
defer span.End()
|
||||
span.SetAttributes(attribute.String("cs3.userid", userid))
|
||||
|
||||
us, ok := c.UserShares.Load(userid)
|
||||
if !ok {
|
||||
span.SetStatus(codes.Ok, "no user shares")
|
||||
return nil
|
||||
}
|
||||
createdBytes, err := json.Marshal(us)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
jsonPath := c.userCreatedPath(userid)
|
||||
if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
ur := metadata.UploadRequest{
|
||||
Path: jsonPath,
|
||||
Content: createdBytes,
|
||||
IfMatchEtag: us.Etag,
|
||||
}
|
||||
// when there is no etag in memory make sure the file has not been created on the server, see https://www.rfc-editor.org/rfc/rfc9110#field.if-match
|
||||
// > If the field value is "*", the condition is false if the origin server has a current representation for the target resource.
|
||||
if us.Etag == "" {
|
||||
ur.IfNoneMatch = []string{"*"}
|
||||
}
|
||||
|
||||
res, err := c.storage.Upload(ctx, ur)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
return err
|
||||
}
|
||||
us.Etag = res.Etag
|
||||
|
||||
span.SetStatus(codes.Ok, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cache) userCreatedPath(userid string) string {
|
||||
return filepath.Join("/", c.namespace, userid, c.filename)
|
||||
}
|
||||
|
||||
func (c *Cache) initializeIfNeeded(userid, ssid string) {
|
||||
us, _ := c.UserShares.LoadOrStore(userid, &UserShareCache{
|
||||
UserShares: map[string]*SpaceShareIDs{},
|
||||
})
|
||||
if ssid != "" && us.UserShares[ssid] == nil {
|
||||
us.UserShares[ssid] = &SpaceShareIDs{
|
||||
IDs: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+45
@@ -0,0 +1,45 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package shareid
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
// IDDelimiter is used to separate the providerid, spaceid and shareid
|
||||
IDDelimiter = ":"
|
||||
)
|
||||
|
||||
// Encode encodes a share id
|
||||
func Encode(providerID, spaceID, shareID string) string {
|
||||
return providerID + IDDelimiter + spaceID + IDDelimiter + shareID
|
||||
}
|
||||
|
||||
// Decode decodes an encoded shareid
|
||||
// share ids are of the format <storageid>:<spaceid>:<shareid>
|
||||
func Decode(id string) (string, string, string) {
|
||||
parts := strings.SplitN(id, IDDelimiter, 3)
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
return "", "", parts[0]
|
||||
case 2:
|
||||
return parts[0], parts[1], ""
|
||||
default:
|
||||
return parts[0], parts[1], parts[2]
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package loader
|
||||
|
||||
import (
|
||||
// Load core share manager drivers.
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/share/manager/memory"
|
||||
// Add your own here
|
||||
)
|
||||
+398
@@ -0,0 +1,398 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share"
|
||||
"github.com/rs/zerolog"
|
||||
"google.golang.org/genproto/protobuf/field_mask"
|
||||
|
||||
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
var counter uint64
|
||||
|
||||
func init() {
|
||||
registry.Register("memory", New)
|
||||
}
|
||||
|
||||
// New returns a new manager.
|
||||
func New(c map[string]any, _ *zerolog.Logger) (share.Manager, error) {
|
||||
state := map[string]map[*collaboration.ShareId]collaboration.ShareState{}
|
||||
mp := map[string]map[*collaboration.ShareId]*provider.Reference{}
|
||||
return &manager{
|
||||
shareState: state,
|
||||
shareMountPoint: mp,
|
||||
lock: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
lock *sync.Mutex
|
||||
shares []*collaboration.Share
|
||||
// shareState contains the share state for a user.
|
||||
// map["alice"]["share-id"]state.
|
||||
shareState map[string]map[*collaboration.ShareId]collaboration.ShareState
|
||||
// shareMountPoint contains the mountpoint of a share for a user.
|
||||
// map["alice"]["share-id"]reference.
|
||||
shareMountPoint map[string]map[*collaboration.ShareId]*provider.Reference
|
||||
}
|
||||
|
||||
func (m *manager) add(ctx context.Context, s *collaboration.Share) {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
m.shares = append(m.shares, s)
|
||||
}
|
||||
|
||||
func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) {
|
||||
id := atomic.AddUint64(&counter, 1)
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
now := time.Now().UnixNano()
|
||||
ts := &typespb.Timestamp{
|
||||
Seconds: uint64(now / 1000000000),
|
||||
Nanos: uint32(now % 1000000000),
|
||||
}
|
||||
|
||||
// check if share already exists.
|
||||
key := &collaboration.ShareKey{
|
||||
Owner: md.Owner,
|
||||
ResourceId: md.Id,
|
||||
Grantee: g.Grantee,
|
||||
}
|
||||
_, err := m.getByKey(ctx, key)
|
||||
// share already exists
|
||||
if err == nil {
|
||||
return nil, errtypes.AlreadyExists(key.String())
|
||||
}
|
||||
|
||||
s := &collaboration.Share{
|
||||
Id: &collaboration.ShareId{
|
||||
OpaqueId: fmt.Sprintf("%d", id),
|
||||
},
|
||||
ResourceId: md.Id,
|
||||
Permissions: g.Permissions,
|
||||
Grantee: g.Grantee,
|
||||
Owner: md.Owner,
|
||||
Creator: user.Id,
|
||||
Ctime: ts,
|
||||
Mtime: ts,
|
||||
}
|
||||
|
||||
m.add(ctx, s)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (m *manager) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
for _, s := range m.shares {
|
||||
if s.GetId().OpaqueId == id.OpaqueId {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
return nil, errtypes.NotFound(id.String())
|
||||
}
|
||||
|
||||
func (m *manager) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
for _, s := range m.shares {
|
||||
if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) &&
|
||||
utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
return nil, errtypes.NotFound(key.String())
|
||||
}
|
||||
|
||||
func (m *manager) get(ctx context.Context, ref *collaboration.ShareReference) (s *collaboration.Share, err error) {
|
||||
switch {
|
||||
case ref.GetId() != nil:
|
||||
s, err = m.getByID(ctx, ref.GetId())
|
||||
case ref.GetKey() != nil:
|
||||
s, err = m.getByKey(ctx, ref.GetKey())
|
||||
default:
|
||||
err = errtypes.NotFound(ref.String())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check if we are the owner
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
if share.IsCreatedByUser(s, user) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// or the grantee
|
||||
if s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER && utils.UserEqual(user.Id, s.Grantee.GetUserId()) {
|
||||
return s, nil
|
||||
} else if s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
|
||||
// check if all user groups match this share; TODO(labkode): filter shares created by us.
|
||||
for _, g := range user.Groups {
|
||||
if g == s.Grantee.GetGroupId().OpaqueId {
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// we return not found to not disclose information
|
||||
return nil, errtypes.NotFound(ref.String())
|
||||
}
|
||||
|
||||
func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) {
|
||||
share, err := m.get(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return share, nil
|
||||
}
|
||||
|
||||
func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
for i, s := range m.shares {
|
||||
if sharesEqual(ref, s) {
|
||||
if share.IsCreatedByUser(s, user) {
|
||||
m.shares[len(m.shares)-1], m.shares[i] = m.shares[i], m.shares[len(m.shares)-1]
|
||||
m.shares = m.shares[:len(m.shares)-1]
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return errtypes.NotFound(ref.String())
|
||||
}
|
||||
|
||||
func sharesEqual(ref *collaboration.ShareReference, s *collaboration.Share) bool {
|
||||
if ref.GetId() != nil && s.Id != nil {
|
||||
if ref.GetId().OpaqueId == s.Id.OpaqueId {
|
||||
return true
|
||||
}
|
||||
} else if ref.GetKey() != nil {
|
||||
if (utils.UserEqual(ref.GetKey().Owner, s.Owner) || utils.UserEqual(ref.GetKey().Owner, s.Creator)) &&
|
||||
utils.ResourceIDEqual(ref.GetKey().ResourceId, s.ResourceId) && utils.GranteeEqual(ref.GetKey().Grantee, s.Grantee) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions, updated *collaboration.Share, fieldMask *field_mask.FieldMask) (*collaboration.Share, error) {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
var shareRef *collaboration.ShareReference
|
||||
if ref != nil {
|
||||
shareRef = ref
|
||||
} else if updated != nil {
|
||||
shareRef = &collaboration.ShareReference{
|
||||
Spec: &collaboration.ShareReference_Id{
|
||||
Id: updated.Id,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for i, s := range m.shares {
|
||||
if sharesEqual(shareRef, s) {
|
||||
if share.IsCreatedByUser(s, user) {
|
||||
now := time.Now().UnixNano()
|
||||
if p != nil {
|
||||
m.shares[i].Permissions = p
|
||||
}
|
||||
if fieldMask != nil {
|
||||
for _, path := range fieldMask.Paths {
|
||||
switch path {
|
||||
case "permissions":
|
||||
m.shares[i].Permissions = updated.Permissions
|
||||
case "expiration":
|
||||
m.shares[i].Expiration = updated.Expiration
|
||||
default:
|
||||
return nil, errtypes.NotSupported("updating " + path + " is not supported")
|
||||
}
|
||||
}
|
||||
}
|
||||
m.shares[i].Mtime = &typespb.Timestamp{
|
||||
Seconds: uint64(now / 1000000000),
|
||||
Nanos: uint32(now % 1000000000),
|
||||
}
|
||||
return m.shares[i], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, errtypes.NotFound(ref.String())
|
||||
}
|
||||
|
||||
func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) {
|
||||
var ss []*collaboration.Share
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
for _, s := range m.shares {
|
||||
if share.IsCreatedByUser(s, user) {
|
||||
// no filter we return earlier
|
||||
if len(filters) == 0 {
|
||||
ss = append(ss, s)
|
||||
continue
|
||||
}
|
||||
// check filters
|
||||
if share.MatchesFilters(s, filters) {
|
||||
ss = append(ss, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
// we list the shares that are targeted to the user in context or to the user groups.
|
||||
func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter, forUser *userv1beta1.UserId) ([]*collaboration.ReceivedShare, error) {
|
||||
var rss []*collaboration.ReceivedShare
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
if user.GetId().GetType() == userv1beta1.UserType_USER_TYPE_SERVICE {
|
||||
// TODO: gateway missing!
|
||||
return nil, errors.New("can't use inmem share manager and service accounts")
|
||||
}
|
||||
for _, s := range m.shares {
|
||||
if share.IsCreatedByUser(s, user) || !share.IsGrantedToUser(s, user) {
|
||||
// omit shares created by the user or shares the user can't access
|
||||
continue
|
||||
}
|
||||
|
||||
if len(filters) == 0 {
|
||||
rs := m.convert(ctx, s)
|
||||
rss = append(rss, rs)
|
||||
continue
|
||||
}
|
||||
|
||||
if share.MatchesFilters(s, filters) {
|
||||
rs := m.convert(ctx, s)
|
||||
rss = append(rss, rs)
|
||||
}
|
||||
}
|
||||
return rss, nil
|
||||
}
|
||||
|
||||
// convert must be called in a lock-controlled block.
|
||||
func (m *manager) convert(ctx context.Context, s *collaboration.Share) *collaboration.ReceivedShare {
|
||||
rs := &collaboration.ReceivedShare{
|
||||
Share: s,
|
||||
State: collaboration.ShareState_SHARE_STATE_PENDING,
|
||||
}
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
if v, ok := m.shareState[user.Id.String()]; ok {
|
||||
if state, ok := v[s.Id]; ok {
|
||||
rs.State = state
|
||||
}
|
||||
}
|
||||
if v, ok := m.shareMountPoint[user.Id.String()]; ok {
|
||||
if mp, ok := v[s.Id]; ok {
|
||||
rs.MountPoint = mp
|
||||
}
|
||||
}
|
||||
return rs
|
||||
}
|
||||
|
||||
func (m *manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) {
|
||||
return m.getReceived(ctx, ref)
|
||||
}
|
||||
|
||||
func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
user := ctxpkg.ContextMustGetUser(ctx)
|
||||
for _, s := range m.shares {
|
||||
if sharesEqual(ref, s) {
|
||||
if user.GetId().GetType() == userv1beta1.UserType_USER_TYPE_SERVICE || share.IsGrantedToUser(s, user) {
|
||||
rs := m.convert(ctx, s)
|
||||
return rs, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, errtypes.NotFound(ref.String())
|
||||
}
|
||||
|
||||
func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask, forUser *userv1beta1.UserId) (*collaboration.ReceivedShare, error) {
|
||||
rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
|
||||
for i := range fieldMask.Paths {
|
||||
switch fieldMask.Paths[i] {
|
||||
case "state":
|
||||
rs.State = receivedShare.State
|
||||
case "mount_point":
|
||||
rs.MountPoint = receivedShare.MountPoint
|
||||
case "hidden":
|
||||
continue
|
||||
default:
|
||||
return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported")
|
||||
}
|
||||
}
|
||||
|
||||
u := ctxpkg.ContextMustGetUser(ctx)
|
||||
uid := u.GetId().String()
|
||||
if u.GetId().GetType() == userv1beta1.UserType_USER_TYPE_SERVICE {
|
||||
uid = forUser.String()
|
||||
}
|
||||
|
||||
// Persist state
|
||||
if v, ok := m.shareState[uid]; ok {
|
||||
v[rs.Share.Id] = rs.State
|
||||
m.shareState[uid] = v
|
||||
} else {
|
||||
a := map[*collaboration.ShareId]collaboration.ShareState{
|
||||
rs.Share.Id: rs.State,
|
||||
}
|
||||
m.shareState[uid] = a
|
||||
}
|
||||
// Persist mount point
|
||||
if v, ok := m.shareMountPoint[uid]; ok {
|
||||
v[rs.Share.Id] = rs.MountPoint
|
||||
m.shareMountPoint[uid] = v
|
||||
} else {
|
||||
a := map[*collaboration.ShareId]*provider.Reference{
|
||||
rs.Share.Id: rs.MountPoint,
|
||||
}
|
||||
m.shareMountPoint[uid] = a
|
||||
}
|
||||
|
||||
return rs, nil
|
||||
}
|
||||
Generated
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package registry
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// NewFunc is the function that share managers
|
||||
// should register at init time.
|
||||
type NewFunc func(map[string]any, *zerolog.Logger) (share.Manager, error)
|
||||
|
||||
// NewFuncs is a map containing all the registered share managers.
|
||||
var NewFuncs = map[string]NewFunc{}
|
||||
|
||||
// Register registers a new share manager new function.
|
||||
// Not safe for concurrent use. Safe for use from package init.
|
||||
func Register(name string, f NewFunc) {
|
||||
NewFuncs[name] = f
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package share
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/grants"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"google.golang.org/genproto/protobuf/field_mask"
|
||||
)
|
||||
|
||||
const (
|
||||
// NoState can be used to signal the filter matching functions to ignore the share state.
|
||||
NoState collaboration.ShareState = -1
|
||||
)
|
||||
|
||||
// Metadata contains Metadata for a share
|
||||
type Metadata struct {
|
||||
ETag string
|
||||
Mtime *types.Timestamp
|
||||
}
|
||||
|
||||
// Manager is the interface that manipulates shares.
|
||||
type Manager interface {
|
||||
// Create a new share in fn with the given acl.
|
||||
Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error)
|
||||
|
||||
// GetShare gets the information for a share by the given ref.
|
||||
GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error)
|
||||
|
||||
// Unshare deletes the share pointed by ref.
|
||||
Unshare(ctx context.Context, ref *collaboration.ShareReference) error
|
||||
|
||||
// UpdateShare updates the mode of the given share.
|
||||
UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions, updated *collaboration.Share, fieldMask *field_mask.FieldMask) (*collaboration.Share, error)
|
||||
|
||||
// ListShares returns the shares created by the user. If md is provided is not nil,
|
||||
// it returns only shares attached to the given resource.
|
||||
ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error)
|
||||
|
||||
// ListReceivedShares returns the list of shares the user has access to. `forUser` parameter for service accounts only
|
||||
ListReceivedShares(ctx context.Context, filters []*collaboration.Filter, forUser *userv1beta1.UserId) ([]*collaboration.ReceivedShare, error)
|
||||
|
||||
// GetReceivedShare returns the information for a received share.
|
||||
GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error)
|
||||
|
||||
// UpdateReceivedShare updates the received share with share state.`forUser` parameter for service accounts only
|
||||
UpdateReceivedShare(ctx context.Context, share *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask, forUser *userv1beta1.UserId) (*collaboration.ReceivedShare, error)
|
||||
}
|
||||
|
||||
// ReceivedShareWithUser holds the relevant information for representing a received share of a user
|
||||
type ReceivedShareWithUser struct {
|
||||
UserID *userv1beta1.UserId
|
||||
ReceivedShare *collaboration.ReceivedShare
|
||||
}
|
||||
|
||||
// DumpableManager defines a share manager which supports dumping its contents
|
||||
type DumpableManager interface {
|
||||
Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- ReceivedShareWithUser) error
|
||||
}
|
||||
|
||||
// LoadableManager defines a share manager which supports loading contents from a dump
|
||||
type LoadableManager interface {
|
||||
Load(ctx context.Context, shareChan <-chan *collaboration.Share, receivedShareChan <-chan ReceivedShareWithUser) error
|
||||
}
|
||||
|
||||
// GroupGranteeFilter is an abstraction for creating filter by grantee type group.
|
||||
func GroupGranteeFilter() *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_GRANTEE_TYPE,
|
||||
Term: &collaboration.Filter_GranteeType{
|
||||
GranteeType: provider.GranteeType_GRANTEE_TYPE_GROUP,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// UserGranteeFilter is an abstraction for creating filter by grantee type user.
|
||||
func UserGranteeFilter() *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_GRANTEE_TYPE,
|
||||
Term: &collaboration.Filter_GranteeType{
|
||||
GranteeType: provider.GranteeType_GRANTEE_TYPE_USER,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ResourceIDFilter is an abstraction for creating filter by resource id.
|
||||
func ResourceIDFilter(id *provider.ResourceId) *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_RESOURCE_ID,
|
||||
Term: &collaboration.Filter_ResourceId{
|
||||
ResourceId: id,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SpaceIDFilter is an abstraction for creating filter by space id.
|
||||
func SpaceIDFilter(id string) *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_SPACE_ID,
|
||||
Term: &collaboration.Filter_SpaceId{
|
||||
SpaceId: id,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// StateFilter is an abstraction for creating filter by share state.
|
||||
func StateFilter(state collaboration.ShareState) *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_STATE,
|
||||
Term: &collaboration.Filter_State{
|
||||
State: state,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SpaceRootFilter is an abstraction for filtering shares by whether the shared
|
||||
// resource is a space root. Pass true to include only space-root shares (space
|
||||
// membership), false to exclude them (file/folder shares only).
|
||||
func SpaceRootFilter(spaceRoot bool) *collaboration.Filter {
|
||||
return &collaboration.Filter{
|
||||
Type: collaboration.Filter_TYPE_SPACE_ROOT,
|
||||
Term: &collaboration.Filter_SpaceRoot{
|
||||
SpaceRoot: spaceRoot,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// IsCreatedByUser checks if the user is the owner or creator of the share.
|
||||
func IsCreatedByUser(share *collaboration.Share, user *userv1beta1.User) bool {
|
||||
return utils.UserEqual(user.Id, share.Owner) || utils.UserEqual(user.Id, share.Creator)
|
||||
}
|
||||
|
||||
// IsGrantedToUser checks if the user is a grantee of the share. Either by a user grant or by a group grant.
|
||||
func IsGrantedToUser(share *collaboration.Share, user *userv1beta1.User) bool {
|
||||
if share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER && utils.UserEqual(user.Id, share.Grantee.GetUserId()) {
|
||||
return true
|
||||
}
|
||||
if share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
|
||||
// check if any of the user's group is the grantee of the share
|
||||
for _, g := range user.Groups {
|
||||
if g == share.Grantee.GetGroupId().OpaqueId {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MatchesFilter tests if the share passes the filter.
|
||||
func MatchesFilter(share *collaboration.Share, state collaboration.ShareState, filter *collaboration.Filter) bool {
|
||||
switch filter.Type {
|
||||
case collaboration.Filter_TYPE_RESOURCE_ID:
|
||||
return utils.ResourceIDEqual(share.ResourceId, filter.GetResourceId())
|
||||
case collaboration.Filter_TYPE_GRANTEE_TYPE:
|
||||
return share.Grantee.Type == filter.GetGranteeType()
|
||||
case collaboration.Filter_TYPE_EXCLUDE_DENIALS:
|
||||
// This filter type is used to filter out "denial shares". These are currently implemented by having the permission "0".
|
||||
// I.e. if the permission is 0 we don't want to show it.
|
||||
return !grants.PermissionsEqual(share.Permissions.Permissions, &provider.ResourcePermissions{})
|
||||
case collaboration.Filter_TYPE_SPACE_ID:
|
||||
return share.ResourceId.SpaceId == filter.GetSpaceId()
|
||||
case collaboration.Filter_TYPE_STATE:
|
||||
return state == filter.GetState()
|
||||
case collaboration.Filter_TYPE_SPACE_ROOT:
|
||||
isSpaceRoot := share.ResourceId.SpaceId == share.ResourceId.OpaqueId
|
||||
return isSpaceRoot == filter.GetSpaceRoot()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// MatchesAnyFilter checks if the share passes at least one of the given filters.
|
||||
func MatchesAnyFilter(share *collaboration.Share, state collaboration.ShareState, filters []*collaboration.Filter) bool {
|
||||
for _, f := range filters {
|
||||
if MatchesFilter(share, state, f) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MatchesFilters checks if the share passes the given filters.
|
||||
// Filters of the same type form a disjuntion, a logical OR. Filters of separate type form a conjunction, a logical AND.
|
||||
// Here is an example:
|
||||
// (resource_id=1 OR resource_id=2) AND (grantee_type=USER OR grantee_type=GROUP)
|
||||
func MatchesFilters(share *collaboration.Share, filters []*collaboration.Filter) bool {
|
||||
if len(filters) == 0 {
|
||||
return true
|
||||
}
|
||||
grouped := GroupFiltersByType(filters)
|
||||
for _, f := range grouped {
|
||||
if !MatchesAnyFilter(share, NoState, f) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MatchesFiltersWithState checks if the share passes the given filters.
|
||||
// This can check filter by share state
|
||||
// Filters of the same type form a disjuntion, a logical OR. Filters of separate type form a conjunction, a logical AND.
|
||||
// Here is an example:
|
||||
// (resource_id=1 OR resource_id=2) AND (grantee_type=USER OR grantee_type=GROUP)
|
||||
func MatchesFiltersWithState(share *collaboration.Share, state collaboration.ShareState, filters []*collaboration.Filter) bool {
|
||||
if len(filters) == 0 {
|
||||
return true
|
||||
}
|
||||
grouped := GroupFiltersByType(filters)
|
||||
for _, f := range grouped {
|
||||
if !MatchesAnyFilter(share, state, f) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// GroupFiltersByType groups the given filters and returns a map using the filter type as the key.
|
||||
func GroupFiltersByType(filters []*collaboration.Filter) map[collaboration.Filter_Type][]*collaboration.Filter {
|
||||
grouped := make(map[collaboration.Filter_Type][]*collaboration.Filter)
|
||||
for _, f := range filters {
|
||||
grouped[f.Type] = append(grouped[f.Type], f)
|
||||
}
|
||||
return grouped
|
||||
}
|
||||
|
||||
// FilterFiltersByType returns a slice of filters by a given type.
|
||||
// If no filter with the given type exists within the filters, then an
|
||||
// empty slice is returned.
|
||||
func FilterFiltersByType(f []*collaboration.Filter, t collaboration.Filter_Type) []*collaboration.Filter {
|
||||
return GroupFiltersByType(f)[t]
|
||||
}
|
||||
|
||||
// IsExpired tests whether a share is expired
|
||||
func IsExpired(s *collaboration.Share) bool {
|
||||
if e := s.GetExpiration(); e != nil {
|
||||
expiration := time.Unix(int64(e.Seconds), int64(e.Nanos))
|
||||
return expiration.Before(time.Now())
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user