chore: bump reva to v2.22.0

This commit is contained in:
Michael Barz
2024-07-29 16:44:23 +02:00
parent 74a0c03246
commit d7adeacc37
12 changed files with 293 additions and 141 deletions
@@ -53,7 +53,7 @@ import (
const getUserCtxErrMsg = "error getting user from context"
func init() {
rgrpc.Register("publicshareprovider", New)
rgrpc.Register("publicshareprovider", NewDefault)
}
type config struct {
@@ -127,9 +127,8 @@ func parsePasswordPolicy(m map[string]interface{}) (*passwordPolicy, error) {
return p, nil
}
// New creates a new user share provider svc
func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
// New creates a new public share provider svc initialized from defaults
func NewDefault(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
@@ -146,6 +145,15 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
return nil, err
}
gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr))
if err != nil {
return nil, err
}
return New(gatewaySelector, sm, c, p)
}
// New creates a new user share provider svc
func New(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], sm publicshare.Manager, c *config, p *passwordPolicy) (rgrpc.Service, error) {
allowedPathsForShares := make([]*regexp.Regexp, 0, len(c.AllowedPathsForShares))
for _, s := range c.AllowedPathsForShares {
regex, err := regexp.Compile(s)
@@ -155,11 +163,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
allowedPathsForShares = append(allowedPathsForShares, regex)
}
gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr))
if err != nil {
return nil, err
}
service := &service{
conf: c,
sm: sm,
@@ -233,7 +236,7 @@ func (s *service) CreatePublicShare(ctx context.Context, req *link.CreatePublicS
}
// check that user has share permissions
if !sRes.GetInfo().GetPermissionSet().AddGrant {
if !isInternalLink && !sRes.GetInfo().GetPermissionSet().AddGrant {
return &link.CreatePublicShareResponse{
Status: status.NewInvalidArg(ctx, "no share permission"),
}, nil
@@ -477,14 +480,12 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
isInternalLink := isInternalLink(req, ps)
// users should always be able to downgrade links to internal links
// when they are the creator of the link
// all other users should have the WritePublicLink permission
if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) {
// check if the user has the permission in the user role
if !publicshare.IsCreatedByUser(ps, user) {
canWriteLink, err := utils.CheckPermission(ctx, permission.WritePublicLink, gatewayClient)
if err != nil {
return &link.UpdatePublicShareResponse{
Status: status.NewInternal(ctx, "error loading public share"),
Status: status.NewInternal(ctx, "error checking permission to write public share"),
}, err
}
if !canWriteLink {
@@ -501,8 +502,14 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
Status: status.NewInternal(ctx, "failed to stat shared resource"),
}, err
}
if sRes.Status.Code != rpc.Code_CODE_OK {
return &link.UpdatePublicShareResponse{
Status: sRes.GetStatus(),
}, nil
if !publicshare.IsCreatedByUser(ps, user) {
}
if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) {
if !sRes.GetInfo().GetPermissionSet().UpdateGrant {
return &link.UpdatePublicShareResponse{
Status: status.NewPermissionDenied(ctx, nil, "no permission to update public share"),
@@ -547,12 +554,16 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
}
// enforce password if needed
canOptOut, err := utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient)
if err != nil {
return &link.UpdatePublicShareResponse{
Status: status.NewInternal(ctx, err.Error()),
}, nil
var canOptOut bool
if !isInternalLink {
canOptOut, err = utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient)
if err != nil {
return &link.UpdatePublicShareResponse{
Status: status.NewInternal(ctx, err.Error()),
}, nil
}
}
updatePassword := req.GetUpdate().GetType() == link.UpdatePublicShareRequest_Update_TYPE_PASSWORD
setPassword := grant.GetPassword()
+204 -98
View File
@@ -46,52 +46,89 @@ import (
type ScanDebouncer struct {
after time.Duration
f func(item scanItem)
pending map[string]*time.Timer
pending sync.Map
inProgress sync.Map
mutex sync.Mutex
}
type EventAction int
const (
ActionCreate EventAction = iota
ActionUpdate
ActionMove
ActionDelete
)
type queueItem struct {
item scanItem
timer *time.Timer
}
// NewScanDebouncer returns a new SpaceDebouncer instance
func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer {
return &ScanDebouncer{
after: d,
f: f,
pending: map[string]*time.Timer{},
pending: sync.Map{},
inProgress: sync.Map{},
}
}
// Debounce restars the debounce timer for the given space
// Debounce restarts the debounce timer for the given space
func (d *ScanDebouncer) Debounce(item scanItem) {
d.mutex.Lock()
defer d.mutex.Unlock()
path := item.Path
force := item.ForceRescan
if t := d.pending[item.Path]; t != nil {
force = force || item.ForceRescan
t.Stop()
recurse := item.Recurse
if i, ok := d.pending.Load(item.Path); ok {
queueItem := i.(*queueItem)
force = force || queueItem.item.ForceRescan
recurse = recurse || queueItem.item.Recurse
queueItem.timer.Stop()
}
d.pending[item.Path] = time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(path); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
d.pending[path].Reset(d.after)
d.mutex.Unlock()
return
}
d.pending.Store(item.Path, &queueItem{
item: item,
timer: time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(path); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
if i, ok := d.pending.Load(path); ok {
i.(*queueItem).timer.Reset(d.after)
}
d.inProgress.Store(path, true)
defer d.inProgress.Delete(path)
d.f(scanItem{
Path: path,
ForceRescan: force,
})
d.mutex.Unlock()
return
}
d.pending.Delete(path)
d.inProgress.Store(path, true)
defer d.inProgress.Delete(path)
d.f(scanItem{
Path: path,
ForceRescan: force,
Recurse: recurse,
})
}),
})
}
// InProgress returns true if the given path is currently being processed
func (d *ScanDebouncer) InProgress(path string) bool {
d.mutex.Lock()
defer d.mutex.Unlock()
if _, ok := d.pending.Load(path); ok {
return true
}
_, ok := d.inProgress.Load(path)
return ok
}
func (t *Tree) workScanQueue() {
for i := 0; i < t.options.MaxConcurrency; i++ {
go func() {
@@ -103,17 +140,73 @@ func (t *Tree) workScanQueue() {
log.Error().Err(err).Str("path", item.Path).Msg("failed to assimilate item")
continue
}
if item.Recurse {
err = t.WarmupIDCache(item.Path, true)
if err != nil {
log.Error().Err(err).Str("path", item.Path).Msg("failed to warmup id cache")
}
}
}
}()
}
}
// Scan scans the given path and updates the id chache
func (t *Tree) Scan(path string, forceRescan bool) error {
t.scanDebouncer.Debounce(scanItem{
Path: path,
ForceRescan: forceRescan,
})
func (t *Tree) Scan(path string, action EventAction, isDir bool, recurse bool) error {
// cases:
switch action {
case ActionCreate:
if !isDir {
// 1. New file (could be emitted as part of a new directory)
// -> assimilate file
// -> scan parent directory recursively
if !t.scanDebouncer.InProgress(filepath.Dir(path)) {
t.scanDebouncer.Debounce(scanItem{
Path: path,
ForceRescan: false,
})
}
t.scanDebouncer.Debounce(scanItem{
Path: filepath.Dir(path),
ForceRescan: true,
Recurse: true,
})
} else {
// 2. New directory
// -> scan directory
t.scanDebouncer.Debounce(scanItem{
Path: path,
ForceRescan: true,
Recurse: true,
})
}
case ActionUpdate:
// 3. Updated file
// -> update file unless parent directory is being rescanned
if !t.scanDebouncer.InProgress(filepath.Dir(path)) {
t.scanDebouncer.Debounce(scanItem{
Path: path,
ForceRescan: true,
})
}
case ActionMove:
// 4. Moved file
// -> update file
// 5. Moved directory
// -> update directory and all children
t.scanDebouncer.Debounce(scanItem{
Path: path,
ForceRescan: isDir,
Recurse: isDir,
})
case ActionDelete:
_ = t.HandleFileDelete(path)
}
return nil
}
@@ -183,44 +276,43 @@ func (t *Tree) getOwnerAndIDs(path string) (*userv1beta1.UserId, string, string,
return owner, nodeID, spaceID, parentID, nil
}
func (t *Tree) assimilate(item scanItem) error {
var err error
func (t *Tree) findSpaceId(path string) (string, node.Attributes, error) {
// find the space id, scope by the according user
spaceID := []byte("")
spaceCandidate := item.Path
spaceCandidate := path
spaceAttrs := node.Attributes{}
for strings.HasPrefix(spaceCandidate, t.options.Root) {
spaceAttrs, err = t.lookup.MetadataBackend().All(context.Background(), spaceCandidate)
if err == nil && len(spaceAttrs[prefixes.SpaceIDAttr]) > 0 {
spaceID = spaceAttrs[prefixes.SpaceIDAttr]
spaceAttrs, err := t.lookup.MetadataBackend().All(context.Background(), spaceCandidate)
spaceID := spaceAttrs[prefixes.SpaceIDAttr]
if err == nil && len(spaceID) > 0 {
if t.options.UseSpaceGroups {
// set the uid and gid for the space
fi, err := os.Stat(spaceCandidate)
if err != nil {
return err
return "", spaceAttrs, err
}
sys := fi.Sys().(*syscall.Stat_t)
gid := int(sys.Gid)
_, err = t.userMapper.ScopeUserByIds(-1, gid)
if err != nil {
return err
return "", spaceAttrs, err
}
}
break
return string(spaceID), spaceAttrs, nil
}
spaceCandidate = filepath.Dir(spaceCandidate)
}
if len(spaceID) == 0 {
return fmt.Errorf("did not find space id for path")
}
return "", spaceAttrs, fmt.Errorf("could not find space for path %s", path)
}
func (t *Tree) assimilate(item scanItem) error {
var id []byte
if !item.ForceRescan {
// already assimilated?
id, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr)
if err == nil {
return t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path)
}
var err error
// First find the space id
spaceID, spaceAttrs, err := t.findSpaceId(item.Path)
if err != nil {
return err
}
// lock the file for assimilation
@@ -240,64 +332,62 @@ func (t *Tree) assimilate(item scanItem) error {
// check for the id attribute again after grabbing the lock, maybe the file was assimilated/created by us in the meantime
id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr)
if err == nil {
previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), string(spaceID), string(id))
previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), spaceID, string(id))
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path)
if item.ForceRescan {
previousParentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
if err != nil {
return err
// This item had already been assimilated in the past. Update the path
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path)
previousParentID, _ := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
fi, err := t.updateFile(item.Path, string(id), spaceID)
if err != nil {
return err
}
// was it moved?
if ok && len(previousParentID) > 0 && previousPath != item.Path {
// purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair.
_ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath)
_ = t.lookup.MetadataBackend().Purge(previousPath)
if fi.IsDir() {
// if it was moved and it is a directory we need to propagate the move
go func() { _ = t.WarmupIDCache(item.Path, false) }()
}
fi, err := t.updateFile(item.Path, string(id), string(spaceID))
if err != nil {
return err
}
// was it moved?
if ok && previousPath != item.Path {
// purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair.
_ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath)
_ = t.lookup.MetadataBackend().Purge(previousPath)
if fi.IsDir() {
// if it was moved and it is a directory we need to propagate the move
go func() { _ = t.WarmupIDCache(item.Path, false) }()
parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
if err == nil && len(parentID) > 0 {
ref := &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: t.options.MountID,
SpaceId: spaceID,
OpaqueId: string(parentID),
},
Path: filepath.Base(item.Path),
}
parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
if err == nil && len(parentID) > 0 {
ref := &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: t.options.MountID,
SpaceId: string(spaceID),
OpaqueId: string(parentID),
},
Path: filepath.Base(item.Path),
}
oldRef := &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: t.options.MountID,
SpaceId: string(spaceID),
OpaqueId: string(previousParentID),
},
Path: filepath.Base(previousPath),
}
t.PublishEvent(events.ItemMoved{
SpaceOwner: user,
Executant: user,
Owner: user,
Ref: ref,
OldReference: oldRef,
Timestamp: utils.TSNow(),
})
oldRef := &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: t.options.MountID,
SpaceId: spaceID,
OpaqueId: string(previousParentID),
},
Path: filepath.Base(previousPath),
}
t.PublishEvent(events.ItemMoved{
SpaceOwner: user,
Executant: user,
Owner: user,
Ref: ref,
OldReference: oldRef,
Timestamp: utils.TSNow(),
})
}
// }
}
} else {
// assimilate new file
newId := uuid.New().String()
fi, err := t.updateFile(item.Path, newId, string(spaceID))
fi, err := t.updateFile(item.Path, newId, spaceID)
if err != nil {
return err
}
@@ -305,7 +395,7 @@ func (t *Tree) assimilate(item scanItem) error {
ref := &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: t.options.MountID,
SpaceId: string(spaceID),
SpaceId: spaceID,
OpaqueId: newId,
},
}
@@ -455,17 +545,25 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error {
return err
}
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
sizes := make(map[string]int64)
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") {
// skip lock files
if isLockFile(path) {
return nil
}
// calculate tree sizes
if !info.IsDir() {
dir := filepath.Dir(path)
sizes[dir] += info.Size()
}
attribs, err := t.lookup.MetadataBackend().All(context.Background(), path)
if err == nil {
if err == nil && len(attribs[prefixes.IDAttr]) > 0 {
nodeSpaceID := attribs[prefixes.SpaceIDAttr]
if len(nodeSpaceID) > 0 {
spaceID = nodeSpaceID
@@ -496,10 +594,18 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error {
id, ok := attribs[prefixes.IDAttr]
if ok {
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), path)
} else if assimilate {
_ = t.Scan(path, false)
}
} else if assimilate {
_ = t.assimilate(scanItem{Path: path, ForceRescan: true})
}
return nil
})
if err != nil {
return err
}
for dir, size := range sizes {
_ = t.lookup.MetadataBackend().Set(context.Background(), dir, prefixes.TreesizeAttr, []byte(fmt.Sprintf("%d", size)))
}
return nil
}
@@ -61,15 +61,15 @@ start:
}
switch ev.Event {
case "CREATE":
go func() { _ = w.tree.Scan(ev.Path, false) }()
go func() { _ = w.tree.Scan(ev.Path, ActionCreate, false, false) }()
case "CLOSE":
bytesWritten, err := strconv.Atoi(ev.BytesWritten)
if err == nil && bytesWritten > 0 {
go func() { _ = w.tree.Scan(ev.Path, true) }()
go func() { _ = w.tree.Scan(ev.Path, ActionUpdate, false, true) }()
}
case "RENAME":
go func() {
_ = w.tree.Scan(ev.Path, true)
_ = w.tree.Scan(ev.Path, ActionMove, false, true)
_ = w.tree.WarmupIDCache(ev.Path, false)
}()
}
@@ -41,21 +41,21 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) {
continue
}
if strings.HasSuffix(lwev.Path, ".flock") || strings.HasSuffix(lwev.Path, ".mlock") {
if isLockFile(lwev.Path) {
continue
}
switch {
case strings.Contains(lwev.Event, "IN_CREATE"):
go func() { _ = w.tree.Scan(lwev.Path, false) }()
go func() { _ = w.tree.Scan(lwev.Path, ActionCreate, false, false) }()
case strings.Contains(lwev.Event, "IN_CLOSE_WRITE"):
bytesWritten, err := strconv.Atoi(lwev.BytesWritten)
if err == nil && bytesWritten > 0 {
go func() { _ = w.tree.Scan(lwev.Path, true) }()
go func() { _ = w.tree.Scan(lwev.Path, ActionUpdate, false, true) }()
}
case strings.Contains(lwev.Event, "IN_MOVED_TO"):
go func() {
_ = w.tree.Scan(lwev.Path, true)
_ = w.tree.Scan(lwev.Path, ActionMove, false, true)
_ = w.tree.WarmupIDCache(lwev.Path, false)
}()
}
@@ -2,7 +2,6 @@ package tree
import (
"fmt"
"strings"
"github.com/pablodz/inotifywaitgo/inotifywaitgo"
)
@@ -43,20 +42,20 @@ func (iw *InotifyWatcher) Watch(path string) {
select {
case event := <-events:
for _, e := range event.Events {
if strings.HasSuffix(event.Filename, ".flock") || strings.HasSuffix(event.Filename, ".mlock") {
if isLockFile(event.Filename) {
continue
}
switch e {
case inotifywaitgo.DELETE:
go func() { _ = iw.tree.HandleFileDelete(event.Filename) }()
case inotifywaitgo.CREATE:
go func() { _ = iw.tree.Scan(event.Filename, false) }()
go func() { _ = iw.tree.Scan(event.Filename, ActionCreate, event.IsDir, false) }()
case inotifywaitgo.MOVED_TO:
go func() {
_ = iw.tree.Scan(event.Filename, true)
_ = iw.tree.Scan(event.Filename, ActionMove, event.IsDir, true)
}()
case inotifywaitgo.CLOSE_WRITE:
go func() { _ = iw.tree.Scan(event.Filename, true) }()
go func() { _ = iw.tree.Scan(event.Filename, ActionUpdate, event.IsDir, true) }()
}
}
+10 -1
View File
@@ -75,6 +75,7 @@ type Watcher interface {
type scanItem struct {
Path string
ForceRescan bool
Recurse bool
}
// Tree manages a hierarchical tree
@@ -110,7 +111,7 @@ func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Opti
idCache: cache,
propagator: propagator.New(lu, &o.Options),
scanQueue: scanQueue,
scanDebouncer: NewScanDebouncer(500*time.Millisecond, func(item scanItem) {
scanDebouncer: NewScanDebouncer(1000*time.Millisecond, func(item scanItem) {
scanQueue <- item
}),
es: es,
@@ -393,6 +394,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
g.Go(func() error {
defer close(work)
for _, name := range names {
if isLockFile(name) {
continue
}
select {
case work <- name:
case <-ctx.Done():
@@ -877,3 +882,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) (
return
}
func isLockFile(path string) bool {
return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock")
}