Merge pull request #9708 from owncloud/update-reva
[full-ci] chore: bump reva to v2.22.0
This commit is contained in:
Michael Barz
@@ -1,4 +1,20 @@
|
||||
Enhancement: Bump reva
|
||||
Enhancement: Bump reva to v.2.22.0
|
||||
|
||||
* Bugfix [cs3org/reva#4741](https://github.com/cs3org/reva/pull/4741): Always find unique providers
|
||||
* Bugfix [cs3org/reva#4762](https://github.com/cs3org/reva/pull/4762): Blanks in dav Content-Disposition header
|
||||
* Bugfix [cs3org/reva#4775](https://github.com/cs3org/reva/pull/4775): Fixed the response code when copying the shared from to personal
|
||||
* Bugfix [cs3org/reva#4633](https://github.com/cs3org/reva/pull/4633): Allow all users to create internal links
|
||||
* Bugfix [cs3org/reva#4771](https://github.com/cs3org/reva/pull/4771): Deleting resources via their id
|
||||
* Bugfix [cs3org/reva#4768](https://github.com/cs3org/reva/pull/4768): Fixed the file name validation if nodeid is used
|
||||
* Bugfix [cs3org/reva#4758](https://github.com/cs3org/reva/pull/4758): Fix moving locked files, enable handling locked files via ocdav
|
||||
* Bugfix [cs3org/reva#4774](https://github.com/cs3org/reva/pull/4774): Fix micro ocdav service init and registration
|
||||
* Bugfix [cs3org/reva#4776](https://github.com/cs3org/reva/pull/4776): Fix response code for DEL file that in postprocessing
|
||||
* Bugfix [cs3org/reva#4746](https://github.com/cs3org/reva/pull/4746): Uploading the same file multiple times leads to orphaned blobs
|
||||
* Bugfix [cs3org/reva#4778](https://github.com/cs3org/reva/pull/4778): Zero byte uploads
|
||||
* Change [cs3org/reva#4759](https://github.com/cs3org/reva/pull/4759): Updated to the latest version of the go-cs3apis
|
||||
* Change [cs3org/reva#4773](https://github.com/cs3org/reva/pull/4773): Ocis bumped
|
||||
* Enhancement [cs3org/reva#4766](https://github.com/cs3org/reva/pull/4766): Set archiver output format via query parameter
|
||||
* Enhancement [cs3org/reva#4763](https://github.com/cs3org/reva/pull/4763): Improve posixfs storage driver
|
||||
|
||||
https://github.com/owncloud/ocis/pull/9690
|
||||
https://github.com/owncloud/ocis/pull/9662
|
||||
|
||||
@@ -15,7 +15,7 @@ require (
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible
|
||||
github.com/coreos/go-oidc/v3 v3.10.0
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb
|
||||
github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6
|
||||
github.com/cs3org/reva/v2 v2.22.0
|
||||
github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25
|
||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
|
||||
github.com/egirna/icap-client v0.1.1
|
||||
@@ -288,7 +288,7 @@ require (
|
||||
github.com/opencontainers/runtime-spec v1.1.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
|
||||
github.com/pablodz/inotifywaitgo v0.0.6 // indirect
|
||||
github.com/pablodz/inotifywaitgo v0.0.7 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
|
||||
@@ -1025,8 +1025,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c=
|
||||
github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME=
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb h1:KmYZDReplv/yfwc1LNYpDcVhVujC3Pasv6WjXx1haSU=
|
||||
github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE=
|
||||
github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6 h1:kGRQf90rhI4L2vU+RxPIP8LHmCkGhESV3VpML3Owkh0=
|
||||
github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6/go.mod h1:vz5FHfuRaLvSO6bscoWWx+29RtyJm0wIWLZciCLN3L0=
|
||||
github.com/cs3org/reva/v2 v2.22.0 h1:dMZGJDdrTCYYyMlVSzchbEUyFrHgvFI2Md/aLLiez54=
|
||||
github.com/cs3org/reva/v2 v2.22.0/go.mod h1:y9ujkcxepugOsv50baQSCtudfY5VY5S5eigNTvGfSZI=
|
||||
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
|
||||
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
|
||||
@@ -1808,8 +1808,8 @@ github.com/owncloud/libre-graph-api-go v1.0.5-0.20240618162722-2298241331d1 h1:w
|
||||
github.com/owncloud/libre-graph-api-go v1.0.5-0.20240618162722-2298241331d1/go.mod h1:yXI+rmE8yYx+ZsGVrnCpprw/gZMcxjwntnX2y2+VKxY=
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
|
||||
github.com/pablodz/inotifywaitgo v0.0.6 h1:BTjQfnixXwG7oYmlIiyhWA6iyO9BtxatB3YgiibOTFc=
|
||||
github.com/pablodz/inotifywaitgo v0.0.6/go.mod h1:OtzRCsYTJlIr+vAzlOtauTkfQ1c25ebFuXq8tbbf8cw=
|
||||
github.com/pablodz/inotifywaitgo v0.0.7 h1:1ii49dGBnRn0t1Sz7RGZS6/NberPEDQprwKHN49Bv6U=
|
||||
github.com/pablodz/inotifywaitgo v0.0.7/go.mod h1:OtzRCsYTJlIr+vAzlOtauTkfQ1c25ebFuXq8tbbf8cw=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
|
||||
@@ -229,16 +229,16 @@ The expected failures in this file are from features in the owncloud/ocis repo.
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1221](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1221)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1222)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1374](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1374)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1633](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1633)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1632](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1632)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1686](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1686)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1687](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1687)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1688](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1688)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1689](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1689)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1757](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1757)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1823](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1823)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1945](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1945)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2072](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2072)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2199)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2327](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2327)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1756](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1756)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1822](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1822)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:1944](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1944)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2071](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2071)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2198](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2198)
|
||||
- [apiSharingNgLinkSharePermission/createLinkShare.feature:2326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2326)
|
||||
- [apiSharingNgLinkSharePermission/updateLinkShare.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L73)
|
||||
- [apiSharingNgLinkSharePermission/updateLinkShare.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L75)
|
||||
- [apiSharingNgLinkSharePermission/updateLinkShare.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L76)
|
||||
|
||||
@@ -1414,7 +1414,6 @@ Feature: Create a link share for a resource
|
||||
Examples:
|
||||
| drive | message |
|
||||
| Personal | cannot create link on personal space root |
|
||||
| Shares | no share permission |
|
||||
|
||||
|
||||
Scenario Outline: try to create an internal link share with password of a Personal and Shares drives using permissions endpoint
|
||||
|
||||
Generated
Vendored
+32
-21
@@ -53,7 +53,7 @@ import (
|
||||
const getUserCtxErrMsg = "error getting user from context"
|
||||
|
||||
func init() {
|
||||
rgrpc.Register("publicshareprovider", New)
|
||||
rgrpc.Register("publicshareprovider", NewDefault)
|
||||
}
|
||||
|
||||
type config struct {
|
||||
@@ -127,9 +127,8 @@ func parsePasswordPolicy(m map[string]interface{}) (*passwordPolicy, error) {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// New creates a new user share provider svc
|
||||
func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
|
||||
// New creates a new public share provider svc initialized from defaults
|
||||
func NewDefault(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
c, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -146,6 +145,15 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(gatewaySelector, sm, c, p)
|
||||
}
|
||||
|
||||
// New creates a new user share provider svc
|
||||
func New(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], sm publicshare.Manager, c *config, p *passwordPolicy) (rgrpc.Service, error) {
|
||||
allowedPathsForShares := make([]*regexp.Regexp, 0, len(c.AllowedPathsForShares))
|
||||
for _, s := range c.AllowedPathsForShares {
|
||||
regex, err := regexp.Compile(s)
|
||||
@@ -155,11 +163,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
|
||||
allowedPathsForShares = append(allowedPathsForShares, regex)
|
||||
}
|
||||
|
||||
gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
service := &service{
|
||||
conf: c,
|
||||
sm: sm,
|
||||
@@ -233,7 +236,7 @@ func (s *service) CreatePublicShare(ctx context.Context, req *link.CreatePublicS
|
||||
}
|
||||
|
||||
// check that user has share permissions
|
||||
if !sRes.GetInfo().GetPermissionSet().AddGrant {
|
||||
if !isInternalLink && !sRes.GetInfo().GetPermissionSet().AddGrant {
|
||||
return &link.CreatePublicShareResponse{
|
||||
Status: status.NewInvalidArg(ctx, "no share permission"),
|
||||
}, nil
|
||||
@@ -477,14 +480,12 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
|
||||
|
||||
isInternalLink := isInternalLink(req, ps)
|
||||
|
||||
// users should always be able to downgrade links to internal links
|
||||
// when they are the creator of the link
|
||||
// all other users should have the WritePublicLink permission
|
||||
if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) {
|
||||
// check if the user has the permission in the user role
|
||||
if !publicshare.IsCreatedByUser(ps, user) {
|
||||
canWriteLink, err := utils.CheckPermission(ctx, permission.WritePublicLink, gatewayClient)
|
||||
if err != nil {
|
||||
return &link.UpdatePublicShareResponse{
|
||||
Status: status.NewInternal(ctx, "error loading public share"),
|
||||
Status: status.NewInternal(ctx, "error checking permission to write public share"),
|
||||
}, err
|
||||
}
|
||||
if !canWriteLink {
|
||||
@@ -501,8 +502,14 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
|
||||
Status: status.NewInternal(ctx, "failed to stat shared resource"),
|
||||
}, err
|
||||
}
|
||||
if sRes.Status.Code != rpc.Code_CODE_OK {
|
||||
return &link.UpdatePublicShareResponse{
|
||||
Status: sRes.GetStatus(),
|
||||
}, nil
|
||||
|
||||
if !publicshare.IsCreatedByUser(ps, user) {
|
||||
}
|
||||
|
||||
if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) {
|
||||
if !sRes.GetInfo().GetPermissionSet().UpdateGrant {
|
||||
return &link.UpdatePublicShareResponse{
|
||||
Status: status.NewPermissionDenied(ctx, nil, "no permission to update public share"),
|
||||
@@ -547,12 +554,16 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS
|
||||
}
|
||||
|
||||
// enforce password if needed
|
||||
canOptOut, err := utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient)
|
||||
if err != nil {
|
||||
return &link.UpdatePublicShareResponse{
|
||||
Status: status.NewInternal(ctx, err.Error()),
|
||||
}, nil
|
||||
var canOptOut bool
|
||||
if !isInternalLink {
|
||||
canOptOut, err = utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient)
|
||||
if err != nil {
|
||||
return &link.UpdatePublicShareResponse{
|
||||
Status: status.NewInternal(ctx, err.Error()),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
updatePassword := req.GetUpdate().GetType() == link.UpdatePublicShareRequest_Update_TYPE_PASSWORD
|
||||
setPassword := grant.GetPassword()
|
||||
|
||||
|
||||
+204
-98
@@ -46,52 +46,89 @@ import (
|
||||
type ScanDebouncer struct {
|
||||
after time.Duration
|
||||
f func(item scanItem)
|
||||
pending map[string]*time.Timer
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type EventAction int
|
||||
|
||||
const (
|
||||
ActionCreate EventAction = iota
|
||||
ActionUpdate
|
||||
ActionMove
|
||||
ActionDelete
|
||||
)
|
||||
|
||||
type queueItem struct {
|
||||
item scanItem
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
// NewScanDebouncer returns a new SpaceDebouncer instance
|
||||
func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer {
|
||||
return &ScanDebouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: map[string]*time.Timer{},
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restars the debounce timer for the given space
|
||||
// Debounce restarts the debounce timer for the given space
|
||||
func (d *ScanDebouncer) Debounce(item scanItem) {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
path := item.Path
|
||||
force := item.ForceRescan
|
||||
if t := d.pending[item.Path]; t != nil {
|
||||
force = force || item.ForceRescan
|
||||
t.Stop()
|
||||
recurse := item.Recurse
|
||||
if i, ok := d.pending.Load(item.Path); ok {
|
||||
queueItem := i.(*queueItem)
|
||||
force = force || queueItem.item.ForceRescan
|
||||
recurse = recurse || queueItem.item.Recurse
|
||||
queueItem.timer.Stop()
|
||||
}
|
||||
|
||||
d.pending[item.Path] = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(path); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
d.pending[path].Reset(d.after)
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
d.pending.Store(item.Path, &queueItem{
|
||||
item: item,
|
||||
timer: time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(path); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(path); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.inProgress.Store(path, true)
|
||||
defer d.inProgress.Delete(path)
|
||||
d.f(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: force,
|
||||
})
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(path)
|
||||
d.inProgress.Store(path, true)
|
||||
defer d.inProgress.Delete(path)
|
||||
d.f(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: force,
|
||||
Recurse: recurse,
|
||||
})
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
// InProgress returns true if the given path is currently being processed
|
||||
func (d *ScanDebouncer) InProgress(path string) bool {
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
if _, ok := d.pending.Load(path); ok {
|
||||
return true
|
||||
}
|
||||
|
||||
_, ok := d.inProgress.Load(path)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (t *Tree) workScanQueue() {
|
||||
for i := 0; i < t.options.MaxConcurrency; i++ {
|
||||
go func() {
|
||||
@@ -103,17 +140,73 @@ func (t *Tree) workScanQueue() {
|
||||
log.Error().Err(err).Str("path", item.Path).Msg("failed to assimilate item")
|
||||
continue
|
||||
}
|
||||
|
||||
if item.Recurse {
|
||||
err = t.WarmupIDCache(item.Path, true)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("path", item.Path).Msg("failed to warmup id cache")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Scan scans the given path and updates the id chache
|
||||
func (t *Tree) Scan(path string, forceRescan bool) error {
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: forceRescan,
|
||||
})
|
||||
func (t *Tree) Scan(path string, action EventAction, isDir bool, recurse bool) error {
|
||||
// cases:
|
||||
switch action {
|
||||
case ActionCreate:
|
||||
if !isDir {
|
||||
// 1. New file (could be emitted as part of a new directory)
|
||||
// -> assimilate file
|
||||
// -> scan parent directory recursively
|
||||
if !t.scanDebouncer.InProgress(filepath.Dir(path)) {
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: false,
|
||||
})
|
||||
}
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: filepath.Dir(path),
|
||||
ForceRescan: true,
|
||||
Recurse: true,
|
||||
})
|
||||
} else {
|
||||
// 2. New directory
|
||||
// -> scan directory
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: true,
|
||||
Recurse: true,
|
||||
})
|
||||
}
|
||||
|
||||
case ActionUpdate:
|
||||
// 3. Updated file
|
||||
// -> update file unless parent directory is being rescanned
|
||||
if !t.scanDebouncer.InProgress(filepath.Dir(path)) {
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: true,
|
||||
})
|
||||
}
|
||||
|
||||
case ActionMove:
|
||||
// 4. Moved file
|
||||
// -> update file
|
||||
// 5. Moved directory
|
||||
// -> update directory and all children
|
||||
t.scanDebouncer.Debounce(scanItem{
|
||||
Path: path,
|
||||
ForceRescan: isDir,
|
||||
Recurse: isDir,
|
||||
})
|
||||
|
||||
case ActionDelete:
|
||||
_ = t.HandleFileDelete(path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -183,44 +276,43 @@ func (t *Tree) getOwnerAndIDs(path string) (*userv1beta1.UserId, string, string,
|
||||
return owner, nodeID, spaceID, parentID, nil
|
||||
}
|
||||
|
||||
func (t *Tree) assimilate(item scanItem) error {
|
||||
var err error
|
||||
func (t *Tree) findSpaceId(path string) (string, node.Attributes, error) {
|
||||
// find the space id, scope by the according user
|
||||
spaceID := []byte("")
|
||||
spaceCandidate := item.Path
|
||||
spaceCandidate := path
|
||||
spaceAttrs := node.Attributes{}
|
||||
for strings.HasPrefix(spaceCandidate, t.options.Root) {
|
||||
spaceAttrs, err = t.lookup.MetadataBackend().All(context.Background(), spaceCandidate)
|
||||
if err == nil && len(spaceAttrs[prefixes.SpaceIDAttr]) > 0 {
|
||||
spaceID = spaceAttrs[prefixes.SpaceIDAttr]
|
||||
spaceAttrs, err := t.lookup.MetadataBackend().All(context.Background(), spaceCandidate)
|
||||
spaceID := spaceAttrs[prefixes.SpaceIDAttr]
|
||||
if err == nil && len(spaceID) > 0 {
|
||||
if t.options.UseSpaceGroups {
|
||||
// set the uid and gid for the space
|
||||
fi, err := os.Stat(spaceCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", spaceAttrs, err
|
||||
}
|
||||
sys := fi.Sys().(*syscall.Stat_t)
|
||||
gid := int(sys.Gid)
|
||||
_, err = t.userMapper.ScopeUserByIds(-1, gid)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", spaceAttrs, err
|
||||
}
|
||||
}
|
||||
break
|
||||
|
||||
return string(spaceID), spaceAttrs, nil
|
||||
}
|
||||
spaceCandidate = filepath.Dir(spaceCandidate)
|
||||
}
|
||||
if len(spaceID) == 0 {
|
||||
return fmt.Errorf("did not find space id for path")
|
||||
}
|
||||
return "", spaceAttrs, fmt.Errorf("could not find space for path %s", path)
|
||||
}
|
||||
|
||||
func (t *Tree) assimilate(item scanItem) error {
|
||||
var id []byte
|
||||
if !item.ForceRescan {
|
||||
// already assimilated?
|
||||
id, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr)
|
||||
if err == nil {
|
||||
return t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path)
|
||||
}
|
||||
var err error
|
||||
|
||||
// First find the space id
|
||||
spaceID, spaceAttrs, err := t.findSpaceId(item.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// lock the file for assimilation
|
||||
@@ -240,64 +332,62 @@ func (t *Tree) assimilate(item scanItem) error {
|
||||
// check for the id attribute again after grabbing the lock, maybe the file was assimilated/created by us in the meantime
|
||||
id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr)
|
||||
if err == nil {
|
||||
previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), string(spaceID), string(id))
|
||||
previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), spaceID, string(id))
|
||||
|
||||
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path)
|
||||
if item.ForceRescan {
|
||||
previousParentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
|
||||
if err != nil {
|
||||
return err
|
||||
// This item had already been assimilated in the past. Update the path
|
||||
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path)
|
||||
|
||||
previousParentID, _ := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
|
||||
|
||||
fi, err := t.updateFile(item.Path, string(id), spaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// was it moved?
|
||||
if ok && len(previousParentID) > 0 && previousPath != item.Path {
|
||||
// purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair.
|
||||
_ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath)
|
||||
_ = t.lookup.MetadataBackend().Purge(previousPath)
|
||||
|
||||
if fi.IsDir() {
|
||||
// if it was moved and it is a directory we need to propagate the move
|
||||
go func() { _ = t.WarmupIDCache(item.Path, false) }()
|
||||
}
|
||||
|
||||
fi, err := t.updateFile(item.Path, string(id), string(spaceID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// was it moved?
|
||||
if ok && previousPath != item.Path {
|
||||
// purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair.
|
||||
_ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath)
|
||||
_ = t.lookup.MetadataBackend().Purge(previousPath)
|
||||
|
||||
if fi.IsDir() {
|
||||
// if it was moved and it is a directory we need to propagate the move
|
||||
go func() { _ = t.WarmupIDCache(item.Path, false) }()
|
||||
parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
|
||||
if err == nil && len(parentID) > 0 {
|
||||
ref := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: t.options.MountID,
|
||||
SpaceId: spaceID,
|
||||
OpaqueId: string(parentID),
|
||||
},
|
||||
Path: filepath.Base(item.Path),
|
||||
}
|
||||
|
||||
parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr)
|
||||
if err == nil && len(parentID) > 0 {
|
||||
ref := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: t.options.MountID,
|
||||
SpaceId: string(spaceID),
|
||||
OpaqueId: string(parentID),
|
||||
},
|
||||
Path: filepath.Base(item.Path),
|
||||
}
|
||||
oldRef := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: t.options.MountID,
|
||||
SpaceId: string(spaceID),
|
||||
OpaqueId: string(previousParentID),
|
||||
},
|
||||
Path: filepath.Base(previousPath),
|
||||
}
|
||||
t.PublishEvent(events.ItemMoved{
|
||||
SpaceOwner: user,
|
||||
Executant: user,
|
||||
Owner: user,
|
||||
Ref: ref,
|
||||
OldReference: oldRef,
|
||||
Timestamp: utils.TSNow(),
|
||||
})
|
||||
oldRef := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: t.options.MountID,
|
||||
SpaceId: spaceID,
|
||||
OpaqueId: string(previousParentID),
|
||||
},
|
||||
Path: filepath.Base(previousPath),
|
||||
}
|
||||
t.PublishEvent(events.ItemMoved{
|
||||
SpaceOwner: user,
|
||||
Executant: user,
|
||||
Owner: user,
|
||||
Ref: ref,
|
||||
OldReference: oldRef,
|
||||
Timestamp: utils.TSNow(),
|
||||
})
|
||||
}
|
||||
// }
|
||||
}
|
||||
} else {
|
||||
// assimilate new file
|
||||
newId := uuid.New().String()
|
||||
fi, err := t.updateFile(item.Path, newId, string(spaceID))
|
||||
fi, err := t.updateFile(item.Path, newId, spaceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -305,7 +395,7 @@ func (t *Tree) assimilate(item scanItem) error {
|
||||
ref := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: t.options.MountID,
|
||||
SpaceId: string(spaceID),
|
||||
SpaceId: spaceID,
|
||||
OpaqueId: newId,
|
||||
},
|
||||
}
|
||||
@@ -455,17 +545,25 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
sizes := make(map[string]int64)
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") {
|
||||
// skip lock files
|
||||
if isLockFile(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculate tree sizes
|
||||
if !info.IsDir() {
|
||||
dir := filepath.Dir(path)
|
||||
sizes[dir] += info.Size()
|
||||
}
|
||||
|
||||
attribs, err := t.lookup.MetadataBackend().All(context.Background(), path)
|
||||
if err == nil {
|
||||
if err == nil && len(attribs[prefixes.IDAttr]) > 0 {
|
||||
nodeSpaceID := attribs[prefixes.SpaceIDAttr]
|
||||
if len(nodeSpaceID) > 0 {
|
||||
spaceID = nodeSpaceID
|
||||
@@ -496,10 +594,18 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error {
|
||||
id, ok := attribs[prefixes.IDAttr]
|
||||
if ok {
|
||||
_ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), path)
|
||||
} else if assimilate {
|
||||
_ = t.Scan(path, false)
|
||||
}
|
||||
} else if assimilate {
|
||||
_ = t.assimilate(scanItem{Path: path, ForceRescan: true})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for dir, size := range sizes {
|
||||
_ = t.lookup.MetadataBackend().Set(context.Background(), dir, prefixes.TreesizeAttr, []byte(fmt.Sprintf("%d", size)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Generated
Vendored
+3
-3
@@ -61,15 +61,15 @@ start:
|
||||
}
|
||||
switch ev.Event {
|
||||
case "CREATE":
|
||||
go func() { _ = w.tree.Scan(ev.Path, false) }()
|
||||
go func() { _ = w.tree.Scan(ev.Path, ActionCreate, false, false) }()
|
||||
case "CLOSE":
|
||||
bytesWritten, err := strconv.Atoi(ev.BytesWritten)
|
||||
if err == nil && bytesWritten > 0 {
|
||||
go func() { _ = w.tree.Scan(ev.Path, true) }()
|
||||
go func() { _ = w.tree.Scan(ev.Path, ActionUpdate, false, true) }()
|
||||
}
|
||||
case "RENAME":
|
||||
go func() {
|
||||
_ = w.tree.Scan(ev.Path, true)
|
||||
_ = w.tree.Scan(ev.Path, ActionMove, false, true)
|
||||
_ = w.tree.WarmupIDCache(ev.Path, false)
|
||||
}()
|
||||
}
|
||||
|
||||
Generated
Vendored
+4
-4
@@ -41,21 +41,21 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasSuffix(lwev.Path, ".flock") || strings.HasSuffix(lwev.Path, ".mlock") {
|
||||
if isLockFile(lwev.Path) {
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(lwev.Event, "IN_CREATE"):
|
||||
go func() { _ = w.tree.Scan(lwev.Path, false) }()
|
||||
go func() { _ = w.tree.Scan(lwev.Path, ActionCreate, false, false) }()
|
||||
case strings.Contains(lwev.Event, "IN_CLOSE_WRITE"):
|
||||
bytesWritten, err := strconv.Atoi(lwev.BytesWritten)
|
||||
if err == nil && bytesWritten > 0 {
|
||||
go func() { _ = w.tree.Scan(lwev.Path, true) }()
|
||||
go func() { _ = w.tree.Scan(lwev.Path, ActionUpdate, false, true) }()
|
||||
}
|
||||
case strings.Contains(lwev.Event, "IN_MOVED_TO"):
|
||||
go func() {
|
||||
_ = w.tree.Scan(lwev.Path, true)
|
||||
_ = w.tree.Scan(lwev.Path, ActionMove, false, true)
|
||||
_ = w.tree.WarmupIDCache(lwev.Path, false)
|
||||
}()
|
||||
}
|
||||
|
||||
+4
-5
@@ -2,7 +2,6 @@ package tree
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pablodz/inotifywaitgo/inotifywaitgo"
|
||||
)
|
||||
@@ -43,20 +42,20 @@ func (iw *InotifyWatcher) Watch(path string) {
|
||||
select {
|
||||
case event := <-events:
|
||||
for _, e := range event.Events {
|
||||
if strings.HasSuffix(event.Filename, ".flock") || strings.HasSuffix(event.Filename, ".mlock") {
|
||||
if isLockFile(event.Filename) {
|
||||
continue
|
||||
}
|
||||
switch e {
|
||||
case inotifywaitgo.DELETE:
|
||||
go func() { _ = iw.tree.HandleFileDelete(event.Filename) }()
|
||||
case inotifywaitgo.CREATE:
|
||||
go func() { _ = iw.tree.Scan(event.Filename, false) }()
|
||||
go func() { _ = iw.tree.Scan(event.Filename, ActionCreate, event.IsDir, false) }()
|
||||
case inotifywaitgo.MOVED_TO:
|
||||
go func() {
|
||||
_ = iw.tree.Scan(event.Filename, true)
|
||||
_ = iw.tree.Scan(event.Filename, ActionMove, event.IsDir, true)
|
||||
}()
|
||||
case inotifywaitgo.CLOSE_WRITE:
|
||||
go func() { _ = iw.tree.Scan(event.Filename, true) }()
|
||||
go func() { _ = iw.tree.Scan(event.Filename, ActionUpdate, event.IsDir, true) }()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-1
@@ -75,6 +75,7 @@ type Watcher interface {
|
||||
type scanItem struct {
|
||||
Path string
|
||||
ForceRescan bool
|
||||
Recurse bool
|
||||
}
|
||||
|
||||
// Tree manages a hierarchical tree
|
||||
@@ -110,7 +111,7 @@ func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Opti
|
||||
idCache: cache,
|
||||
propagator: propagator.New(lu, &o.Options),
|
||||
scanQueue: scanQueue,
|
||||
scanDebouncer: NewScanDebouncer(500*time.Millisecond, func(item scanItem) {
|
||||
scanDebouncer: NewScanDebouncer(1000*time.Millisecond, func(item scanItem) {
|
||||
scanQueue <- item
|
||||
}),
|
||||
es: es,
|
||||
@@ -393,6 +394,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
|
||||
g.Go(func() error {
|
||||
defer close(work)
|
||||
for _, name := range names {
|
||||
if isLockFile(name) {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case work <- name:
|
||||
case <-ctx.Done():
|
||||
@@ -877,3 +882,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) (
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func isLockFile(path string) bool {
|
||||
return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock")
|
||||
}
|
||||
|
||||
+4
@@ -59,6 +59,9 @@ const (
|
||||
EventDeleteSelf = "delete_self"
|
||||
// The filesystem on which a watched file or directory resides was unmounted. After this event the file or directory is no longer being watched. Note that this event can occur even if it is not explicitly being listened to.
|
||||
EventUnmount = "unmount"
|
||||
|
||||
// The subject of this event is a directory
|
||||
FlagIsdir = "ISDIR"
|
||||
)
|
||||
|
||||
type EVENT int
|
||||
@@ -84,6 +87,7 @@ const (
|
||||
type FileEvent struct {
|
||||
Filename string
|
||||
Events []EVENT
|
||||
IsDir bool
|
||||
}
|
||||
|
||||
var EVENT_MAP = map[int]string{
|
||||
|
||||
+7
@@ -75,8 +75,14 @@ func WatchPath(s *Settings) {
|
||||
}
|
||||
}
|
||||
var eventsEvents []EVENT
|
||||
isDir := false
|
||||
|
||||
for _, eventStr := range eventsStr {
|
||||
if eventStr == FlagIsdir {
|
||||
isDir = true
|
||||
continue
|
||||
}
|
||||
|
||||
eventStr = strings.ToLower(eventStr)
|
||||
event, ok := EVENT_MAP_REVERSE[eventStr]
|
||||
if !ok {
|
||||
@@ -89,6 +95,7 @@ func WatchPath(s *Settings) {
|
||||
event := FileEvent{
|
||||
Filename: prefix + file,
|
||||
Events: eventsEvents,
|
||||
IsDir: isDir,
|
||||
}
|
||||
|
||||
// Send the file name to the channel
|
||||
|
||||
Vendored
+2
-2
@@ -366,7 +366,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
|
||||
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
|
||||
# github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6
|
||||
# github.com/cs3org/reva/v2 v2.22.0
|
||||
## explicit; go 1.21
|
||||
github.com/cs3org/reva/v2/cmd/revad/internal/grace
|
||||
github.com/cs3org/reva/v2/cmd/revad/runtime
|
||||
@@ -1626,7 +1626,7 @@ github.com/owncloud/libre-graph-api-go
|
||||
# github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
|
||||
## explicit; go 1.12
|
||||
github.com/oxtoacart/bpool
|
||||
# github.com/pablodz/inotifywaitgo v0.0.6
|
||||
# github.com/pablodz/inotifywaitgo v0.0.7
|
||||
## explicit; go 1.21
|
||||
github.com/pablodz/inotifywaitgo/inotifywaitgo
|
||||
# github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
|
||||
Reference in New Issue
Block a user