[full-ci] chore: bump reva to v2.45.0 (#2787)

This commit is contained in:
Viktor Scharf
2026-05-20 10:17:25 +02:00
committed by GitHub
parent 56a65e68b9
commit a8885a3d4d
14 changed files with 323 additions and 68 deletions
+1 -1
View File
@@ -64,7 +64,7 @@ require (
github.com/open-policy-agent/opa v1.15.2 github.com/open-policy-agent/opa v1.15.2
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 github.com/opencloud-eu/reva/v2 v2.45.0
github.com/opensearch-project/opensearch-go/v4 v4.6.0 github.com/opensearch-project/opensearch-go/v4 v4.6.0
github.com/orcaman/concurrent-map v1.0.0 github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
+2 -2
View File
@@ -952,8 +952,8 @@ github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9 h1:dIft
github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI= github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 h1:PH9Ia0HwdvpfaThYCid2atc5y+uwn4EJxvJU6L9wU6M= github.com/opencloud-eu/reva/v2 v2.45.0 h1:62XctwzrGKjmWng9QI+tACEDQb6R9fG7oHlMPfQkjNs=
github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM= github.com/opencloud-eu/reva/v2 v2.45.0/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -24,9 +24,9 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream" "github.com/nats-io/nats.go/jetstream"
"github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
) )
type IDCache struct { type IDCache struct {
@@ -59,17 +59,20 @@ func (c *IDCache) Delete(ctx context.Context, spaceID, nodeID string) error {
func (c *IDCache) DeleteByPath(ctx context.Context, path string) error { func (c *IDCache) DeleteByPath(ctx context.Context, path string) error {
baseKey := reverseCacheKey(path) baseKey := reverseCacheKey(path)
spaceID, nodeID, ok := c.GetByPath(ctx, path) spaceID, nodeID, err := c.GetByPath(ctx, path)
if !ok { if err != nil {
appctx.GetLogger(ctx).Error().Str("record", path).Msg("could not get spaceID and nodeID from cache") if _, ok := err.(errtypes.NotFound); !ok {
return err
}
appctx.GetLogger(ctx).Error().Err(err).Str("record", path).Msg("could not get spaceID and nodeID from cache")
} else { } else {
err := c.kv.Purge(ctx, baseKey) err := c.kv.Purge(ctx, baseKey)
if err != nil && err != nats.ErrKeyNotFound { if err != nil && err != jetstream.ErrKeyNotFound {
appctx.GetLogger(ctx).Error().Err(err).Str("record", baseKey).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") appctx.GetLogger(ctx).Error().Err(err).Str("record", baseKey).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
} }
err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID)) err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID))
if err != nil && err != nats.ErrKeyNotFound { if err != nil && err != jetstream.ErrKeyNotFound {
appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
} }
} }
@@ -85,19 +88,19 @@ func (c *IDCache) DeleteByPath(ctx context.Context, path string) error {
break break
} }
key := update.Key() key := update.Key()
spaceID, nodeID, ok := c.getByReverseCacheKey(ctx, key) spaceID, nodeID, err := c.getByReverseCacheKey(ctx, key)
if !ok { if err != nil {
appctx.GetLogger(ctx).Error().Str("record", key).Msg("could not get spaceID and nodeID from cache") appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Msg("could not get spaceID and nodeID from cache")
continue continue
} }
err := c.kv.Purge(ctx, key) err = c.kv.Purge(ctx, key)
if err != nil && err != nats.ErrKeyNotFound { if err != nil && err != jetstream.ErrKeyNotFound {
appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
} }
err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID)) err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID))
if err != nil && err != nats.ErrKeyNotFound { if err != nil && err != jetstream.ErrKeyNotFound {
appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache")
} }
} }
@@ -121,32 +124,38 @@ func (c *IDCache) Set(ctx context.Context, spaceID, nodeID, val string) error {
} }
// Get returns the value for a given key // Get returns the value for a given key
func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, bool) { func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, error) {
record, err := c.kv.Get(ctx, cacheKey(spaceID, nodeID)) record, err := c.kv.Get(ctx, cacheKey(spaceID, nodeID))
if err != nil { if err != nil {
return "", false if err == jetstream.ErrKeyNotFound {
return "", errtypes.NotFound("record not found in cache")
}
return "", err
} }
return string(record.Value()), true return string(record.Value()), nil
} }
func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, bool) { func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, error) {
record, err := c.kv.Get(ctx, reverseKey) record, err := c.kv.Get(ctx, reverseKey)
if err != nil { if err != nil {
return "", "", false if err == jetstream.ErrKeyNotFound {
return "", "", errtypes.NotFound("record not found in cache")
}
return "", "", err
} }
decoded, err := base32.StdEncoding.DecodeString(string(record.Value())) decoded, err := base32.StdEncoding.DecodeString(string(record.Value()))
if err != nil { if err != nil {
return "", "", false return "", "", err
} }
parts := strings.SplitN(string(decoded), "!", 2) parts := strings.SplitN(string(decoded), "!", 2)
if len(parts) != 2 { if len(parts) != 2 {
return "", "", false return "", "", errtypes.InternalError("invalid cache record")
} }
return parts[0], parts[1], true return parts[0], parts[1], nil
} }
// GetByPath returns the key for a given value // GetByPath returns the key for a given value
func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, bool) { func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, error) {
return c.getByReverseCacheKey(ctx, reverseCacheKey(path)) return c.getByReverseCacheKey(ctx, reverseCacheKey(path))
} }
+32 -18
View File
@@ -41,6 +41,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rogpeppe/go-internal/lockedfile" "github.com/rogpeppe/go-internal/lockedfile"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel" "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace"
) )
@@ -58,8 +59,8 @@ func init() {
// IDCache is a cache for node ids // IDCache is a cache for node ids
type IDCache interface { type IDCache interface {
Get(ctx context.Context, spaceID, nodeID string) (string, bool) Get(ctx context.Context, spaceID, nodeID string) (string, error)
GetByPath(ctx context.Context, path string) (string, string, bool) GetByPath(ctx context.Context, path string) (string, string, error)
Set(ctx context.Context, spaceID, nodeID, val string) error Set(ctx context.Context, spaceID, nodeID, val string) error
@@ -79,10 +80,12 @@ type Lookup struct {
metadataBackend metadata.Backend metadataBackend metadata.Backend
userMapper usermapper.Mapper userMapper usermapper.Mapper
tm node.TimeManager tm node.TimeManager
log *zerolog.Logger
} }
// New returns a new Lookup instance // New returns a new Lookup instance
func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache) (*Lookup, error) { func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache, log *zerolog.Logger) (*Lookup, error) {
spaceRootCache, _ := lru.New[string, string](1000) spaceRootCache, _ := lru.New[string, string](1000)
lu := &Lookup{ lu := &Lookup{
@@ -93,6 +96,7 @@ func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.T
spaceRootCache: spaceRootCache, spaceRootCache: spaceRootCache,
userMapper: um, userMapper: um,
tm: tm, tm: tm,
log: log,
} }
return lu, nil return lu, nil
@@ -107,7 +111,7 @@ func (lu *Lookup) CacheID(ctx context.Context, spaceID, nodeID, val string) erro
} }
// GetCachedID returns the cached path for the given space and node id // GetCachedID returns the cached path for the given space and node id
func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) { func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error) {
if spaceID == nodeID { if spaceID == nodeID {
return lu.getSpaceRootPathWithStatus(ctx, spaceID) return lu.getSpaceRootPathWithStatus(ctx, spaceID)
} }
@@ -116,18 +120,26 @@ func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (stri
func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, error) { func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, error) {
// IDsForPath returns the space and opaque id for the given path // IDsForPath returns the space and opaque id for the given path
spaceID, nodeID, ok := lu.IDCache.GetByPath(ctx, path) spaceID, nodeID, err := lu.IDCache.GetByPath(ctx, path)
if !ok { if err != nil {
return "", "", errtypes.NotFound("path not found in cache:" + path) if _, ok := err.(errtypes.NotFound); !ok {
lu.log.Error().Err(err).Str("path", path).Msg("error looking up path in cache")
}
// fallback to disk
sID, nID, _, _, mErr := lu.metadataBackend.IdentifyPath(ctx, path)
if mErr == nil && nID != "" {
return sID, nID, nil
}
return "", "", err
} }
return spaceID, nodeID, nil return spaceID, nodeID, nil
} }
// NodeFromPath returns the node for the given path // NodeFromPath returns the node for the given path
func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) {
parentPath, ok := lu.GetCachedID(ctx, parent.SpaceID, parent.ID) parentPath, err := lu.GetCachedID(ctx, parent.SpaceID, parent.ID)
if !ok { if err != nil {
return "", errtypes.NotFound(parent.ID) return "", err
} }
childPath := filepath.Join(parentPath, name) childPath := filepath.Join(parentPath, name)
@@ -290,15 +302,15 @@ func (lu *Lookup) InternalRoot() string {
return lu.Options.Root return lu.Options.Root
} }
func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, bool) { func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, error) {
if val, ok := lu.spaceRootCache.Get(spaceID); ok { if val, ok := lu.spaceRootCache.Get(spaceID); ok {
return val, true return val, nil
} }
val, ok := lu.IDCache.Get(ctx, spaceID, spaceID) val, err := lu.IDCache.Get(ctx, spaceID, spaceID)
if ok { if err == nil {
lu.spaceRootCache.Add(spaceID, val) lu.spaceRootCache.Add(spaceID, val)
} }
return val, ok return val, err
} }
func (lu *Lookup) getSpaceRootPath(ctx context.Context, spaceID string) string { func (lu *Lookup) getSpaceRootPath(ctx context.Context, spaceID string) string {
@@ -338,9 +350,11 @@ func (lu *Lookup) LockfilePaths(n *node.Node) []string {
} }
paths := []string{filepath.Join(spaceRoot, MetadataDir, Pathify(n.ID, 4, 2)+".lock")} paths := []string{filepath.Join(spaceRoot, MetadataDir, Pathify(n.ID, 4, 2)+".lock")}
nodepath := n.InternalPath() if lu.Options.WatchFS {
if len(nodepath) > 0 { nodepath := n.InternalPath()
paths = append(paths, nodepath+".lock") if len(nodepath) > 0 {
paths = append(paths, nodepath+".lock")
}
} }
return paths return paths
+2 -2
View File
@@ -122,7 +122,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache.
var lu *lookup.Lookup var lu *lookup.Lookup
switch o.MetadataBackend { switch o.MetadataBackend {
case "xattrs": case "xattrs":
lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache) lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -136,7 +136,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache.
return filepath.Join(spaceRoot, lookup.MetadataDir) return filepath.Join(spaceRoot, lookup.MetadataDir)
}, },
o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache) o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -39,6 +39,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
@@ -453,9 +454,9 @@ func (t *Tree) assimilate(item scanItem) error {
_ = unlock() _ = unlock()
}() }()
previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id) previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id)
if previousPath == "" || !ok { if previousPath == "" || err != nil {
previousPath, ok = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id) previousPath, err = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id)
} }
// compare metadata mtime with actual mtime. if it matches AND the path hasn't changed (move operation) // compare metadata mtime with actual mtime. if it matches AND the path hasn't changed (move operation)
@@ -465,7 +466,7 @@ func (t *Tree) assimilate(item scanItem) error {
} }
// was it moved or copied/restored with a clashing id? // was it moved or copied/restored with a clashing id?
if ok && len(parentID) > 0 && previousPath != item.Path { if err == nil && len(parentID) > 0 && previousPath != item.Path {
_, err := os.Stat(previousPath) _, err := os.Stat(previousPath)
if err == nil { if err == nil {
// this id clashes with an existing item -> clear metadata and re-assimilate // this id clashes with an existing item -> clear metadata and re-assimilate
@@ -942,14 +943,22 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error {
if id != "" { if id != "" {
// Check if the item on the previous path still exists. In this case it might have been a copy with extended attributes -> set new ID // Check if the item on the previous path still exists. In this case it might have been a copy with extended attributes -> set new ID
isCopy := false isCopy := false
previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id) previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id)
if ok && previousPath != path { switch err.(type) {
_, err := os.Stat(previousPath) case errtypes.NotFound:
if err == nil { // previous path not found -> not a copy
// previous path (using the same id) still exists -> this is a copy case nil:
isCopy = true if previousPath != path {
_, err := os.Stat(previousPath)
if err == nil {
// previous path (using the same id) still exists -> this is a copy
isCopy = true
}
} }
default:
return errors.Wrap(err, "failed to get previous path from cache")
} }
if isCopy { if isCopy {
// copy detected -> re-assimilate // copy detected -> re-assimilate
_ = t.assimilate(scanItem{Path: path}) _ = t.assimilate(scanItem{Path: path})
@@ -0,0 +1,45 @@
import re
with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "r") as f:
text = f.read()
new_content = """
Describe("WarmupIDCache", func() {
var (
tree *Tree
tmpDir string
logger zerolog.Logger
)
BeforeEach(func() {
var err error
tmpDir, err = os.MkdirTemp("", "warmupidcache-*")
Expect(err).ToNot(HaveOccurred())
logger = zerolog.Nop()
tree = &Tree{
log: &logger,
options: &options.Options{
Options: decomposedoptions.Options{
Root: tmpDir,
},
},
}
})
AfterEach(func() {
os.RemoveAll(tmpDir)
})
It("returns nil for an empty directory", func() {
err := tree.WarmupIDCache(tmpDir, false, false)
Expect(err).ToNot(HaveOccurred())
})
})
"""
text = re.sub(r'Describe\("WarmupIDCache", func\(\).*$', new_content, text, flags=re.DOTALL)
text += "\n})\n"
with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "w") as f:
f.write(text)
@@ -0,0 +1,10 @@
import re
with open("assimilation_test.go", "r") as f:
text = f.read()
# remove CreateTestStorageSpace which throws error
text = text.replace('_, err = env.CreateTestStorageSpace("personal", nil)\n\t\tExpect(err).ToNot(HaveOccurred())', '')
with open("assimilation_test.go", "w") as f:
f.write(text)
@@ -0,0 +1,128 @@
import sys
content = """
Describe("WarmupIDCache", func() {
var (
tree *Tree
tmpDir string
logger zerolog.Logger
ctx context.Context
)
BeforeEach(func() {
ctx = context.Background()
var err error
tmpDir, err = os.MkdirTemp("", "warmupidcache-*")
Expect(err).ToNot(HaveOccurred())
logger = zerolog.Nop()
o := &options.Options{
Options: decomposedoptions.Options{
Root: tmpDir,
},
}
// We need a backend and caches
c, _ := idcache.NewMemoryIDCache()
historyCache, _ := idcache.NewMemoryIDCache()
um := &usermapper.NullMapper{}
backend := metadata.NewMessagePackBackend(o.FileMetadataCache)
lu, err := lookup.New(backend, um, o, &timemanager.Manager{}, c, historyCache)
Expect(err).ToNot(HaveOccurred())
tree = &Tree{
log: &logger,
options: o,
lookup: lu,
}
})
AfterEach(func() {
os.RemoveAll(tmpDir)
})
It("returns nil for an empty directory", func() {
err := tree.WarmupIDCache(tmpDir, false, false)
Expect(err).ToNot(HaveOccurred())
})
It("picks up new files and directories", func() {
subDir := filepath.Join(tmpDir, "sub")
err := os.Mkdir(subDir, 0755)
Expect(err).ToNot(HaveOccurred())
filePath := filepath.Join(subDir, "test.txt")
err = os.WriteFile(filePath, []byte("hello world"), 0644)
Expect(err).ToNot(HaveOccurred())
// Should not crash, tests basic traverse
err = tree.WarmupIDCache(tmpDir, false, false)
Expect(err).ToNot(HaveOccurred())
})
It("verifies tree sizes and recursion", func() {
// Setup a small directory structure
subDir := filepath.Join(tmpDir, "sub2")
err := os.Mkdir(subDir, 0755)
Expect(err).ToNot(HaveOccurred())
nestedDir := filepath.Join(subDir, "nested")
err = os.Mkdir(nestedDir, 0755)
Expect(err).ToNot(HaveOccurred())
filePath := filepath.Join(nestedDir, "test.txt")
err = os.WriteFile(filePath, []byte("hello world"), 0644) // 11 bytes
Expect(err).ToNot(HaveOccurred())
// Run assimilation
err = tree.WarmupIDCache(tmpDir, true, false)
Expect(err).ToNot(HaveOccurred())
// If assimilation runs, the files will have xattrs and tree sizes evaluated
// wait, but without mocked idResolver for the Tree, it might fail? Let's check!
})
})
"""
with open("assimilation_test.go", "r") as f:
text = f.read()
import re
# Add imports
imports = """
import (
"context"
"os"
"path/filepath"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
decomposedoptions "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
)
"""
# Replace imports
start_import = text.find('import (')
end_import = text.find(')', start_import) + 1
text = text[:start_import] + imports.strip() + text[end_import:]
start_idx = text.find('Describe("WarmupIDCache", func() {')
end_idx = text.find('})\n\n})', start_idx) + 2
if start_idx != -1 and end_idx != -1:
new_text = text[:start_idx] + content.strip() + text[end_idx:]
with open("assimilation_test.go", "w") as f:
f.write(new_text)
@@ -0,0 +1,23 @@
with open("assimilation_test.go", "r") as f:
text = f.read()
import re
# We will inject the xattr check.
insert = """// verify that tree sizes are updated
// Since we used assimilate=true, the treesize xattr on sub2 and nested should be 11.
b, err := env.Lookup.MetadataBackend().Get(env.Ctx, subDir, "user.oc.treesize")
Expect(err).ToNot(HaveOccurred())
Expect(string(b)).To(Equal("11"))
b, err = env.Lookup.MetadataBackend().Get(env.Ctx, nestedDir, "user.oc.treesize")
Expect(err).ToNot(HaveOccurred())
Expect(string(b)).To(Equal("11"))"""
# inject right after env.Tree.WarmupIDCache call
text = text.replace("err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())", "err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())\n\n" + insert)
with open("assimilation_test.go", "w") as f:
f.write(text)
+28 -11
View File
@@ -73,6 +73,10 @@ type Watcher interface {
Watch(path string) Watch(path string)
} }
type IDResolver interface {
IDsForPath(ctx context.Context, path string) (spaceID string, nodeID string, err error)
}
type scanItem struct { type scanItem struct {
Path string Path string
Recurse bool Recurse bool
@@ -80,12 +84,15 @@ type scanItem struct {
// Tree manages a hierarchical tree // Tree manages a hierarchical tree
type Tree struct { type Tree struct {
lookup *lookup.Lookup
blobstore node.Blobstore blobstore node.Blobstore
trashbin *trashbin.Trashbin trashbin *trashbin.Trashbin
propagator propagator.Propagator propagator propagator.Propagator
permissions permissions.Permissions permissions permissions.Permissions
lookup *lookup.Lookup
idResolver IDResolver // points at the lookup but can be overridden for testing
assimilateFunc func(item scanItem) error // function to call to assimilate a node, can be overridden for testing
options *options.Options options *options.Options
personalSpacesRoot string personalSpacesRoot string
projectSpacesRoot string projectSpacesRoot string
@@ -108,9 +115,10 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
scanQueue := make(chan scanItem) scanQueue := make(chan scanItem)
t := &Tree{ t := &Tree{
lookup: lu.(*lookup.Lookup), lookup: lu.(*lookup.Lookup),
blobstore: bs, blobstore: bs,
userMapper: um, userMapper: um,
// idResolver and assimilateFunc are wired below once t exists.
trashbin: trashbin, trashbin: trashbin,
permissions: permissions, permissions: permissions,
options: o, options: o,
@@ -125,6 +133,8 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
personalSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.PersonalSpacePathTemplate))), personalSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.PersonalSpacePathTemplate))),
projectSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.GeneralSpacePathTemplate))), projectSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.GeneralSpacePathTemplate))),
} }
t.idResolver = t.lookup
t.assimilateFunc = t.assimilate
if err := t.checkStorage(); err != nil { if err := t.checkStorage(); err != nil {
return nil, errors.Wrap(err, "tree: unfit storage '"+o.Root+"'") return nil, errors.Wrap(err, "tree: unfit storage '"+o.Root+"'")
} }
@@ -518,16 +528,23 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
for name := range work { for name := range work {
path := filepath.Join(dir, name) path := filepath.Join(dir, name)
_, nodeID, err := t.lookup.IDsForPath(ctx, path) _, nodeID, err := t.idResolver.IDsForPath(ctx, path)
if err != nil { if err != nil {
// we don't know about this node yet for some reason, assimilate it on the fly if _, ok := err.(errtypes.IsNotFound); !ok {
// a NotFound error just means we don't know about this
// node yet. Any other error (e.g. an unavailable id
// cache backend) is a real failure that must not be
// silently turned into an assimilation attempt.
return errors.Wrap(err, "tree: error resolving ids for "+path)
}
// we don't know about this node yet, assimilate it on the fly
t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.") t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.")
err = t.assimilate(scanItem{Path: path}) err = t.assimilateFunc(scanItem{Path: path})
if err != nil { if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node") t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node")
continue continue
} }
_, nodeID, err = t.lookup.IDsForPath(ctx, path) _, nodeID, err = t.idResolver.IDsForPath(ctx, path)
if err != nil || nodeID == "" { if err != nil || nodeID == "" {
t.log.Error().Err(err).Str("path", path).Msg("still could not resolve node after assimilation") t.log.Error().Err(err).Str("path", path).Msg("still could not resolve node after assimilation")
continue continue
@@ -711,9 +728,9 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
idcache := t.lookup.IDCache idcache := t.lookup.IDCache
// create a directory node // create a directory node
parentPath, ok := idcache.Get(ctx, n.SpaceID, n.ParentID) parentPath, err := idcache.Get(ctx, n.SpaceID, n.ParentID)
if !ok { if err != nil {
return errtypes.NotFound(n.ParentID) return err
} }
path := filepath.Join(parentPath, n.Name) path := filepath.Join(parentPath, n.Name)
@@ -1379,7 +1379,7 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, space *provider.
// Warmup posix IDCache if restored path is a directory // Warmup posix IDCache if restored path is a directory
if cachingTree, ok := fs.tp.(IDCachingTree); ok { if cachingTree, ok := fs.tp.(IDCachingTree); ok {
_ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), false, false) _ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), true, false)
} }
} else { } else {
sizeDiff = restoredNode.Blobsize sizeDiff = restoredNode.Blobsize
@@ -173,7 +173,7 @@ type PathLookup interface {
type IDCacher interface { type IDCacher interface {
CacheID(ctx context.Context, spaceID, nodeID, val string) error CacheID(ctx context.Context, spaceID, nodeID, val string) error
GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error)
} }
type BaseNode struct { type BaseNode struct {
+1 -1
View File
@@ -1362,7 +1362,7 @@ github.com/opencloud-eu/icap-client
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d # github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
## explicit; go 1.18 ## explicit; go 1.18
github.com/opencloud-eu/libre-graph-api-go github.com/opencloud-eu/libre-graph-api-go
# github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 # github.com/opencloud-eu/reva/v2 v2.45.0
## explicit; go 1.25.0 ## explicit; go 1.25.0
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
github.com/opencloud-eu/reva/v2/cmd/revad/runtime github.com/opencloud-eu/reva/v2/cmd/revad/runtime