Merge pull request #6904 from aduffeck/rollback-migrations2

[full-ci] Add support for running/rolling back decomposedfs migrations

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d
-	github.com/cs3org/reva/v2 v2.15.1-0.20230726135727-c444e4c5a24f
+	github.com/cs3org/reva/v2 v2.15.1-0.20230731061316-db79e9b61738
 	github.com/disintegration/imaging v1.6.2
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
 	github.com/egirna/icap-client v0.1.1

go.sum

@@ -625,8 +625,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo
 github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4=
 github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc=
 github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA=
-github.com/cs3org/reva/v2 v2.15.1-0.20230726135727-c444e4c5a24f h1:u1otcEA1Otgv7rxS4y/nHU/7zkc1kU/KdCX1YNGSzsE=
+github.com/cs3org/reva/v2 v2.15.1-0.20230731061316-db79e9b61738 h1:EILZCEJMYRla6cktKLpi1c3KwISyoYMGTX0AKCuUTZA=
-github.com/cs3org/reva/v2 v2.15.1-0.20230726135727-c444e4c5a24f/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w=
+github.com/cs3org/reva/v2 v2.15.1-0.20230731061316-db79e9b61738/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

ocis/pkg/command/migrate.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"os"
+	"sort"
 	"sync"
 
 	collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
@@ -10,6 +11,11 @@ import (
 	publicregistry "github.com/cs3org/reva/v2/pkg/publicshare/manager/registry"
 	"github.com/cs3org/reva/v2/pkg/share"
 	"github.com/cs3org/reva/v2/pkg/share/manager/registry"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator"
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
+	tw "github.com/olekukonko/tablewriter"
+	"github.com/rs/zerolog"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/config"
 	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
@@ -28,6 +34,7 @@ func Migrate(cfg *config.Config) *cli.Command {
 		Usage:    "migrate data from an existing to another instance",
 		Category: "migration",
 		Subcommands: []*cli.Command{
+			MigrateDecomposedfs(cfg),
 			MigrateShares(cfg),
 			MigratePublicShares(cfg),
 		},
@@ -38,6 +45,103 @@ func init() {
 	register.AddCommand(Migrate)
 }
 
+// MigrateDecomposedfs is the entrypoint for the decomposedfs migrate command
+func MigrateDecomposedfs(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "decomposedfs",
+		Usage: "run a decomposedfs migration",
+		Subcommands: []*cli.Command{
+			ListDecomposedfsMigrations(cfg),
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "direction",
+				Aliases: []string{"d"},
+				Value:   "migrate",
+				Usage:   "direction of the migration to run ('migrate' or 'rollback')",
+			},
+			&cli.StringFlag{
+				Name:    "migration",
+				Aliases: []string{"m"},
+				Value:   "",
+				Usage:   "ID of the migration to run",
+			},
+			&cli.StringFlag{
+				Name:     "root",
+				Aliases:  []string{"r"},
+				Required: true,
+				Usage:    "Path to the root directory of the decomposedfs",
+			},
+		},
+		Before: func(c *cli.Context) error {
+			// Parse base config
+			if err := parser.ParseConfig(cfg, true); err != nil {
+				return configlog.ReturnError(err)
+			}
+			return nil
+		},
+		Action: func(c *cli.Context) error {
+			log := logger()
+			rootFlag := c.String("root")
+			bod := lookup.DetectBackendOnDisk(rootFlag)
+			backend := backend(rootFlag, bod)
+			lu := lookup.New(backend, &options.Options{
+				Root:            rootFlag,
+				MetadataBackend: bod,
+			})
+
+			m := migrator.New(lu, log)
+
+			err := m.RunMigration(c.String("migration"), c.String("direction") == "down")
+			if err != nil {
+				log.Error().Err(err).Msg("failed")
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+// ListDecomposedfsMigrations is the entrypoint for the decomposedfs list migrations command
+func ListDecomposedfsMigrations(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "list",
+		Usage: "list decomposedfs migrations",
+		Action: func(c *cli.Context) error {
+			rootFlag := c.String("root")
+			bod := lookup.DetectBackendOnDisk(rootFlag)
+			backend := backend(rootFlag, bod)
+			lu := lookup.New(backend, &options.Options{
+				Root:            rootFlag,
+				MetadataBackend: bod,
+			})
+
+			m := migrator.New(lu, logger())
+			migrationStates, err := m.Migrations()
+			if err != nil {
+				return err
+			}
+
+			migrations := []string{}
+			for m := range migrationStates {
+				migrations = append(migrations, m)
+			}
+			sort.Strings(migrations)
+
+			table := tw.NewWriter(os.Stdout)
+			table.SetHeader([]string{"Migration", "State", "Message"})
+			table.SetAutoFormatHeaders(false)
+			for _, migration := range migrations {
+				table.Append([]string{migration, migrationStates[migration].State, migrationStates[migration].Message})
+			}
+			table.Render()
+
+			return nil
+		},
+	}
+}
+
 func MigrateShares(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "shares",
@@ -65,7 +169,7 @@ func MigrateShares(cfg *config.Config) *cli.Command {
 			return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
 		},
 		Action: func(c *cli.Context) error {
-			log := oclog.LoggerFromConfig("migrate", cfg.Log)
+			log := logger()
 			ctx := log.WithContext(context.Background())
 			rcfg := revaShareConfig(cfg.Sharing)
 			oldDriver := c.String("from")
@@ -160,7 +264,7 @@ func MigratePublicShares(cfg *config.Config) *cli.Command {
 			return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
 		},
 		Action: func(c *cli.Context) error {
-			log := oclog.LoggerFromConfig("migrate", cfg.Log)
+			log := logger()
 			ctx := log.WithContext(context.Background())
 
 			rcfg := revaPublicShareConfig(cfg.Sharing)
@@ -301,3 +405,12 @@ func revaPublicShareConfig(cfg *sharing.Config) map[string]interface{} {
 		},
 	}
 }
+
+func logger() *zerolog.Logger {
+	log := oclog.NewLogger(
+		oclog.Name("migrate"),
+		oclog.Level("info"),
+		oclog.Pretty(true),
+		oclog.Color(true)).Logger
+	return &log
+}

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go

@@ -29,44 +29,55 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
 )
 
-// Migration0001 creates the spaces directory structure
+func init() {
-func (m *Migrator) Migration0001() (Result, error) {
+	registerMigration("0001", Migration0001{})
-	m.log.Info().Msg("Migrating spaces directory structure...")
+}
+
+type Migration0001 struct{}
+
+// Migrate creates the spaces directory structure
+func (m Migration0001) Migrate(migrator *Migrator) (Result, error) {
+	migrator.log.Info().Msg("Migrating spaces directory structure...")
 
 	// create spaces folder and iterate over existing nodes to populate it
-	nodesPath := filepath.Join(m.lu.InternalRoot(), "nodes")
+	nodesPath := filepath.Join(migrator.lu.InternalRoot(), "nodes")
 	fi, err := os.Stat(nodesPath)
 	if err == nil && fi.IsDir() {
 		f, err := os.Open(nodesPath)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 		nodes, err := f.Readdir(0)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 
 		for _, n := range nodes {
 			nodePath := filepath.Join(nodesPath, n.Name())
 
-			attr, err := m.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr)
+			attr, err := migrator.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr)
 			if err == nil && string(attr) == node.RootID {
-				if err := m.moveNode(n.Name(), n.Name()); err != nil {
+				if err := m.moveNode(migrator, n.Name(), n.Name()); err != nil {
-					m.log.Error().Err(err).
+					migrator.log.Error().Err(err).
 						Str("space", n.Name()).
 						Msg("could not move space")
 					continue
 				}
-				m.linkSpaceNode("personal", n.Name())
+				m.linkSpaceNode(migrator, "personal", n.Name())
 			}
 		}
 		// TODO delete nodesPath if empty
 	}
-	return resultSucceeded, nil
+	return stateSucceeded, nil
 }
 
-func (m *Migrator) moveNode(spaceID, nodeID string) error {
+// Rollback is not implemented
-	dirPath := filepath.Join(m.lu.InternalRoot(), "nodes", nodeID)
+func (Migration0001) Rollback(_ *Migrator) (Result, error) {
+	return stateFailed, errors.New("rollback not implemented")
+}
+
+func (m Migration0001) moveNode(migrator *Migrator, spaceID, nodeID string) error {
+	dirPath := filepath.Join(migrator.lu.InternalRoot(), "nodes", nodeID)
 	f, err := os.Open(dirPath)
 	if err != nil {
 		return err
@@ -76,10 +87,10 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error {
 		return err
 	}
 	for _, child := range children {
-		old := filepath.Join(m.lu.InternalRoot(), "nodes", child.Name())
+		old := filepath.Join(migrator.lu.InternalRoot(), "nodes", child.Name())
-		new := filepath.Join(m.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2))
+		new := filepath.Join(migrator.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2))
 		if err := os.Rename(old, new); err != nil {
-			m.log.Error().Err(err).
+			migrator.log.Error().Err(err).
 				Str("space", spaceID).
 				Str("nodes", child.Name()).
 				Str("oldpath", old).
@@ -87,7 +98,7 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error {
 				Msg("could not rename node")
 		}
 		if child.IsDir() {
-			if err := m.moveNode(spaceID, child.Name()); err != nil {
+			if err := m.moveNode(migrator, spaceID, child.Name()); err != nil {
 				return err
 			}
 		}
@@ -96,27 +107,27 @@ func (m *Migrator) moveNode(spaceID, nodeID string) error {
 }
 
 // linkSpace creates a new symbolic link for a space with the given type st, and node id
-func (m *Migrator) linkSpaceNode(spaceType, spaceID string) {
+func (m Migration0001) linkSpaceNode(migrator *Migrator, spaceType, spaceID string) {
-	spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, spaceID)
+	spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, spaceID)
 	expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2)
 	linkTarget, err := os.Readlink(spaceTypesPath)
 	if errors.Is(err, os.ErrNotExist) {
 		err = os.Symlink(expectedTarget, spaceTypesPath)
 		if err != nil {
-			m.log.Error().Err(err).
+			migrator.log.Error().Err(err).
 				Str("space_type", spaceType).
 				Str("space", spaceID).
 				Msg("could not create symlink")
 		}
 	} else {
 		if err != nil {
-			m.log.Error().Err(err).
+			migrator.log.Error().Err(err).
 				Str("space_type", spaceType).
 				Str("space", spaceID).
 				Msg("could not read symlink")
 		}
 		if linkTarget != expectedTarget {
-			m.log.Warn().
+			migrator.log.Warn().
 				Str("space_type", spaceType).
 				Str("space", spaceID).
 				Str("expected", expectedTarget).

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go

@@ -19,6 +19,7 @@
 package migrator
 
 import (
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -26,25 +27,31 @@ import (
 	"github.com/cs3org/reva/v2/pkg/logger"
 )
 
-// Migration0002 migrates spacetypes to indexes
+func init() {
-func (m *Migrator) Migration0002() (Result, error) {
+	registerMigration("0002", Migration0002{})
-	m.log.Info().Msg("Migrating space types indexes...")
+}
 
-	spaceTypesPath := filepath.Join(m.lu.InternalRoot(), "spacetypes")
+type Migration0002 struct{}
+
+// Migrate migrates spacetypes to indexes
+func (m Migration0002) Migrate(migrator *Migrator) (Result, error) {
+	migrator.log.Info().Msg("Migrating space types indexes...")
+
+	spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes")
 	fi, err := os.Stat(spaceTypesPath)
 	if err == nil && fi.IsDir() {
 
 		f, err := os.Open(spaceTypesPath)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 		spaceTypes, err := f.Readdir(0)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 
 		for _, st := range spaceTypes {
-			err := m.moveSpaceType(st.Name())
+			err := m.moveSpaceType(migrator, st.Name())
 			if err != nil {
 				logger.New().Error().Err(err).
 					Str("space", st.Name()).
@@ -59,7 +66,7 @@ func (m *Migrator) Migration0002() (Result, error) {
 			logger.New().Error().Err(err).
 				Str("spacetypesdir", spaceTypesPath).
 				Msg("could not open spacetypesdir")
-			return resultFailed, nil
+			return stateFailed, nil
 		}
 		defer d.Close()
 		_, err = d.Readdirnames(1) // Or f.Readdir(1)
@@ -77,11 +84,16 @@ func (m *Migrator) Migration0002() (Result, error) {
 				Msg("could not delete, not empty")
 		}
 	}
-	return resultSucceeded, nil
+	return stateSucceeded, nil
 }
 
-func (m *Migrator) moveSpaceType(spaceType string) error {
+// Rollback is not implemented
-	dirPath := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType)
+func (Migration0002) Rollback(_ *Migrator) (Result, error) {
+	return stateFailed, errors.New("rollback not implemented")
+}
+
+func (m Migration0002) moveSpaceType(migrator *Migrator, spaceType string) error {
+	dirPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType)
 	f, err := os.Open(dirPath)
 	if err != nil {
 		return err
@@ -91,7 +103,7 @@ func (m *Migrator) moveSpaceType(spaceType string) error {
 		return err
 	}
 	for _, child := range children {
-		old := filepath.Join(m.lu.InternalRoot(), "spacetypes", spaceType, child.Name())
+		old := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, child.Name())
 		target, err := os.Readlink(old)
 		if err != nil {
 			logger.New().Error().Err(err).
@@ -101,7 +113,7 @@ func (m *Migrator) moveSpaceType(spaceType string) error {
 				Msg("could not read old symlink")
 			continue
 		}
-		newDir := filepath.Join(m.lu.InternalRoot(), "indexes", "by-type", spaceType)
+		newDir := filepath.Join(migrator.lu.InternalRoot(), "indexes", "by-type", spaceType)
 		if err := os.MkdirAll(newDir, 0700); err != nil {
 			logger.New().Error().Err(err).
 				Str("space", spaceType).

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go

@@ -31,23 +31,29 @@ import (
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata"
 )
 
-// Migration0003 migrates the file metadata to the current backend.
+func init() {
+	registerMigration("0003", Migration0003{})
+}
+
+type Migration0003 struct{}
+
+// Migrate migrates the file metadata to the current backend.
 // Only the xattrs -> messagepack path is supported.
-func (m *Migrator) Migration0003() (Result, error) {
+func (m Migration0003) Migrate(migrator *Migrator) (Result, error) {
-	bod := lookup.DetectBackendOnDisk(m.lu.InternalRoot())
+	bod := lookup.DetectBackendOnDisk(migrator.lu.InternalRoot())
 	if bod == "" {
-		return resultFailed, errors.New("could not detect metadata backend on disk")
+		return stateFailed, errors.New("could not detect metadata backend on disk")
 	}
 
-	if bod != "xattrs" || m.lu.MetadataBackend().Name() != "messagepack" {
+	if bod != "xattrs" || migrator.lu.MetadataBackend().Name() != "messagepack" {
-		return resultSucceededRunAgain, nil
+		return stateSucceededRunAgain, nil
 	}
 
-	m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...")
+	migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...")
 	xattrs := metadata.XattrsBackend{}
-	mpk := metadata.NewMessagePackBackend(m.lu.InternalRoot(), cache.Config{})
+	mpk := metadata.NewMessagePackBackend(migrator.lu.InternalRoot(), cache.Config{})
 
-	spaces, _ := filepath.Glob(filepath.Join(m.lu.InternalRoot(), "spaces", "*", "*"))
+	spaces, _ := filepath.Glob(filepath.Join(migrator.lu.InternalRoot(), "spaces", "*", "*"))
 	for _, space := range spaces {
 		err := filepath.WalkDir(filepath.Join(space, "nodes"), func(path string, _ fs.DirEntry, err error) error {
 			// Do not continue on error
@@ -77,7 +83,7 @@ func (m *Migrator) Migration0003() (Result, error) {
 
 			attribs, err := xattrs.All(context.Background(), path)
 			if err != nil {
-				m.log.Error().Err(err).Str("path", path).Msg("error converting file")
+				migrator.log.Error().Err(err).Str("path", path).Msg("error converting file")
 				return err
 			}
 			if len(attribs) == 0 {
@@ -86,24 +92,29 @@ func (m *Migrator) Migration0003() (Result, error) {
 
 			err = mpk.SetMultiple(context.Background(), path, attribs, false)
 			if err != nil {
-				m.log.Error().Err(err).Str("path", path).Msg("error setting attributes")
+				migrator.log.Error().Err(err).Str("path", path).Msg("error setting attributes")
 				return err
 			}
 
 			for k := range attribs {
 				err = xattrs.Remove(context.Background(), path, k)
 				if err != nil {
-					m.log.Debug().Err(err).Str("path", path).Msg("error removing xattr")
+					migrator.log.Debug().Err(err).Str("path", path).Msg("error removing xattr")
 				}
 			}
 
 			return nil
 		})
 		if err != nil {
-			m.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend")
+			migrator.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend")
 		}
 	}
 
-	m.log.Info().Msg("done.")
+	migrator.log.Info().Msg("done.")
-	return resultSucceeded, nil
+	return stateSucceeded, nil
+}
+
+// Rollback is not implemented
+func (Migration0003) Rollback(_ *Migrator) (Result, error) {
+	return stateFailed, errors.New("rollback not implemented")
 }

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go

@@ -21,18 +21,25 @@ package migrator
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/shamaton/msgpack/v2"
 )
 
-// Migration0004 migrates the directory tree based space indexes to messagepack
+func init() {
-func (m *Migrator) Migration0004() (Result, error) {
+	registerMigration("0004", Migration0004{})
-	root := m.lu.InternalRoot()
+}
+
+type Migration0004 struct{}
+
+// Migrate migrates the directory tree based space indexes to messagepack
+func (Migration0004) Migrate(migrator *Migrator) (Result, error) {
+	root := migrator.lu.InternalRoot()
 
 	// migrate user indexes
 	users, err := os.ReadDir(filepath.Join(root, "indexes", "by-user-id"))
 	if err != nil {
-		m.log.Warn().Err(err).Msg("error listing user indexes")
+		migrator.log.Warn().Err(err).Msg("error listing user indexes")
 	}
 	for _, user := range users {
 		if !user.IsDir() {
@@ -41,19 +48,18 @@ func (m *Migrator) Migration0004() (Result, error) {
 		id := user.Name()
 		indexPath := filepath.Join(root, "indexes", "by-user-id", id+".mpk")
 		dirIndexPath := filepath.Join(root, "indexes", "by-user-id", id)
-		cacheKey := "by-user-id:" + id
 
-		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
+		migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
-		err := migrateSpaceIndex(indexPath, dirIndexPath, cacheKey)
+		err := migrateSpaceIndex(indexPath, dirIndexPath)
 		if err != nil {
-			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+			migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
 		}
 	}
 
 	// migrate group indexes
 	groups, err := os.ReadDir(filepath.Join(root, "indexes", "by-group-id"))
 	if err != nil {
-		m.log.Warn().Err(err).Msg("error listing group indexes")
+		migrator.log.Warn().Err(err).Msg("error listing group indexes")
 	}
 	for _, group := range groups {
 		if !group.IsDir() {
@@ -62,12 +68,11 @@ func (m *Migrator) Migration0004() (Result, error) {
 		id := group.Name()
 		indexPath := filepath.Join(root, "indexes", "by-group-id", id+".mpk")
 		dirIndexPath := filepath.Join(root, "indexes", "by-group-id", id)
-		cacheKey := "by-group-id:" + id
 
-		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
+		migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
-		err := migrateSpaceIndex(indexPath, dirIndexPath, cacheKey)
+		err := migrateSpaceIndex(indexPath, dirIndexPath)
 		if err != nil {
-			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+			migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
 		}
 	}
 
@@ -75,25 +80,24 @@ func (m *Migrator) Migration0004() (Result, error) {
 	for _, spaceType := range []string{"personal", "project", "share"} {
 		indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk")
 		dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType)
-		cacheKey := "by-type:" + spaceType
 
 		_, err := os.Stat(dirIndexPath)
 		if err != nil {
 			continue
 		}
 
-		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
+		migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
-		err = migrateSpaceIndex(indexPath, dirIndexPath, cacheKey)
+		err = migrateSpaceIndex(indexPath, dirIndexPath)
 		if err != nil {
-			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+			migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
 		}
 	}
 
-	m.log.Info().Msg("done.")
+	migrator.log.Info().Msg("done.")
-	return resultSucceeded, nil
+	return stateSucceeded, nil
 }
 
-func migrateSpaceIndex(indexPath, dirIndexPath, cacheKey string) error {
+func migrateSpaceIndex(indexPath, dirIndexPath string) error {
 	links := map[string][]byte{}
 	m, err := filepath.Glob(dirIndexPath + "/*")
 	if err != nil {
@@ -118,3 +122,82 @@ func migrateSpaceIndex(indexPath, dirIndexPath, cacheKey string) error {
 	}
 	return os.RemoveAll(dirIndexPath)
 }
+
+// Rollback migrates the directory messagepack indexes to symlinks
+func (Migration0004) Rollback(m *Migrator) (Result, error) {
+	root := m.lu.InternalRoot()
+
+	// migrate user indexes
+	users, err := filepath.Glob(filepath.Join(root, "indexes", "by-user-id", "*.mpk"))
+	if err != nil {
+		m.log.Warn().Err(err).Msg("error listing user indexes")
+	}
+	for _, indexPath := range users {
+		dirIndexPath := strings.TrimSuffix(indexPath, ".mpk")
+
+		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
+		err := downSpaceIndex(indexPath, dirIndexPath)
+		if err != nil {
+			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+		}
+	}
+
+	// migrate group indexes
+	groups, err := filepath.Glob(filepath.Join(root, "indexes", "by-group-id", "*.mpk"))
+	if err != nil {
+		m.log.Warn().Err(err).Msg("error listing group indexes")
+	}
+	for _, indexPath := range groups {
+		dirIndexPath := strings.TrimSuffix(indexPath, ".mpk")
+
+		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
+		err := downSpaceIndex(indexPath, dirIndexPath)
+		if err != nil {
+			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+		}
+	}
+
+	// migrate project indexes
+	for _, spaceType := range []string{"personal", "project", "share"} {
+		indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk")
+		dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType)
+
+		_, err := os.Stat(indexPath)
+		if err != nil || os.IsNotExist(err) {
+			continue
+		}
+
+		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
+		err = downSpaceIndex(indexPath, dirIndexPath)
+		if err != nil {
+			m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
+		}
+	}
+	return stateDown, nil
+}
+
+func downSpaceIndex(indexPath, dirIndexPath string) error {
+	d, err := os.ReadFile(indexPath)
+	if err != nil {
+		return err
+	}
+
+	links := map[string][]byte{}
+	err = msgpack.Unmarshal(d, &links)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(dirIndexPath, 0700)
+	if err != nil {
+		return err
+	}
+	for link, target := range links {
+		err = os.Symlink(string(target), filepath.Join(dirIndexPath, link))
+		if err != nil {
+			return err
+		}
+	}
+
+	return os.Remove(indexPath)
+}

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go

@@ -25,27 +25,33 @@ import (
 	"github.com/shamaton/msgpack/v2"
 )
 
-// Migration0005 fixes the messagepack space index data structure
+func init() {
-func (m *Migrator) Migration0005() (Result, error) {
+	registerMigration("0005", Migration0005{})
-	root := m.lu.InternalRoot()
+}
+
+type Migration0005 struct{}
+
+// Migrate fixes the messagepack space index data structure
+func (Migration0005) Migrate(migrator *Migrator) (Result, error) {
+	root := migrator.lu.InternalRoot()
 
 	indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk"))
 	if err != nil {
-		return resultFailed, err
+		return stateFailed, err
 	}
 	for _, i := range indexes {
-		m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Fixing index format of " + i)
+		migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i)
 
 		// Read old-format index
 		oldData, err := os.ReadFile(i)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 		oldIndex := map[string][]byte{}
 		err = msgpack.Unmarshal(oldData, &oldIndex)
 		if err != nil {
 			// likely already migrated -> skip
-			m.log.Warn().Str("root", m.lu.InternalRoot()).Msg("Invalid index format found in " + i)
+			migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i)
 			continue
 		}
 
@@ -56,13 +62,53 @@ func (m *Migrator) Migration0005() (Result, error) {
 		}
 		newData, err := msgpack.Marshal(newIndex)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 		err = os.WriteFile(i, newData, 0600)
 		if err != nil {
-			return resultFailed, err
+			return stateFailed, err
 		}
 	}
-	m.log.Info().Msg("done.")
+	migrator.log.Info().Msg("done.")
-	return resultSucceeded, nil
+	return stateSucceeded, nil
+}
+
+// Rollback rolls back the migration
+func (Migration0005) Rollback(migrator *Migrator) (Result, error) {
+	root := migrator.lu.InternalRoot()
+
+	indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk"))
+	if err != nil {
+		return stateFailed, err
+	}
+	for _, i := range indexes {
+		migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i)
+
+		oldData, err := os.ReadFile(i)
+		if err != nil {
+			return stateFailed, err
+		}
+		oldIndex := map[string]string{}
+		err = msgpack.Unmarshal(oldData, &oldIndex)
+		if err != nil {
+			migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i)
+			continue
+		}
+
+		// Write new-format index
+		newIndex := map[string][]byte{}
+		for k, v := range oldIndex {
+			newIndex[k] = []byte(v)
+		}
+		newData, err := msgpack.Marshal(newIndex)
+		if err != nil {
+			return stateFailed, err
+		}
+		err = os.WriteFile(i, newData, 0600)
+		if err != nil {
+			return stateFailed, err
+		}
+	}
+	migrator.log.Info().Msg("done.")
+	return stateDown, nil
 }

vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator/migrator.go

@@ -20,28 +20,53 @@ package migrator
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
-	"reflect"
+	"sort"
 
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
 	"github.com/rogpeppe/go-internal/lockedfile"
 	"github.com/rs/zerolog"
 )
 
-var allMigrations = []string{"0001", "0002", "0003", "0004", "0005"}
-
 const (
-	resultFailed            = "failed"
+	statePending           = "pending"
-	resultSucceeded         = "succeeded"
+	stateFailed            = "failed"
-	resultSucceededRunAgain = "runagain"
+	stateSucceeded         = "succeeded"
+	stateDown              = "down"
+	stateSucceededRunAgain = "runagain"
 )
 
-type migrationState struct {
+type migration interface {
+	Migrate(*Migrator) (Result, error)
+	Rollback(*Migrator) (Result, error)
+}
+
+var migrations = map[string]migration{}
+
+type migrationStates map[string]MigrationState
+
+func registerMigration(name string, migration migration) {
+	migrations[name] = migration
+}
+
+func allMigrations() []string {
+	ms := []string{}
+
+	for k := range migrations {
+		ms = append(ms, k)
+	}
+
+	sort.Strings(ms)
+	return ms
+}
+
+// MigrationState holds the state of a migration
+type MigrationState struct {
 	State   string
 	Message string
 }
-type migrationStates map[string]migrationState
 
 // Result represents the result of a migration run
 type Result string
@@ -61,6 +86,71 @@ func New(lu *lookup.Lookup, log *zerolog.Logger) Migrator {
 	}
 }
 
+// Migrations returns the list of migrations and their states
+func (m *Migrator) Migrations() (map[string]MigrationState, error) {
+	err := m.readStates()
+	if err != nil {
+		return nil, err
+	}
+
+	states := map[string]MigrationState{}
+	for _, migration := range allMigrations() {
+		if s, ok := m.states[migration]; ok {
+			states[migration] = s
+		} else {
+			states[migration] = MigrationState{
+				State: statePending,
+			}
+		}
+	}
+
+	return states, nil
+}
+
+// RunMigration runs or rolls back a migration
+func (m *Migrator) RunMigration(id string, rollback bool) error {
+	if _, ok := migrations[id]; !ok {
+		return fmt.Errorf("invalid migration '%s'", id)
+	}
+
+	lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	defer lock.Close()
+
+	err = m.readStates()
+	if err != nil {
+		return err
+	}
+
+	var res Result
+	if !rollback {
+		m.log.Info().Msg("Running migration " + id + "...")
+		res, err = migrations[id].Migrate(m)
+	} else {
+		m.log.Info().Msg("Rolling back migration " + id + "...")
+		res, err = migrations[id].Rollback(m)
+	}
+
+	// write back state
+	s := m.states[id]
+	s.State = string(res)
+
+	if err != nil {
+		m.log.Error().Err(err).Msg("migration " + id + " failed")
+		s.Message = err.Error()
+	}
+
+	m.states[id] = s
+	err = m.writeStates()
+	if err != nil {
+		return err
+	}
+	m.log.Info().Msg("done")
+	return nil
+}
+
 // RunMigrations runs all migrations in sequence. Note this sequence must not be changed or it might
 // damage existing decomposed fs.
 func (m *Migrator) RunMigrations() error {
@@ -75,17 +165,15 @@ func (m *Migrator) RunMigrations() error {
 		return err
 	}
 
-	for _, migration := range allMigrations {
+	for _, migration := range allMigrations() {
 		s := m.states[migration]
-		if s.State == "succeeded" {
+		if s.State == stateSucceeded || s.State == stateDown {
 			continue
 		}
 
-		migrateMethod := reflect.ValueOf(m).MethodByName("Migration" + migration)
+		res, err := migrations[migration].Migrate(m)
-		v := migrateMethod.Call(nil)
+		s.State = string(res)
-		s.State = string(v[0].Interface().(Result))
+		if err != nil {
-		if v[1].Interface() != nil {
-			err := v[1].Interface().(error)
 			m.log.Error().Err(err).Msg("migration " + migration + " failed")
 			s.Message = err.Error()
 		}

vendor/modules.txt

@@ -352,7 +352,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
 github.com/cs3org/go-cs3apis/cs3/types/v1beta1
-# github.com/cs3org/reva/v2 v2.15.1-0.20230726135727-c444e4c5a24f
+# github.com/cs3org/reva/v2 v2.15.1-0.20230731061316-db79e9b61738
 ## explicit; go 1.20
 github.com/cs3org/reva/v2/cmd/revad/internal/grace
 github.com/cs3org/reva/v2/cmd/revad/runtime