Merge pull request #9497 from kobergj/RemoveVersionsCli

Purge-Revision Cli
2024-07-08 07:46:34 +00:00
parent c4101fa9e2 a08860c62f
commit 6b71bfe63d
4 changed files with 393 additions and 1 deletions
@@ -0,0 +1,5 @@
+Enhancement: Add cli to purge revisions
+
+Adds a cli that allows removing all revisions for a storage-provider.
+
+https://github.com/owncloud/ocis/pull/9497
@@ -23,4 +23,82 @@ To authenticate the connection to the etcd registry, you have to set `ETCD_USERN

 ## Memory limits

-oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).
+oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).
+
+## CLI Commands
+
+The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common mandatory parameter: `--basePath` (or `-p`) which needs to point to a storage provider. Example paths are:
+
+```bash
+.ocis/storage/users          # bare metal installation
+/var/tmp/ocis/storage/users  # docker installation
+...
+```
+
+These paths can vary depending on your ocis installation.
+
+### Backup CLI
+
+The backup command allows inspecting the consistency of an ocis storage:
+
+```bash
+ocis backup consistency -p /base/path/storage/users
+```
+
+This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be:
+
+* **Orphaned Blobs**\
+A blob in the blobstore that is not referenced by any file metadata.
+* **Missing Blobs**\
+A blob referenced by file metadata that is not present in the blobstore.
+* **Missing Nodes**\
+A node that is referenced by a symlink but doesn't exist.
+* **Missing Link**\
+A node that is not referenced by any symlink but should be.
+* **Missing Files**\
+A node that is missing essential files (such as the `.mpk` metadata file).
+* **Missing/Malformed Metadata**\
+A node that doesn't have any (or malformed) metadata.
+
+This command provides additional options:
+
+* `-b` / `--blobstore`\
+Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details).
+* `--fail`\
+Exits with non-zero exit code if inconsistencies are found. Useful for automation.
+
+### Revisions CLI
+
+The revisions command allows removing the revisions of files in the storage.
+
+```bash
+ocis revisions purge -p /base/path/storage/users
+```
+
+It takes the `--resource-id` (or `--r`) parameter which specify the scope of the command:
+
+* An empty string (default) removes all revisions from all spaces.
+* A spaceID (like `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space.
+* A resourceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49\!e8a73d49-2e00-4322-9f34-9d7f178577b2`) removes all revisions from that specific file.
+
+This command provides additional options:
+
+* `--dry-run` (default: `true`)\
+Do not remove any revisions but print the revisions that would be removed.
+* `-b` / `--blobstore`\
+Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details).
+* `-v` / `--verbose`\
+Prints additional information about the revisions that are removed.
+
+### Trash CLI
+
+The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations.
+
+```bash
+ocis trash purge-empty-dirs -p /base/path/storage/users
+```
+
+This command provides additional options:
+
+* `--dry-run` (default: `true`)\
+Do not remove any empty folders but print the empty folders that would be removed.
@@ -0,0 +1,152 @@
+package command
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
+	"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
+	s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
+	"github.com/cs3org/reva/v2/pkg/storagespace"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/parser"
+	"github.com/owncloud/ocis/v2/ocis/pkg/register"
+	"github.com/owncloud/ocis/v2/ocis/pkg/revisions"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// _nodesGlobPattern is the glob pattern to find all nodes
+	_nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*"
+)
+
+// RevisionsCommand is the entrypoint for the revisions command.
+func RevisionsCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "revisions",
+		Usage: "ocis revisions functionality",
+		Subcommands: []*cli.Command{
+			PurgeRevisionsCommand(cfg),
+		},
+		Before: func(c *cli.Context) error {
+			return configlog.ReturnError(parser.ParseConfig(cfg, true))
+		},
+		Action: func(_ *cli.Context) error {
+			fmt.Println("Read the docs")
+			return nil
+		},
+	}
+}
+
+// PurgeRevisionsCommand allows removing all revisions from a storage provider.
+func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "purge",
+		Usage: "purge revisions",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "basepath",
+				Aliases:  []string{"p"},
+				Usage:    "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:    "blobstore",
+				Aliases: []string{"b"},
+				Usage:   "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service",
+				Value:   "ocis",
+			},
+			&cli.BoolFlag{
+				Name:  "dry-run",
+				Usage: "do not delete anything, just print what would be deleted",
+				Value: true,
+			},
+			&cli.BoolFlag{
+				Name:    "verbose",
+				Aliases: []string{"v"},
+				Usage:   "print verbose output",
+				Value:   false,
+			},
+			&cli.StringFlag{
+				Name:    "resource-id",
+				Aliases: []string{"r"},
+				Usage:   "purge all revisions of this file/space. If not set, all revisions will be purged",
+			},
+		},
+		Action: func(c *cli.Context) error {
+			basePath := c.String("basepath")
+			if basePath == "" {
+				fmt.Println("basepath is required")
+				return cli.ShowCommandHelp(c, "revisions")
+			}
+
+			var (
+				bs  revisions.DelBlobstore
+				err error
+			)
+			switch c.String("blobstore") {
+			case "s3ng":
+				bs, err = s3bs.New(
+					cfg.StorageUsers.Drivers.S3NG.Endpoint,
+					cfg.StorageUsers.Drivers.S3NG.Region,
+					cfg.StorageUsers.Drivers.S3NG.Bucket,
+					cfg.StorageUsers.Drivers.S3NG.AccessKey,
+					cfg.StorageUsers.Drivers.S3NG.SecretKey,
+					s3bs.Options{},
+				)
+			case "ocis":
+				bs, err = ocisbs.New(basePath)
+			case "none":
+				bs = nil
+			default:
+				err = errors.New("blobstore type not supported")
+			}
+			if err != nil {
+				fmt.Println(err)
+				return err
+			}
+
+			p, err := generatePath(basePath, c.String("resource-id"))
+			if err != nil {
+				fmt.Printf("❌ Error parsing resourceID: %s", err)
+				return err
+			}
+
+			if err := revisions.PurgeRevisions(p, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil {
+				fmt.Printf("❌ Error purging revisions: %s", err)
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func generatePath(basePath string, resourceID string) (string, error) {
+	if resourceID == "" {
+		return filepath.Join(basePath, _nodesGlobPattern), nil
+	}
+
+	rid, err := storagespace.ParseID(resourceID)
+	if err != nil {
+		return "", err
+	}
+
+	sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
+	if sid == "" {
+		sid = "*/*"
+	}
+
+	nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
+	if nid == "" {
+		nid = "*/*/*/*/"
+	}
+
+	return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*"), nil
+}
+
+func init() {
+	register.AddCommand(RevisionsCommand)
+}
@@ -0,0 +1,157 @@
+// Package revisions allows manipulating revisions in a storage provider.
+package revisions
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
+	"github.com/shamaton/msgpack/v2"
+)
+
+var (
+	// regex to determine if a node versioned. Examples:
+	// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
+	// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk
+	// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock
+	_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`)
+)
+
+// DelBlobstore is the interface for a blobstore that can delete blobs.
+type DelBlobstore interface {
+	Delete(node *node.Node) error
+}
+
+// PurgeRevisions removes all revisions from a storage provider.
+func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) error {
+	if verbose {
+		fmt.Println("Looking for nodes in", pattern)
+	}
+
+	nodes, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	if len(nodes) == 0 {
+		return errors.New("no nodes found, double check storage path")
+	}
+
+	countFiles := 0
+	countBlobs := 0
+	countRevisions := 0
+	for _, d := range nodes {
+		if !_versionRegex.MatchString(d) {
+			continue
+		}
+
+		var blobID string
+		e := filepath.Ext(d)
+		switch e {
+		case ".mpk":
+			blobID, err = getBlobID(d)
+			if err != nil {
+				fmt.Printf("error getting blobID from %s: %v\n", d, err)
+				continue
+			}
+
+			countBlobs++
+		case ".mlock":
+			// no extra action on .mlock files
+		default:
+			countRevisions++
+		}
+
+		if !dryRun {
+			if blobID != "" {
+				//  TODO: needs spaceID for s3ng
+				if err := bs.Delete(&node.Node{BlobID: blobID}); err != nil {
+					fmt.Printf("error deleting blob %s: %v\n", blobID, err)
+					continue
+				}
+			}
+
+			if err := os.Remove(d); err != nil {
+				fmt.Printf("error removing %s: %v\n", d, err)
+				continue
+			}
+
+		}
+
+		countFiles++
+
+		if verbose {
+			spaceID, nodeID := getIDsFromPath(d)
+			if dryRun {
+				fmt.Println("Would delete")
+				fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
+				fmt.Println("\tSpaceID:", spaceID)
+				fmt.Println("\tPath:", d)
+				if blobID != "" {
+					fmt.Println("\tBlob:", blobID)
+				}
+			} else {
+				fmt.Println("Deleted")
+				fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
+				fmt.Println("\tSpaceID:", spaceID)
+				fmt.Println("\tPath:", d)
+				if blobID != "" {
+					fmt.Println("\tBlob:", blobID)
+				}
+			}
+		}
+	}
+
+	switch {
+	case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
+		fmt.Println("❎ No revisions found. Storage provider is clean.")
+	case !dryRun:
+		fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
+	default:
+		fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
+	}
+	return nil
+}
+
+func getBlobID(path string) (string, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	m := map[string][]byte{}
+	if err := msgpack.Unmarshal(b, &m); err != nil {
+		return "", err
+	}
+
+	if bid := m["user.ocis.blobid"]; string(bid) != "" {
+		return string(bid), nil
+	}
+
+	return "", nil
+}
+
+func getIDsFromPath(path string) (string, string) {
+	rawIDs := strings.Split(path, "/nodes/")
+	if len(rawIDs) != 2 {
+		return "", ""
+	}
+
+	s := strings.Split(rawIDs[0], "/spaces/")
+	if len(s) != 2 {
+		return "", ""
+	}
+
+	n := strings.Split(rawIDs[1], ".REV.")
+	if len(n) != 2 {
+		return "", ""
+	}
+
+	spaceID := strings.Replace(s[1], "/", "", -1)
+	nodeID := strings.Replace(n[0], "/", "", -1)
+	return spaceID, filepath.Base(nodeID)
+}