feat(ocis): improve consistency command

Signed-off-by: jkoberg <jkoberg@owncloud.com>
Co-authored-by: dragonchaser <crichter@owncloud.com>
Signed-off-by: jkoberg <jkoberg@owncloud.com>

ocis/pkg/backup/backup.go

@@ -7,28 +7,45 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
 	"github.com/shamaton/msgpack/v2"
 )
 
-// Inconsistency describes the type of incosistency
+// Inconsistency describes the type of inconsistency
 type Inconsistency string
 
 var (
-	InconsistencyBlobMissing     Inconsistency = "blob missing"
+	// InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore
-	InconsistencyBlobOrphaned    Inconsistency = "blob orphaned"
+	InconsistencyBlobMissing Inconsistency = "blob missing"
-	InconsistencyNodeMissing     Inconsistency = "node missing"
+	// InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference
+	InconsistencyBlobOrphaned Inconsistency = "blob orphaned"
+	// InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node
+	InconsistencyNodeMissing Inconsistency = "node missing"
+	// InconsistencyMetadataMissing is an inconsistency where a node is missing metadata
 	InconsistencyMetadataMissing Inconsistency = "metadata missing"
-	InconsistencySymlinkMissing  Inconsistency = "symlink missing"
+	// InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink
+	InconsistencySymlinkMissing Inconsistency = "symlink missing"
+	// InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock
+	InconsistencyFilesMissing Inconsistency = "files missing"
+	// InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file
+	InconsistencyMalformedFile Inconsistency = "malformed file"
+
+	// regex to determine if a node is trashed or versioned.
+	// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
+	_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`)
+	//   9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink
+	_trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`)
 )
 
 // Consistency holds the node and blob data of a space
 type Consistency struct {
-	Nodes map[string]Inconsistency
+	Nodes          map[string][]Inconsistency
-	Links map[string]Inconsistency
+	Links          map[string][]Inconsistency
-	Blobs map[string]Inconsistency
+	BlobReferences map[string][]Inconsistency
+	Blobs          map[string][]Inconsistency
 
 	fsys     fs.FS
 	discpath string
@@ -45,9 +62,10 @@ type ListBlobstore interface {
 // New creates a new Consistency object
 func New(fsys fs.FS, discpath string) *Consistency {
 	return &Consistency{
-		Nodes: make(map[string]Inconsistency),
+		Nodes:          make(map[string][]Inconsistency),
-		Links: make(map[string]Inconsistency),
+		Links:          make(map[string][]Inconsistency),
-		Blobs: make(map[string]Inconsistency),
+		BlobReferences: make(map[string][]Inconsistency),
+		Blobs:          make(map[string][]Inconsistency),
 
 		fsys:     fsys,
 		discpath: discpath,
@@ -55,71 +73,24 @@ func New(fsys fs.FS, discpath string) *Consistency {
 }
 
 // CheckSpaceConsistency checks the consistency of a space
-func CheckSpaceConsistency(pathToSpace string, lbs ListBlobstore) error {
+func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error {
-	fsys := os.DirFS(pathToSpace)
+	fsys := os.DirFS(storagepath)
 
-	con := New(fsys, pathToSpace)
+	c := New(fsys, storagepath)
-	err := con.Initialize()
+	if err := c.Initialize(); err != nil {
-	if err != nil {
 		return err
 	}
 
-	for n := range con.Nodes {
+	if err := c.Evaluate(lbs); err != nil {
-		if _, ok := con.Links[n]; !ok {
-			// TODO: This is no inconsistency if
-			// * this is the spaceroot
-			// * this is a trashed node
-			con.Nodes[n] = InconsistencySymlinkMissing
-			continue
-		}
-		delete(con.Links, n)
-		delete(con.Nodes, n)
-	}
-
-	for l := range con.Links {
-		con.Links[l] = InconsistencyNodeMissing
-	}
-
-	blobs, err := lbs.List()
-	if err != nil {
 		return err
 	}
 
-	deadBlobs := make(map[string]Inconsistency)
+	return c.PrintResults()
-	for _, b := range blobs {
-		if _, ok := con.Blobs[b]; !ok {
-			deadBlobs[b] = InconsistencyBlobOrphaned
-			continue
-		}
-		delete(con.Blobs, b)
-		delete(deadBlobs, b)
-	}
-
-	for b := range con.Blobs {
-		con.Blobs[b] = InconsistencyBlobMissing
-	}
-	// get blobs from blobstore.
-	// initialize blobstore (s3, local)
-	// compare con.Blobs with blobstore.GetBlobs()
-
-	for n := range con.Nodes {
-		fmt.Println("Inconsistency", n, con.Nodes[n])
-	}
-	for l := range con.Links {
-		fmt.Println("Inconsistency", l, con.Links[l])
-	}
-	for b := range con.Blobs {
-		fmt.Println("Inconsistency", b, con.Blobs[b])
-	}
-	for b := range deadBlobs {
-		fmt.Println("Inconsistency", b, deadBlobs[b])
-	}
-
-	return nil
 }
 
+// Initialize initializes the Consistency object
 func (c *Consistency) Initialize() error {
-	dirs, err := fs.Glob(c.fsys, "nodes/*/*/*/*")
+	dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*")
 	if err != nil {
 		return err
 	}
@@ -129,6 +100,7 @@ func (c *Consistency) Initialize() error {
 		if err != nil {
 			return err
 		}
+
 		for _, e := range entries {
 			switch {
 			case e.IsDir():
@@ -138,52 +110,162 @@ func (c *Consistency) Initialize() error {
 					continue
 				}
 				for _, l := range ls {
-					p, err := filepath.EvalSymlinks(filepath.Join(c.discpath, d, e.Name(), l.Name()))
+					p, err := os.Readlink(filepath.Join(c.discpath, d, e.Name(), l.Name()))
 					if err != nil {
-						fmt.Println("error evaluating symlink", filepath.Join(d, e.Name(), l.Name()), err)
+						fmt.Println("error reading symlink", err)
-						continue
 					}
-					c.Links[p] = ""
+					p = filepath.Join(c.discpath, d, e.Name(), p)
+					c.Links[p] = []Inconsistency{}
 				}
-			case filepath.Ext(e.Name()) == ".mpk":
+				fallthrough
-				inc, err := c.checkNode(filepath.Join(d, e.Name()))
+			case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()):
-				if err != nil {
+				if !c.filesExist(filepath.Join(d, e.Name())) {
-					fmt.Println("error checking node", err)
+					dp := filepath.Join(c.discpath, d, e.Name())
-					continue
+					c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing)
+				}
+				inc := c.checkNode(filepath.Join(d, e.Name()+".mpk"))
+				dp := filepath.Join(c.discpath, d, e.Name())
+				if inc != "" {
+					c.Nodes[dp] = append(c.Nodes[dp], inc)
+				} else if len(c.Nodes[dp]) == 0 {
+					c.Nodes[dp] = []Inconsistency{}
 				}
-				c.Nodes[filepath.Join(c.discpath, d, strings.TrimSuffix(e.Name(), ".mpk"))] = inc
-			default:
-				// fmt.Println("unknown", e.Name())
 			}
 		}
 	}
+
+	links, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*")
+	if err != nil {
+		return err
+	}
+	for _, l := range links {
+		p, err := os.Readlink(filepath.Join(c.discpath, l))
+		if err != nil {
+			fmt.Println("error reading symlink", err)
+		}
+		p = filepath.Join(c.discpath, l, "..", p)
+		c.Links[p] = []Inconsistency{}
+	}
 	return nil
 }
 
-func (c *Consistency) checkNode(path string) (Inconsistency, error) {
+// Evaluate evaluates inconsistencies
+func (c *Consistency) Evaluate(lbs ListBlobstore) error {
+	for n := range c.Nodes {
+		if _, ok := c.Links[n]; !ok && c.requiresSymlink(n) {
+			c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing)
+			continue
+		}
+
+		deleteInconsistency(c.Links, n)
+		deleteInconsistency(c.Nodes, n)
+	}
+
+	for l := range c.Links {
+		c.Links[l] = append(c.Links[l], InconsistencyNodeMissing)
+	}
+
+	blobs, err := lbs.List()
+	if err != nil {
+		return err
+	}
+
+	for _, b := range blobs {
+		if _, ok := c.BlobReferences[b]; !ok {
+			c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned)
+			continue
+		}
+		deleteInconsistency(c.BlobReferences, b)
+	}
+
+	for b := range c.BlobReferences {
+		c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing)
+	}
+
+	return nil
+}
+
+// PrintResults prints the results of the evaluation
+func (c *Consistency) PrintResults() error {
+	if len(c.Nodes) != 0 {
+		fmt.Println("🚨 Inconsistent Nodes:")
+	}
+	for n := range c.Nodes {
+		fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n)
+	}
+	if len(c.Links) != 0 {
+		fmt.Println("🚨 Inconsistent Links:")
+	}
+	for l := range c.Links {
+		fmt.Printf("\t👉️ %v\tpath: %s\n", c.Links[l], l)
+	}
+	if len(c.Blobs) != 0 {
+		fmt.Println("🚨 Inconsistent Blobs:")
+	}
+	for b := range c.Blobs {
+		fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b)
+	}
+	if len(c.BlobReferences) != 0 {
+		fmt.Println("🚨 Inconsistent BlobReferences:")
+	}
+	for b := range c.BlobReferences {
+		fmt.Printf("\t👉️ %v\tblob: %s\n", c.BlobReferences[b], b)
+	}
+	if len(c.Nodes) == 0 && len(c.Links) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 {
+		fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath)
+	}
+	return nil
+
+}
+
+func (c *Consistency) checkNode(path string) Inconsistency {
 	b, err := fs.ReadFile(c.fsys, path)
 	if err != nil {
-		return "", err
+		return InconsistencyFilesMissing
 	}
 
 	m := map[string][]byte{}
 	if err := msgpack.Unmarshal(b, &m); err != nil {
-		return "", err
+		return InconsistencyMalformedFile
 	}
 
 	if bid := m["user.ocis.blobid"]; string(bid) != "" {
-		c.Blobs[string(bid)] = ""
+		c.BlobReferences[string(bid)] = []Inconsistency{}
 	}
 
-	return "", nil
+	return ""
 }
 
-func iterate(fsys fs.FS, path string, d *Consistency) ([]string, error) {
+func (c *Consistency) requiresSymlink(path string) bool {
-	// open symlink -> NodeMissing
+	rawIDs := strings.Split(path, "/nodes/")
-	// remove node from data.Nodes
+	if len(rawIDs) != 2 {
-	// check blob -> BlobMissing
+		return true
-	// remove blob from data.Blobs
+	}
-	// list children (symlinks!)
+
-	// return children (symlinks!)
+	s := strings.Split(rawIDs[0], "/spaces/")
-	return nil, nil
+	if len(s) != 2 {
+		return true
+	}
+
+	spaceID := strings.Replace(s[1], "/", "", -1)
+	nodeID := strings.Replace(rawIDs[1], "/", "", -1)
+	if spaceID == nodeID || _versionRegex.MatchString(nodeID) {
+		return false
+	}
+
+	return true
+}
+
+func (c *Consistency) filesExist(path string) bool {
+	check := func(p string) bool {
+		_, err := fs.Stat(c.fsys, p)
+		return err == nil
+	}
+	return check(path) && check(path+".mpk")
+}
+
+func deleteInconsistency(incs map[string][]Inconsistency, path string) {
+	if len(incs[path]) == 0 {
+		delete(incs, path)
+	}
 }

ocis/pkg/command/backup.go

@@ -2,9 +2,9 @@ package command
 
 import (
 	"fmt"
-	"path/filepath"
 
-	"github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
+	ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
+	s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
 	"github.com/owncloud/ocis/v2/ocis-pkg/config"
 	"github.com/owncloud/ocis/v2/ocis/pkg/backup"
 	"github.com/owncloud/ocis/v2/ocis/pkg/register"
@@ -19,7 +19,7 @@ func BackupCommand(cfg *config.Config) *cli.Command {
 		Subcommands: []*cli.Command{
 			ConsistencyCommand(cfg),
 		},
-		Action: func(c *cli.Context) error {
+		Action: func(_ *cli.Context) error {
 			fmt.Println("Read the docs")
 			return nil
 		},
@@ -27,11 +27,17 @@ func BackupCommand(cfg *config.Config) *cli.Command {
 }
 
 // ConsistencyCommand is the entrypoint for the consistency Command
-func ConsistencyCommand(_ *config.Config) *cli.Command {
+func ConsistencyCommand(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "consistency",
 		Usage: "check backup consistency",
 		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "basepath",
+				Aliases:  []string{"p"},
+				Usage:    "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)",
+				Required: true,
+			},
 			&cli.StringFlag{
 				Name:    "blobstore",
 				Aliases: []string{"b"},
@@ -39,16 +45,34 @@ func ConsistencyCommand(_ *config.Config) *cli.Command {
 			},
 		},
 		Action: func(c *cli.Context) error {
-			basePath := "/home/jkoberg/.ocis/storage/users"
+			basePath := c.String("basepath")
+			if basePath == "" {
+				fmt.Println("basepath is required")
+				return cli.ShowCommandHelp(c, "consistency")
+			}
 
-			// TODO: switch for s3ng blobstore
+			var (
-			bs, err := blobstore.New(basePath)
+				bs  backup.ListBlobstore
+				err error
+			)
+			switch c.String("blobstore") {
+			case "s3ng":
+				bs, err = s3bs.New(
+					cfg.StorageUsers.Drivers.S3NG.Endpoint,
+					cfg.StorageUsers.Drivers.S3NG.Region,
+					cfg.StorageUsers.Drivers.S3NG.Bucket,
+					cfg.StorageUsers.Drivers.S3NG.AccessKey,
+					cfg.StorageUsers.Drivers.S3NG.SecretKey,
+					s3bs.Options{},
+				)
+			case "ocis":
+				bs, err = ocisbs.New(basePath)
+			}
 			if err != nil {
 				fmt.Println(err)
 				return err
 			}
-
+			if err := backup.CheckSpaceConsistency(basePath, bs); err != nil {
-			if err := backup.CheckSpaceConsistency(filepath.Join(basePath, "spaces/23/ebf113-76d4-43c0-8594-df974b02cd74"), bs); err != nil {
 				fmt.Println(err)
 				return err
 			}