diff --git a/changelog/unreleased/add-consistency-check-command.md b/changelog/unreleased/add-consistency-check-command.md new file mode 100644 index 000000000..1f44a7175 --- /dev/null +++ b/changelog/unreleased/add-consistency-check-command.md @@ -0,0 +1,5 @@ +Enhancement: Add command to check ocis backup consistency + +Adds a command that checks the consistency of an ocis backup. + +https://github.com/owncloud/ocis/pull/9238 diff --git a/docs/ocis/backup.md b/docs/ocis/backup.md index 093e180b4..07965a82c 100644 --- a/docs/ocis/backup.md +++ b/docs/ocis/backup.md @@ -217,3 +217,13 @@ BACKUP RECOMMENDED/OMITABLE. This folder contains custom web assets. Can be spec When using an external idp/idm/nats or blobstore, its data needs to be backed up separately. Refer to your idp/idm/nats/blobstore documentation for backup details. +## Backup Consistency Command + +Infinite Scale now allows checking an existing backup for consistency. Use the command: +```bash +ocis backup consistency -p "" +``` + +`path-to-base-folder` needs to be replaced with the path to the storage providers base path. Should be same as the `STORAGE_USERS_OCIS_ROOT` + +Use the `-b s3ng` option when using an external (s3) blobstore. Note: When using this flag, the path to the blobstore must be configured via envvars or a yaml file to match the configuration of the original instance. Consistency checks for other blobstores than `ocis` and `s3ng` are not supported at the moment. diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go new file mode 100644 index 000000000..27186a6f6 --- /dev/null +++ b/ocis/pkg/backup/backup.go @@ -0,0 +1,173 @@ +// Package backup contains ocis backup functionality. +package backup + +import ( + "fmt" + "os" + "regexp" +) + +// Inconsistency describes the type of inconsistency +type Inconsistency string + +var ( + // InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore + InconsistencyBlobMissing Inconsistency = "blob missing" + // InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference + InconsistencyBlobOrphaned Inconsistency = "blob orphaned" + // InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node + InconsistencyNodeMissing Inconsistency = "node missing" + // InconsistencyMetadataMissing is an inconsistency where a node is missing metadata + InconsistencyMetadataMissing Inconsistency = "metadata missing" + // InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink + InconsistencySymlinkMissing Inconsistency = "symlink missing" + // InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock + InconsistencyFilesMissing Inconsistency = "files missing" + // InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file + InconsistencyMalformedFile Inconsistency = "malformed file" + + // regex to determine if a node is trashed or versioned. + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z + _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) + // 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink + _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) +) + +// Consistency holds the node and blob data of a storage provider +type Consistency struct { + // Storing the data like this might take a lot of memory + // we might need to optimize this if we run into memory issues + Nodes map[string][]Inconsistency + LinkedNodes map[string][]Inconsistency + BlobReferences map[string][]Inconsistency + Blobs map[string][]Inconsistency + + nodeToLink map[string]string + blobToNode map[string]string +} + +// NewConsistency creates a new Consistency object +func NewConsistency() *Consistency { + return &Consistency{ + Nodes: make(map[string][]Inconsistency), + LinkedNodes: make(map[string][]Inconsistency), + BlobReferences: make(map[string][]Inconsistency), + Blobs: make(map[string][]Inconsistency), + + nodeToLink: make(map[string]string), + blobToNode: make(map[string]string), + } +} + +// CheckProviderConsistency checks the consistency of a space +func CheckProviderConsistency(storagepath string, lbs ListBlobstore) error { + fsys := os.DirFS(storagepath) + + p := NewProvider(fsys, storagepath, lbs) + if err := p.ProduceData(); err != nil { + return err + } + + c := NewConsistency() + c.GatherData(p.Events) + + return c.PrintResults(storagepath) +} + +// GatherData gathers and evaluates data produced by the DataProvider +func (c *Consistency) GatherData(events <-chan interface{}) { + for ev := range events { + switch d := ev.(type) { + case NodeData: + // does it have inconsistencies? + if len(d.Inconsistencies) != 0 { + c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...) + } + // is it linked? + if _, ok := c.LinkedNodes[d.NodePath]; ok { + deleteInconsistency(c.LinkedNodes, d.NodePath) + } else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil { + c.Nodes[d.NodePath] = []Inconsistency{} + } + // does it have a blob? + if d.BlobPath != "" { + if _, ok := c.Blobs[d.BlobPath]; ok { + deleteInconsistency(c.Blobs, d.BlobPath) + } else { + c.BlobReferences[d.BlobPath] = []Inconsistency{} + c.blobToNode[d.BlobPath] = d.NodePath + } + } + case LinkData: + // does it have a node? + if _, ok := c.Nodes[d.NodePath]; ok { + deleteInconsistency(c.Nodes, d.NodePath) + } else { + c.LinkedNodes[d.NodePath] = []Inconsistency{} + c.nodeToLink[d.NodePath] = d.LinkPath + } + case BlobData: + // does it have a reference? + if _, ok := c.BlobReferences[d.BlobPath]; ok { + deleteInconsistency(c.BlobReferences, d.BlobPath) + } else { + c.Blobs[d.BlobPath] = []Inconsistency{} + } + + } + } + + for n := range c.Nodes { + if len(c.Nodes[n]) == 0 { + c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) + } + } + for l := range c.LinkedNodes { + c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) + } + for b := range c.Blobs { + c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) + } + for b := range c.BlobReferences { + c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) + } +} + +// PrintResults prints the results of the evaluation +func (c *Consistency) PrintResults(discpath string) error { + if len(c.Nodes) != 0 { + fmt.Println("\n🚨 Inconsistent Nodes:") + } + for n := range c.Nodes { + fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n) + } + if len(c.LinkedNodes) != 0 { + fmt.Println("\n🚨 Inconsistent Links:") + } + for l := range c.LinkedNodes { + fmt.Printf("\t👉️ %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l) + } + if len(c.Blobs) != 0 { + fmt.Println("\n🚨 Inconsistent Blobs:") + } + for b := range c.Blobs { + fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b) + } + if len(c.BlobReferences) != 0 { + fmt.Println("\n🚨 Inconsistent BlobReferences:") + } + for b := range c.BlobReferences { + fmt.Printf("\t👉️ %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b]) + } + if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { + fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", discpath) + } + return nil + +} + +func deleteInconsistency(incs map[string][]Inconsistency, path string) { + if len(incs[path]) == 0 { + delete(incs, path) + } +} diff --git a/ocis/pkg/backup/backup_test.go b/ocis/pkg/backup/backup_test.go new file mode 100644 index 000000000..21e20842d --- /dev/null +++ b/ocis/pkg/backup/backup_test.go @@ -0,0 +1,162 @@ +package backup_test + +import ( + "testing" + + "github.com/owncloud/ocis/v2/ocis/pkg/backup" + "github.com/test-go/testify/require" +) + +func TestGatherData(t *testing.T) { + testcases := []struct { + Name string + Events []interface{} + Expected *backup.Consistency + }{ + { + Name: "no symlinks - no blobs", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencySymlinkMissing) + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + { + Name: "symlink not required - no blobs", + Events: []interface{}{ + nodeData("nodepath", "blobpath", false), + }, + Expected: consistency(func(c *backup.Consistency) { + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + { + Name: "no inconsistencies", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + }), + }, + { + Name: "orphaned blob", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + blobData("anotherpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + blob(c, "anotherpath", backup.InconsistencyBlobOrphaned) + }), + }, + { + Name: "missing node", + Events: []interface{}{ + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + linkedNode(c, "nodepath", backup.InconsistencyNodeMissing) + blob(c, "blobpath", backup.InconsistencyBlobOrphaned) + }), + }, + { + Name: "corrupt metadata", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing), + linkData("linkpath", "nodepath"), + blobData("blobpath"), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencyMetadataMissing) + }), + }, + { + Name: "corrupt metadata, no blob", + Events: []interface{}{ + nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing), + linkData("linkpath", "nodepath"), + }, + Expected: consistency(func(c *backup.Consistency) { + node(c, "nodepath", backup.InconsistencyMetadataMissing) + blobReference(c, "blobpath", backup.InconsistencyBlobMissing) + }), + }, + } + + for _, tc := range testcases { + events := make(chan interface{}) + + go func() { + for _, ev := range tc.Events { + switch e := ev.(type) { + case backup.NodeData: + events <- e + case backup.LinkData: + events <- e + case backup.BlobData: + events <- e + } + } + close(events) + }() + + c := backup.NewConsistency() + c.GatherData(events) + + require.Equal(t, tc.Expected.Nodes, c.Nodes) + require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes) + require.Equal(t, tc.Expected.Blobs, c.Blobs) + require.Equal(t, tc.Expected.BlobReferences, c.BlobReferences) + } + +} + +func nodeData(nodePath, blobPath string, requiresSymlink bool, incs ...backup.Inconsistency) backup.NodeData { + return backup.NodeData{ + NodePath: nodePath, + BlobPath: blobPath, + RequiresSymlink: requiresSymlink, + Inconsistencies: incs, + } +} + +func linkData(linkPath, nodePath string) backup.LinkData { + return backup.LinkData{ + LinkPath: linkPath, + NodePath: nodePath, + } +} + +func blobData(blobPath string) backup.BlobData { + return backup.BlobData{ + BlobPath: blobPath, + } +} + +func consistency(f func(*backup.Consistency)) *backup.Consistency { + c := backup.NewConsistency() + f(c) + return c +} + +func node(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.Nodes[path] = inc +} + +func linkedNode(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.LinkedNodes[path] = inc +} + +func blob(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.Blobs[path] = inc +} + +func blobReference(c *backup.Consistency, path string, inc ...backup.Inconsistency) { + c.BlobReferences[path] = inc +} diff --git a/ocis/pkg/backup/provider.go b/ocis/pkg/backup/provider.go new file mode 100644 index 000000000..02d4fbec8 --- /dev/null +++ b/ocis/pkg/backup/provider.go @@ -0,0 +1,237 @@ +package backup + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +// ListBlobstore required to check blob consistency +type ListBlobstore interface { + List() ([]*node.Node, error) + Path(node *node.Node) string +} + +// DataProvider provides data for the consistency check +type DataProvider struct { + Events chan interface{} + + fsys fs.FS + discpath string + lbs ListBlobstore + skipBlobs bool +} + +// NodeData holds data about the nodes +type NodeData struct { + NodePath string + BlobPath string + RequiresSymlink bool + Inconsistencies []Inconsistency +} + +// LinkData about the symlinks +type LinkData struct { + LinkPath string + NodePath string +} + +// BlobData about the blobs in the blobstore +type BlobData struct { + BlobPath string +} + +// NewProvider creates a new DataProvider object +func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { + return &DataProvider{ + Events: make(chan interface{}), + + fsys: fsys, + discpath: discpath, + lbs: lbs, + skipBlobs: lbs == nil, + } +} + +// ProduceData produces data for the consistency check +// Spawns 4 go-routines at the moment. If needed, this can be optimized. +func (dp *DataProvider) ProduceData() error { + dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*") + if err != nil { + return err + } + + if len(dirs) == 0 { + return errors.New("no backup found. Double check storage path") + } + + wg := sync.WaitGroup{} + + // crawl spaces + wg.Add(1) + go func() { + for _, d := range dirs { + dp.evaluateNodeDir(d) + } + wg.Done() + }() + + // crawl trash + wg.Add(1) + go func() { + dp.evaluateTrashDir() + wg.Done() + }() + + // crawl blobstore + if !dp.skipBlobs { + wg.Add(1) + go func() { + bs, err := dp.lbs.List() + if err != nil { + fmt.Println("error listing blobs", err) + } + + for _, bn := range bs { + dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)} + } + wg.Done() + }() + } + + // wait for all crawlers to finish + go func() { + wg.Wait() + dp.quit() + }() + + return nil +} + +func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) { + if dp.skipBlobs { + return "", "" + } + + b, err := fs.ReadFile(dp.fsys, path+".mpk") + if err != nil { + return "", InconsistencyFilesMissing + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", InconsistencyMalformedFile + } + + // FIXME: how to check if metadata is complete? + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path)) + return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), "" + } + + return "", "" +} + +func (dp *DataProvider) evaluateNodeDir(d string) { + // d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24 + // we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension + entries, err := fs.ReadDir(dp.fsys, d) + if err != nil { + fmt.Println("error reading dir", err) + return + } + + if len(entries) == 0 { + fmt.Println("empty dir", filepath.Join(dp.discpath, d)) + return + } + + for _, e := range entries { + switch { + case e.IsDir(): + ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + for _, l := range ls { + linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name()) + + r, _ := os.Readlink(linkpath) + nodePath := filepath.Join(dp.discpath, d, e.Name(), r) + dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath} + } + fallthrough + case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): + np := filepath.Join(dp.discpath, d, e.Name()) + var inc []Inconsistency + if !dp.filesExist(filepath.Join(d, e.Name())) { + inc = append(inc, InconsistencyFilesMissing) + } + bp, i := dp.getBlobPath(filepath.Join(d, e.Name())) + if i != "" { + inc = append(inc, i) + } + + dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc} + } + } +} + +func (dp *DataProvider) evaluateTrashDir() { + linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*") + if err != nil { + fmt.Println("error reading trash", err) + } + for _, l := range linkpaths { + linkpath := filepath.Join(dp.discpath, l) + r, _ := os.Readlink(linkpath) + p := filepath.Join(dp.discpath, l, "..", r) + dp.Events <- LinkData{LinkPath: linkpath, NodePath: p} + } +} + +func (dp *DataProvider) filesExist(path string) bool { + check := func(p string) bool { + _, err := fs.Stat(dp.fsys, p) + return err == nil + } + return check(path) && check(path+".mpk") +} + +func (dp *DataProvider) quit() { + close(dp.Events) +} + +func requiresSymlink(path string) bool { + spaceID, nodeID := getIDsFromPath(path) + if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { + return false + } + + return true +} + +func getIDsFromPath(path string) (string, string) { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return "", "" + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return "", "" + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(rawIDs[1], "/", "", -1) + return spaceID, nodeID +} diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go new file mode 100644 index 000000000..d985a020a --- /dev/null +++ b/ocis/pkg/command/backup.go @@ -0,0 +1,98 @@ +package command + +import ( + "errors" + "fmt" + + ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" + "github.com/owncloud/ocis/v2/ocis/pkg/backup" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/urfave/cli/v2" +) + +// BackupCommand is the entrypoint for the backup command +func BackupCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "backup", + Usage: "ocis backup functionality", + Subcommands: []*cli.Command{ + ConsistencyCommand(cfg), + }, + Before: func(c *cli.Context) error { + return configlog.ReturnError(parser.ParseConfig(cfg, true)) + }, + Action: func(_ *cli.Context) error { + fmt.Println("Read the docs") + return nil + }, + } +} + +// ConsistencyCommand is the entrypoint for the consistency Command +func ConsistencyCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "consistency", + Usage: "check backup consistency", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "basepath", + Aliases: []string{"p"}, + Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)", + Required: true, + }, + &cli.StringFlag{ + Name: "blobstore", + Aliases: []string{"b"}, + Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", + Value: "ocis", + }, + }, + Action: func(c *cli.Context) error { + basePath := c.String("basepath") + if basePath == "" { + fmt.Println("basepath is required") + return cli.ShowCommandHelp(c, "consistency") + } + + var ( + bs backup.ListBlobstore + err error + ) + switch c.String("blobstore") { + case "s3ng": + bs, err = s3bs.New( + cfg.StorageUsers.Drivers.S3NG.Endpoint, + cfg.StorageUsers.Drivers.S3NG.Region, + cfg.StorageUsers.Drivers.S3NG.Bucket, + cfg.StorageUsers.Drivers.S3NG.AccessKey, + cfg.StorageUsers.Drivers.S3NG.SecretKey, + s3bs.Options{}, + ) + case "ocis": + bs, err = ocisbs.New(basePath) + case "none": + bs = nil + default: + err = errors.New("blobstore type not supported") + } + if err != nil { + fmt.Println(err) + return err + } + if err := backup.CheckProviderConsistency(basePath, bs); err != nil { + fmt.Println(err) + return err + } + + return nil + }, + } +} + +func init() { + register.AddCommand(BackupCommand) +}