Initial QSfera import

Server/qsfera/pkg/backup/backup.go

@@ -0,0 +1,175 @@
+// Package backup contains КуСфера backup functionality.
+package backup
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+)
+
+// Inconsistency describes the type of inconsistency
+type Inconsistency string
+
+var (
+	// InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore
+	InconsistencyBlobMissing Inconsistency = "blob missing"
+	// InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference
+	InconsistencyBlobOrphaned Inconsistency = "blob orphaned"
+	// InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node
+	InconsistencyNodeMissing Inconsistency = "node missing"
+	// InconsistencyMetadataMissing is an inconsistency where a node is missing metadata
+	InconsistencyMetadataMissing Inconsistency = "metadata missing"
+	// InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink
+	InconsistencySymlinkMissing Inconsistency = "symlink missing"
+	// InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock
+	InconsistencyFilesMissing Inconsistency = "files missing"
+	// InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file
+	InconsistencyMalformedFile Inconsistency = "malformed file"
+
+	// regex to determine if a node is trashed or versioned.
+	// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
+	_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z(\.\d+)?$`)
+	//   9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink
+	_trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$`)
+)
+
+// Consistency holds the node and blob data of a storage provider
+type Consistency struct {
+	// Storing the data like this might take a lot of memory
+	// we might need to optimize this if we run into memory issues
+	Nodes          map[string][]Inconsistency
+	LinkedNodes    map[string][]Inconsistency
+	BlobReferences map[string][]Inconsistency
+	Blobs          map[string][]Inconsistency
+
+	nodeToLink map[string]string
+	blobToNode map[string]string
+}
+
+// NewConsistency creates a new Consistency object
+func NewConsistency() *Consistency {
+	return &Consistency{
+		Nodes:          make(map[string][]Inconsistency),
+		LinkedNodes:    make(map[string][]Inconsistency),
+		BlobReferences: make(map[string][]Inconsistency),
+		Blobs:          make(map[string][]Inconsistency),
+
+		nodeToLink: make(map[string]string),
+		blobToNode: make(map[string]string),
+	}
+}
+
+// CheckProviderConsistency checks the consistency of a space
+func CheckProviderConsistency(storagepath string, lbs ListBlobstore, fail bool) error {
+	fsys := os.DirFS(storagepath)
+
+	p := NewProvider(fsys, storagepath, lbs)
+	if err := p.ProduceData(); err != nil {
+		return err
+	}
+
+	c := NewConsistency()
+	c.GatherData(p.Events)
+
+	return c.PrintResults(storagepath, fail)
+}
+
+// GatherData gathers and evaluates data produced by the DataProvider
+func (c *Consistency) GatherData(events <-chan any) {
+	for ev := range events {
+		switch d := ev.(type) {
+		case NodeData:
+			// does it have inconsistencies?
+			if len(d.Inconsistencies) != 0 {
+				c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...)
+			}
+			// is it linked?
+			if _, ok := c.LinkedNodes[d.NodePath]; ok {
+				deleteInconsistency(c.LinkedNodes, d.NodePath)
+			} else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil {
+				c.Nodes[d.NodePath] = []Inconsistency{}
+			}
+			// does it have a blob?
+			if d.BlobPath != "" {
+				if _, ok := c.Blobs[d.BlobPath]; ok {
+					deleteInconsistency(c.Blobs, d.BlobPath)
+				} else {
+					c.BlobReferences[d.BlobPath] = []Inconsistency{}
+					c.blobToNode[d.BlobPath] = d.NodePath
+				}
+			}
+		case LinkData:
+			// does it have a node?
+			if _, ok := c.Nodes[d.NodePath]; ok {
+				deleteInconsistency(c.Nodes, d.NodePath)
+			} else {
+				c.LinkedNodes[d.NodePath] = []Inconsistency{}
+				c.nodeToLink[d.NodePath] = d.LinkPath
+			}
+		case BlobData:
+			// does it have a reference?
+			if _, ok := c.BlobReferences[d.BlobPath]; ok {
+				deleteInconsistency(c.BlobReferences, d.BlobPath)
+			} else {
+				c.Blobs[d.BlobPath] = []Inconsistency{}
+			}
+
+		}
+	}
+
+	for n := range c.Nodes {
+		if len(c.Nodes[n]) == 0 {
+			c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing)
+		}
+	}
+	for l := range c.LinkedNodes {
+		c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing)
+	}
+	for b := range c.Blobs {
+		c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned)
+	}
+	for b := range c.BlobReferences {
+		c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing)
+	}
+}
+
+// PrintResults prints the results of the evaluation
+func (c *Consistency) PrintResults(discpath string, fail bool) error {
+	if len(c.Nodes) != 0 {
+		fmt.Println("\n🚨 Inconsistent Nodes:")
+	}
+	for n := range c.Nodes {
+		fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n)
+	}
+	if len(c.LinkedNodes) != 0 {
+		fmt.Println("\n🚨 Inconsistent Links:")
+	}
+	for l := range c.LinkedNodes {
+		fmt.Printf("\t👉️ %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l)
+	}
+	if len(c.Blobs) != 0 {
+		fmt.Println("\n🚨 Inconsistent Blobs:")
+	}
+	for b := range c.Blobs {
+		fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b)
+	}
+	if len(c.BlobReferences) != 0 {
+		fmt.Println("\n🚨 Inconsistent BlobReferences:")
+	}
+	for b := range c.BlobReferences {
+		fmt.Printf("\t👉️ %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b])
+	}
+	if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 {
+		fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", discpath)
+	} else if fail {
+		os.Exit(1)
+	}
+	return nil
+
+}
+
+func deleteInconsistency(incs map[string][]Inconsistency, path string) {
+	if len(incs[path]) == 0 {
+		delete(incs, path)
+	}
+}

Server/qsfera/pkg/backup/backup_test.go

@@ -0,0 +1,162 @@
+package backup_test
+
+import (
+	"testing"
+
+	"github.com/qsfera/server/qsfera/pkg/backup"
+	"github.com/test-go/testify/require"
+)
+
+func TestGatherData(t *testing.T) {
+	testcases := []struct {
+		Name     string
+		Events   []any
+		Expected *backup.Consistency
+	}{
+		{
+			Name: "no symlinks - no blobs",
+			Events: []any{
+				nodeData("nodepath", "blobpath", true),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				node(c, "nodepath", backup.InconsistencySymlinkMissing)
+				blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
+			}),
+		},
+		{
+			Name: "symlink not required - no blobs",
+			Events: []any{
+				nodeData("nodepath", "blobpath", false),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
+			}),
+		},
+		{
+			Name: "no inconsistencies",
+			Events: []any{
+				nodeData("nodepath", "blobpath", true),
+				linkData("linkpath", "nodepath"),
+				blobData("blobpath"),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+			}),
+		},
+		{
+			Name: "orphaned blob",
+			Events: []any{
+				nodeData("nodepath", "blobpath", true),
+				linkData("linkpath", "nodepath"),
+				blobData("blobpath"),
+				blobData("anotherpath"),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				blob(c, "anotherpath", backup.InconsistencyBlobOrphaned)
+			}),
+		},
+		{
+			Name: "missing node",
+			Events: []any{
+				linkData("linkpath", "nodepath"),
+				blobData("blobpath"),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				linkedNode(c, "nodepath", backup.InconsistencyNodeMissing)
+				blob(c, "blobpath", backup.InconsistencyBlobOrphaned)
+			}),
+		},
+		{
+			Name: "corrupt metadata",
+			Events: []any{
+				nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
+				linkData("linkpath", "nodepath"),
+				blobData("blobpath"),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				node(c, "nodepath", backup.InconsistencyMetadataMissing)
+			}),
+		},
+		{
+			Name: "corrupt metadata, no blob",
+			Events: []any{
+				nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
+				linkData("linkpath", "nodepath"),
+			},
+			Expected: consistency(func(c *backup.Consistency) {
+				node(c, "nodepath", backup.InconsistencyMetadataMissing)
+				blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
+			}),
+		},
+	}
+
+	for _, tc := range testcases {
+		events := make(chan any)
+
+		go func() {
+			for _, ev := range tc.Events {
+				switch e := ev.(type) {
+				case backup.NodeData:
+					events <- e
+				case backup.LinkData:
+					events <- e
+				case backup.BlobData:
+					events <- e
+				}
+			}
+			close(events)
+		}()
+
+		c := backup.NewConsistency()
+		c.GatherData(events)
+
+		require.Equal(t, tc.Expected.Nodes, c.Nodes)
+		require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes)
+		require.Equal(t, tc.Expected.Blobs, c.Blobs)
+		require.Equal(t, tc.Expected.BlobReferences, c.BlobReferences)
+	}
+
+}
+
+func nodeData(nodePath, blobPath string, requiresSymlink bool, incs ...backup.Inconsistency) backup.NodeData {
+	return backup.NodeData{
+		NodePath:        nodePath,
+		BlobPath:        blobPath,
+		RequiresSymlink: requiresSymlink,
+		Inconsistencies: incs,
+	}
+}
+
+func linkData(linkPath, nodePath string) backup.LinkData {
+	return backup.LinkData{
+		LinkPath: linkPath,
+		NodePath: nodePath,
+	}
+}
+
+func blobData(blobPath string) backup.BlobData {
+	return backup.BlobData{
+		BlobPath: blobPath,
+	}
+}
+
+func consistency(f func(*backup.Consistency)) *backup.Consistency {
+	c := backup.NewConsistency()
+	f(c)
+	return c
+}
+
+func node(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
+	c.Nodes[path] = inc
+}
+
+func linkedNode(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
+	c.LinkedNodes[path] = inc
+}
+
+func blob(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
+	c.Blobs[path] = inc
+}
+
+func blobReference(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
+	c.BlobReferences[path] = inc
+}

Server/qsfera/pkg/backup/provider.go

@@ -0,0 +1,231 @@
+package backup
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+// ListBlobstore required to check blob consistency
+type ListBlobstore interface {
+	List() ([]*node.Node, error)
+	Path(node *node.Node) string
+}
+
+// DataProvider provides data for the consistency check
+type DataProvider struct {
+	Events chan any
+
+	fsys      fs.FS
+	discpath  string
+	lbs       ListBlobstore
+	skipBlobs bool
+}
+
+// NodeData holds data about the nodes
+type NodeData struct {
+	NodePath        string
+	BlobPath        string
+	RequiresSymlink bool
+	Inconsistencies []Inconsistency
+}
+
+// LinkData about the symlinks
+type LinkData struct {
+	LinkPath string
+	NodePath string
+}
+
+// BlobData about the blobs in the blobstore
+type BlobData struct {
+	BlobPath string
+}
+
+// NewProvider creates a new DataProvider object
+func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider {
+	return &DataProvider{
+		Events: make(chan any),
+
+		fsys:      fsys,
+		discpath:  discpath,
+		lbs:       lbs,
+		skipBlobs: lbs == nil,
+	}
+}
+
+// ProduceData produces data for the consistency check
+// Spawns 4 go-routines at the moment. If needed, this can be optimized.
+func (dp *DataProvider) ProduceData() error {
+	dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*")
+	if err != nil {
+		return err
+	}
+
+	if len(dirs) == 0 {
+		return errors.New("no backup found. Double check storage path")
+	}
+
+	wg := sync.WaitGroup{}
+
+	// crawl spaces
+	wg.Go(func() {
+		for _, d := range dirs {
+			dp.evaluateNodeDir(d)
+		}
+	})
+
+	// crawl trash
+	wg.Go(func() {
+		dp.evaluateTrashDir()
+	})
+
+	// crawl blobstore
+	if !dp.skipBlobs {
+		wg.Go(func() {
+			bs, err := dp.lbs.List()
+			if err != nil {
+				fmt.Println("error listing blobs", err)
+			}
+
+			for _, bn := range bs {
+				dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)}
+			}
+		})
+	}
+
+	// wait for all crawlers to finish
+	go func() {
+		wg.Wait()
+		dp.quit()
+	}()
+
+	return nil
+}
+
+func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
+	if dp.skipBlobs {
+		return "", ""
+	}
+
+	b, err := fs.ReadFile(dp.fsys, path+".mpk")
+	if err != nil {
+		return "", InconsistencyFilesMissing
+	}
+
+	m := map[string][]byte{}
+	if err := msgpack.Unmarshal(b, &m); err != nil {
+		return "", InconsistencyMalformedFile
+	}
+
+	// FIXME: how to check if metadata is complete?
+
+	if bid := m["user.oc.blobid"]; string(bid) != "" {
+		spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
+		return dp.lbs.Path(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: string(bid)}), ""
+	}
+
+	return "", ""
+}
+
+func (dp *DataProvider) evaluateNodeDir(d string) {
+	// d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24
+	// we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension
+	entries, err := fs.ReadDir(dp.fsys, d)
+	if err != nil {
+		fmt.Println("error reading dir", err)
+		return
+	}
+
+	if len(entries) == 0 {
+		fmt.Println("empty dir", filepath.Join(dp.discpath, d))
+		return
+	}
+
+	for _, e := range entries {
+		switch {
+		case e.IsDir():
+			ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name()))
+			if err != nil {
+				fmt.Println("error reading dir", err)
+				continue
+			}
+			for _, l := range ls {
+				linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name())
+
+				r, _ := os.Readlink(linkpath)
+				nodePath := filepath.Join(dp.discpath, d, e.Name(), r)
+				dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath}
+			}
+			fallthrough
+		case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()):
+			np := filepath.Join(dp.discpath, d, e.Name())
+			var inc []Inconsistency
+			if !dp.filesExist(filepath.Join(d, e.Name())) {
+				inc = append(inc, InconsistencyFilesMissing)
+			}
+			bp, i := dp.getBlobPath(filepath.Join(d, e.Name()))
+			if i != "" {
+				inc = append(inc, i)
+			}
+
+			dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc}
+		}
+	}
+}
+
+func (dp *DataProvider) evaluateTrashDir() {
+	linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*")
+	if err != nil {
+		fmt.Println("error reading trash", err)
+	}
+	for _, l := range linkpaths {
+		linkpath := filepath.Join(dp.discpath, l)
+		r, _ := os.Readlink(linkpath)
+		p := filepath.Join(dp.discpath, l, "..", r)
+		dp.Events <- LinkData{LinkPath: linkpath, NodePath: p}
+	}
+}
+
+func (dp *DataProvider) filesExist(path string) bool {
+	check := func(p string) bool {
+		_, err := fs.Stat(dp.fsys, p)
+		return err == nil
+	}
+	return check(path) && check(path+".mpk")
+}
+
+func (dp *DataProvider) quit() {
+	close(dp.Events)
+}
+
+func requiresSymlink(path string) bool {
+	spaceID, nodeID := getIDsFromPath(path)
+	if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) {
+		return false
+	}
+
+	return true
+}
+
+func getIDsFromPath(path string) (string, string) {
+	rawIDs := strings.Split(path, "/nodes/")
+	if len(rawIDs) != 2 {
+		return "", ""
+	}
+
+	s := strings.Split(rawIDs[0], "/spaces/")
+	if len(s) != 2 {
+		return "", ""
+	}
+
+	spaceID := strings.Replace(s[1], "/", "", -1)
+	nodeID := strings.Replace(rawIDs[1], "/", "", -1)
+	return spaceID, nodeID
+}