sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ocis): finally unit tests for backup consistency

Browse Source 
Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
2024-06-06 15:55:09 +02:00
parent e96921708e
commit d655c8140a
3 changed files with 257 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ocis/pkg/backup/provider.go
+82 -34
View File
@@ -6,6 +6,7 @@ import (
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
@@ -20,6 +21,11 @@ type ListBlobstore interface {
// DataProvider provides data for the consistency check
type DataProvider struct {
Nodes chan NodeData
Links chan LinkData
Blobs chan BlobData
Quit chan struct{}
fsys fs.FS
discpath string
lbs ListBlobstore
@@ -29,6 +35,7 @@ type DataProvider struct {
type NodeData struct {
NodePath string
BlobPath string
RequiresSymlink bool
Inconsistencies []Inconsistency
}
@@ -46,6 +53,11 @@ type BlobData struct {
// NewProvider creates a new DataProvider object
func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider {
return &DataProvider{
Nodes: make(chan NodeData),
Links: make(chan LinkData),
Blobs: make(chan BlobData),
Quit: make(chan struct{}),
fsys: fsys,
discpath: discpath,
lbs: lbs,
@@ -53,65 +65,62 @@ func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider {
}
// ProduceData produces data for the consistency check
func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobData, chan struct{}, error) {
dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*")
// Spawns 4 go-routines at the moment. If needed, this can be optimized.
func (dp *DataProvider) ProduceData() error {
dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*")
if err != nil {
return nil, nil, nil, nil, err
return err
}
if len(dirs) == 0 {
return nil, nil, nil, nil, errors.New("no backup found. Double check storage path")
return errors.New("no backup found. Double check storage path")
}
nodes := make(chan NodeData)
links := make(chan LinkData)
blobs := make(chan BlobData)
quit := make(chan struct{})
wg := sync.WaitGroup{}
// crawl spaces
wg.Add(1)
go func() {
for _, d := range dirs {
entries, err := fs.ReadDir(c.fsys, d)
entries, err := fs.ReadDir(dp.fsys, d)
if err != nil {
fmt.Println("error reading dir", err)
continue
}
if len(entries) == 0 {
fmt.Println("empty dir", filepath.Join(c.discpath, d))
fmt.Println("empty dir", filepath.Join(dp.discpath, d))
continue
}
for _, e := range entries {
switch {
case e.IsDir():
ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name()))
ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name()))
if err != nil {
fmt.Println("error reading dir", err)
continue
}
for _, l := range ls {
linkpath := filepath.Join(c.discpath, d, e.Name(), l.Name())
linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name())
r, _ := os.Readlink(linkpath)
nodePath := filepath.Join(c.discpath, d, e.Name(), r)
links <- LinkData{LinkPath: linkpath, NodePath: nodePath}
nodePath := filepath.Join(dp.discpath, d, e.Name(), r)
dp.Links <- LinkData{LinkPath: linkpath, NodePath: nodePath}
}
fallthrough
case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()):
np := filepath.Join(c.discpath, d, e.Name())
np := filepath.Join(dp.discpath, d, e.Name())
var inc []Inconsistency
if !c.filesExist(filepath.Join(d, e.Name())) {
if !dp.filesExist(filepath.Join(d, e.Name())) {
inc = append(inc, InconsistencyFilesMissing)
}
bp, i := c.getBlobPath(filepath.Join(d, e.Name()))
bp, i := dp.getBlobPath(filepath.Join(d, e.Name()))
if i != "" {
inc = append(inc, i)
}
nodes <- NodeData{NodePath: np, BlobPath: bp, Inconsistencies: inc}
dp.Nodes <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc}
}
}
}
@@ -121,15 +130,15 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat
// crawl trash
wg.Add(1)
go func() {
linkpaths, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*")
linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*")
if err != nil {
fmt.Println("error reading trash", err)
}
for _, l := range linkpaths {
linkpath := filepath.Join(c.discpath, l)
linkpath := filepath.Join(dp.discpath, l)
r, _ := os.Readlink(linkpath)
p := filepath.Join(c.discpath, l, "..", r)
links <- LinkData{LinkPath: linkpath, NodePath: p}
p := filepath.Join(dp.discpath, l, "..", r)
dp.Links <- LinkData{LinkPath: linkpath, NodePath: p}
}
wg.Done()
}()
@@ -137,13 +146,13 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat
// crawl blobstore
wg.Add(1)
go func() {
bs, err := c.lbs.List()
bs, err := dp.lbs.List()
if err != nil {
fmt.Println("error listing blobs", err)
}
for _, bn := range bs {
blobs <- BlobData{BlobPath: c.lbs.Path(bn)}
dp.Blobs <- BlobData{BlobPath: dp.lbs.Path(bn)}
}
wg.Done()
}()
@@ -151,18 +160,14 @@ func (c *DataProvider) ProduceData() (chan NodeData, chan LinkData, chan BlobDat
// wait for all crawlers to finish
go func() {
wg.Wait()
quit <- struct{}{}
close(nodes)
close(links)
close(blobs)
close(quit)
dp.quit()
}()
return nodes, links, blobs, quit, nil
return nil
}
func (c *DataProvider) getBlobPath(path string) (string, Inconsistency) {
b, err := fs.ReadFile(c.fsys, path+".mpk")
func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
b, err := fs.ReadFile(dp.fsys, path+".mpk")
if err != nil {
return "", InconsistencyFilesMissing
}
@@ -172,10 +177,53 @@ func (c *DataProvider) getBlobPath(path string) (string, Inconsistency) {
return "", InconsistencyMalformedFile
}
// FIXME: how to check if metadata is complete?
if bid := m["user.ocis.blobid"]; string(bid) != "" {
spaceID, _ := getIDsFromPath(filepath.Join(c.discpath, path))
return c.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), ""
spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), ""
}
return "", ""
}
func (dp *DataProvider) filesExist(path string) bool {
check := func(p string) bool {
_, err := fs.Stat(dp.fsys, p)
return err == nil
}
return check(path) && check(path+".mpk")
}
func (dp *DataProvider) quit() {
dp.Quit <- struct{}{}
close(dp.Nodes)
close(dp.Links)
close(dp.Blobs)
close(dp.Quit)
}
func requiresSymlink(path string) bool {
spaceID, nodeID := getIDsFromPath(path)
if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) {
return false
}
return true
}
func getIDsFromPath(path string) (string, string) {
rawIDs := strings.Split(path, "/nodes/")
if len(rawIDs) != 2 {
return "", ""
}
s := strings.Split(rawIDs[0], "/spaces/")
if len(s) != 2 {
return "", ""
}
spaceID := strings.Replace(s[1], "/", "", -1)
nodeID := strings.Replace(rawIDs[1], "/", "", -1)
return spaceID, nodeID
}