package backup import ( "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "sync" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/vmihailenco/msgpack/v5" ) // ListBlobstore required to check blob consistency type ListBlobstore interface { List() ([]*node.Node, error) Path(node *node.Node) string } // DataProvider provides data for the consistency check type DataProvider struct { Events chan any fsys fs.FS discpath string lbs ListBlobstore skipBlobs bool } // NodeData holds data about the nodes type NodeData struct { NodePath string BlobPath string RequiresSymlink bool Inconsistencies []Inconsistency } // LinkData about the symlinks type LinkData struct { LinkPath string NodePath string } // BlobData about the blobs in the blobstore type BlobData struct { BlobPath string } // NewProvider creates a new DataProvider object func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider { return &DataProvider{ Events: make(chan any), fsys: fsys, discpath: discpath, lbs: lbs, skipBlobs: lbs == nil, } } // ProduceData produces data for the consistency check // Spawns 4 go-routines at the moment. If needed, this can be optimized. func (dp *DataProvider) ProduceData() error { dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { return err } if len(dirs) == 0 { return errors.New("no backup found. Double check storage path") } wg := sync.WaitGroup{} // crawl spaces wg.Go(func() { for _, d := range dirs { dp.evaluateNodeDir(d) } }) // crawl trash wg.Go(func() { dp.evaluateTrashDir() }) // crawl blobstore if !dp.skipBlobs { wg.Go(func() { bs, err := dp.lbs.List() if err != nil { fmt.Println("error listing blobs", err) } for _, bn := range bs { dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)} } }) } // wait for all crawlers to finish go func() { wg.Wait() dp.quit() }() return nil } func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) { if dp.skipBlobs { return "", "" } b, err := fs.ReadFile(dp.fsys, path+".mpk") if err != nil { return "", InconsistencyFilesMissing } m := map[string][]byte{} if err := msgpack.Unmarshal(b, &m); err != nil { return "", InconsistencyMalformedFile } // FIXME: how to check if metadata is complete? if bid := m["user.oc.blobid"]; string(bid) != "" { spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path)) return dp.lbs.Path(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: string(bid)}), "" } return "", "" } func (dp *DataProvider) evaluateNodeDir(d string) { // d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24 // we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension entries, err := fs.ReadDir(dp.fsys, d) if err != nil { fmt.Println("error reading dir", err) return } if len(entries) == 0 { fmt.Println("empty dir", filepath.Join(dp.discpath, d)) return } for _, e := range entries { switch { case e.IsDir(): ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name())) if err != nil { fmt.Println("error reading dir", err) continue } for _, l := range ls { linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name()) r, _ := os.Readlink(linkpath) nodePath := filepath.Join(dp.discpath, d, e.Name(), r) dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath} } fallthrough case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): np := filepath.Join(dp.discpath, d, e.Name()) var inc []Inconsistency if !dp.filesExist(filepath.Join(d, e.Name())) { inc = append(inc, InconsistencyFilesMissing) } bp, i := dp.getBlobPath(filepath.Join(d, e.Name())) if i != "" { inc = append(inc, i) } dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc} } } } func (dp *DataProvider) evaluateTrashDir() { linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*") if err != nil { fmt.Println("error reading trash", err) } for _, l := range linkpaths { linkpath := filepath.Join(dp.discpath, l) r, _ := os.Readlink(linkpath) p := filepath.Join(dp.discpath, l, "..", r) dp.Events <- LinkData{LinkPath: linkpath, NodePath: p} } } func (dp *DataProvider) filesExist(path string) bool { check := func(p string) bool { _, err := fs.Stat(dp.fsys, p) return err == nil } return check(path) && check(path+".mpk") } func (dp *DataProvider) quit() { close(dp.Events) } func requiresSymlink(path string) bool { spaceID, nodeID := getIDsFromPath(path) if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) { return false } return true } func getIDsFromPath(path string) (string, string) { rawIDs := strings.Split(path, "/nodes/") if len(rawIDs) != 2 { return "", "" } s := strings.Split(rawIDs[0], "/spaces/") if len(s) != 2 { return "", "" } spaceID := strings.Replace(s[1], "/", "", -1) nodeID := strings.Replace(rawIDs[1], "/", "", -1) return spaceID, nodeID }