Initial QSfera import
This commit is contained in:
Курнат Андрей
@@ -0,0 +1,284 @@
|
||||
// Package revisions allows manipulating revisions in a storage provider.
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
)
|
||||
|
||||
var (
|
||||
// regex to determine if a node versioned. Examples:
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock
|
||||
_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`)
|
||||
_spaceIDRegex = regexp.MustCompile(`/spaces/(.+)/nodes/`)
|
||||
)
|
||||
|
||||
// DelBlobstore is the interface for a blobstore that can delete blobs.
|
||||
type DelBlobstore interface {
|
||||
Delete(node *node.Node) error
|
||||
}
|
||||
|
||||
// Glob uses globbing to find all revision nodes in a storage provider.
|
||||
func Glob(pattern string) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
nodes, err := filepath.Glob(filepath.Join(pattern))
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", pattern, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
fmt.Println("no nodes found. Double check storage path")
|
||||
return
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
if _versionRegex.MatchString(n) {
|
||||
ch <- n
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// GlobWorkers uses multiple go routine to glob all revision nodes in a storage provider.
|
||||
func GlobWorkers(pattern string, depth string, remainder string) <-chan string {
|
||||
wg := sync.WaitGroup{}
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
nodes, err := filepath.Glob(pattern + depth)
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", pattern, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
fmt.Println("no nodes found. Double check storage path")
|
||||
return
|
||||
}
|
||||
|
||||
for _, node := range nodes {
|
||||
wg.Add(1)
|
||||
go func(node string) {
|
||||
defer wg.Done()
|
||||
nodes, err := filepath.Glob(node + remainder)
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", node, err)
|
||||
return
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if _versionRegex.MatchString(n) {
|
||||
ch <- n
|
||||
}
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Walk walks the storage provider to find all revision nodes.
|
||||
func Walk(base string) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if !_versionRegex.MatchString(info.Name()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
ch <- path
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return
|
||||
}
|
||||
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
// List uses directory listing to find all revision nodes in a storage provider.
|
||||
func List(base string, workers int) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
if err := listFolder(base, ch, make(chan struct{}, workers)); err != nil {
|
||||
fmt.Println("error listing", base, err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// PurgeRevisions removes all revisions from a storage provider.
|
||||
func PurgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) {
|
||||
countFiles := 0
|
||||
countBlobs := 0
|
||||
countRevisions := 0
|
||||
|
||||
var err error
|
||||
for d := range nodes {
|
||||
if !_versionRegex.MatchString(d) {
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
spaceID, blobID string
|
||||
)
|
||||
|
||||
e := filepath.Ext(d)
|
||||
switch e {
|
||||
case ".mpk":
|
||||
blobID, err = getBlobID(d)
|
||||
if err != nil {
|
||||
fmt.Printf("error getting blobID from %s: %v\n", d, err)
|
||||
continue
|
||||
}
|
||||
matches := _spaceIDRegex.FindStringSubmatch(d)
|
||||
if len(matches) != 2 {
|
||||
fmt.Printf("error extracting spaceID from %s\n", d)
|
||||
continue
|
||||
}
|
||||
spaceID = strings.ReplaceAll(matches[1], "/", "")
|
||||
|
||||
countBlobs++
|
||||
case ".mlock":
|
||||
// no extra action on .mlock files
|
||||
default:
|
||||
countRevisions++
|
||||
}
|
||||
|
||||
if !dryRun {
|
||||
if blobID != "" {
|
||||
// TODO: needs spaceID for decomposeds3
|
||||
if err := bs.Delete(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: blobID}); err != nil {
|
||||
fmt.Printf("error deleting blob %s: %v\n", blobID, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Remove(d); err != nil {
|
||||
fmt.Printf("error removing %s: %v\n", d, err)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
countFiles++
|
||||
|
||||
if verbose {
|
||||
spaceID, nodeID := getIDsFromPath(d)
|
||||
if dryRun {
|
||||
fmt.Println("Would delete")
|
||||
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
|
||||
fmt.Println("\tSpaceID:", spaceID)
|
||||
fmt.Println("\tPath:", d)
|
||||
if blobID != "" {
|
||||
fmt.Println("\tBlob:", blobID)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Deleted")
|
||||
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
|
||||
fmt.Println("\tSpaceID:", spaceID)
|
||||
fmt.Println("\tPath:", d)
|
||||
if blobID != "" {
|
||||
fmt.Println("\tBlob:", blobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return countFiles, countBlobs, countRevisions
|
||||
}
|
||||
|
||||
func listFolder(path string, ch chan<- string, workers chan struct{}) error {
|
||||
workers <- struct{}{}
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
children, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
<-workers
|
||||
return err
|
||||
}
|
||||
|
||||
for _, child := range children {
|
||||
if child.IsDir() {
|
||||
wg.Go(func() {
|
||||
if err := listFolder(filepath.Join(path, child.Name()), ch, workers); err != nil {
|
||||
fmt.Println("error listing", path, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if _versionRegex.MatchString(child.Name()) {
|
||||
ch <- filepath.Join(path, child.Name())
|
||||
}
|
||||
|
||||
}
|
||||
<-workers
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBlobID(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
m := map[string][]byte{}
|
||||
if err := msgpack.Unmarshal(b, &m); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if bid := m["user.oc.blobid"]; string(bid) != "" {
|
||||
return string(bid), nil
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func getIDsFromPath(path string) (string, string) {
|
||||
rawIDs := strings.Split(path, "/nodes/")
|
||||
if len(rawIDs) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
s := strings.Split(rawIDs[0], "/spaces/")
|
||||
if len(s) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
n := strings.Split(rawIDs[1], ".REV.")
|
||||
if len(n) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
spaceID := strings.Replace(s[1], "/", "", -1)
|
||||
nodeID := strings.Replace(n[0], "/", "", -1)
|
||||
return spaceID, filepath.Base(nodeID)
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
|
||||
"github.com/test-go/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
_basePath = "/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/"
|
||||
)
|
||||
|
||||
// func TestInit(t *testing.T) {
|
||||
// initialize(10, 2)
|
||||
// defer os.RemoveAll("test_temp")
|
||||
// }
|
||||
|
||||
func TestGlob30(t *testing.T) { test(t, 10, 2, glob) }
|
||||
func TestGlob80(t *testing.T) { test(t, 20, 3, glob) }
|
||||
func TestGlob250(t *testing.T) { test(t, 50, 4, glob) }
|
||||
func TestGlob600(t *testing.T) { test(t, 100, 5, glob) }
|
||||
|
||||
func TestWalk30(t *testing.T) { test(t, 10, 2, walk) }
|
||||
func TestWalk80(t *testing.T) { test(t, 20, 3, walk) }
|
||||
func TestWalk250(t *testing.T) { test(t, 50, 4, walk) }
|
||||
func TestWalk600(t *testing.T) { test(t, 100, 5, walk) }
|
||||
|
||||
func TestList30(t *testing.T) { test(t, 10, 2, list2) }
|
||||
func TestList80(t *testing.T) { test(t, 20, 3, list10) }
|
||||
func TestList250(t *testing.T) { test(t, 50, 4, list20) }
|
||||
func TestList600(t *testing.T) { test(t, 100, 5, list2) }
|
||||
|
||||
func TestGlobWorkers30(t *testing.T) { test(t, 10, 2, globWorkersD1) }
|
||||
func TestGlobWorkers80(t *testing.T) { test(t, 20, 3, globWorkersD2) }
|
||||
func TestGlobWorkers250(t *testing.T) { test(t, 50, 4, globWorkersD4) }
|
||||
func TestGlobWorkers600(t *testing.T) { test(t, 100, 5, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob30(b *testing.B) { benchmark(b, 10, 2, glob) }
|
||||
func BenchmarkWalk30(b *testing.B) { benchmark(b, 10, 2, walk) }
|
||||
func BenchmarkList30(b *testing.B) { benchmark(b, 10, 2, list2) }
|
||||
func BenchmarkGlobWorkers30(b *testing.B) { benchmark(b, 10, 2, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob80(b *testing.B) { benchmark(b, 20, 3, glob) }
|
||||
func BenchmarkWalk80(b *testing.B) { benchmark(b, 20, 3, walk) }
|
||||
func BenchmarkList80(b *testing.B) { benchmark(b, 20, 3, list2) }
|
||||
func BenchmarkGlobWorkers80(b *testing.B) { benchmark(b, 20, 3, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob250(b *testing.B) { benchmark(b, 50, 4, glob) }
|
||||
func BenchmarkWalk250(b *testing.B) { benchmark(b, 50, 4, walk) }
|
||||
func BenchmarkList250(b *testing.B) { benchmark(b, 50, 4, list2) }
|
||||
func BenchmarkGlobWorkers250(b *testing.B) { benchmark(b, 50, 4, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlobAT600(b *testing.B) { benchmark(b, 100, 5, glob) }
|
||||
func BenchmarkWalkAT600(b *testing.B) { benchmark(b, 100, 5, walk) }
|
||||
func BenchmarkList2AT600(b *testing.B) { benchmark(b, 100, 5, list2) }
|
||||
func BenchmarkList10AT600(b *testing.B) { benchmark(b, 100, 5, list10) }
|
||||
func BenchmarkList20AT600(b *testing.B) { benchmark(b, 100, 5, list20) }
|
||||
func BenchmarkGlobWorkersD1AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD1) }
|
||||
func BenchmarkGlobWorkersD2AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD2) }
|
||||
func BenchmarkGlobWorkersD4AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD4) }
|
||||
|
||||
func BenchmarkGlobAT22000(b *testing.B) { benchmark(b, 2000, 10, glob) }
|
||||
func BenchmarkWalkAT22000(b *testing.B) { benchmark(b, 2000, 10, walk) }
|
||||
func BenchmarkList2AT22000(b *testing.B) { benchmark(b, 2000, 10, list2) }
|
||||
func BenchmarkList10AT22000(b *testing.B) { benchmark(b, 2000, 10, list10) }
|
||||
func BenchmarkList20AT22000(b *testing.B) { benchmark(b, 2000, 10, list20) }
|
||||
func BenchmarkGlobWorkersD1AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD1) }
|
||||
func BenchmarkGlobWorkersD2AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD2) }
|
||||
func BenchmarkGlobWorkersD4AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD4) }
|
||||
|
||||
func BenchmarkGlob110000(b *testing.B) { benchmark(b, 10000, 10, glob) }
|
||||
func BenchmarkWalk110000(b *testing.B) { benchmark(b, 10000, 10, walk) }
|
||||
func BenchmarkList110000(b *testing.B) { benchmark(b, 10000, 10, list2) }
|
||||
func BenchmarkGlobWorkers110000(b *testing.B) { benchmark(b, 10000, 10, globWorkersD2) }
|
||||
|
||||
func benchmark(b *testing.B, numNodes int, numRevisions int, f func(string) <-chan string) {
|
||||
base := initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll(base)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
ch := f(base)
|
||||
PurgeRevisions(ch, nil, false, false)
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func test(t *testing.T, numNodes int, numRevisions int, f func(string) <-chan string) {
|
||||
base := initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll(base)
|
||||
|
||||
ch := f(base)
|
||||
_, _, revisions := PurgeRevisions(ch, nil, false, false)
|
||||
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
|
||||
}
|
||||
|
||||
func glob(base string) <-chan string {
|
||||
return Glob(base + _basePath + "*/*/*/*/*")
|
||||
}
|
||||
|
||||
func walk(base string) <-chan string {
|
||||
return Walk(base + _basePath)
|
||||
}
|
||||
|
||||
func list2(base string) <-chan string {
|
||||
return List(base+_basePath, 2)
|
||||
}
|
||||
|
||||
func list10(base string) <-chan string {
|
||||
return List(base+_basePath, 10)
|
||||
}
|
||||
|
||||
func list20(base string) <-chan string {
|
||||
return List(base+_basePath, 20)
|
||||
}
|
||||
|
||||
func globWorkersD1(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*", "/*/*/*/*")
|
||||
}
|
||||
|
||||
func globWorkersD2(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*/*", "/*/*/*")
|
||||
}
|
||||
|
||||
func globWorkersD4(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*/*/*/*", "/*")
|
||||
}
|
||||
|
||||
func initialize(numNodes int, numRevisions int) string {
|
||||
base := "test_temp_" + uuid.New().String()
|
||||
if err := os.Mkdir(base, os.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// create base path
|
||||
if err := os.MkdirAll(base+_basePath, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating base path", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
path := lookup.Pathify(uuid.New().String(), 4, 2)
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(base+_basePath+dir, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if _, err := os.Create(base + _basePath + path); err != nil {
|
||||
fmt.Println("Error creating file", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
for i := 0; i < numRevisions; i++ {
|
||||
os.Create(base + _basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z")
|
||||
}
|
||||
}
|
||||
return base
|
||||
}
|
||||
Reference in New Issue
Block a user