change binary name
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
@@ -0,0 +1,175 @@
|
||||
// Package backup contains ocis backup functionality.
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Inconsistency describes the type of inconsistency
|
||||
type Inconsistency string
|
||||
|
||||
var (
|
||||
// InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore
|
||||
InconsistencyBlobMissing Inconsistency = "blob missing"
|
||||
// InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference
|
||||
InconsistencyBlobOrphaned Inconsistency = "blob orphaned"
|
||||
// InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node
|
||||
InconsistencyNodeMissing Inconsistency = "node missing"
|
||||
// InconsistencyMetadataMissing is an inconsistency where a node is missing metadata
|
||||
InconsistencyMetadataMissing Inconsistency = "metadata missing"
|
||||
// InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink
|
||||
InconsistencySymlinkMissing Inconsistency = "symlink missing"
|
||||
// InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock
|
||||
InconsistencyFilesMissing Inconsistency = "files missing"
|
||||
// InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file
|
||||
InconsistencyMalformedFile Inconsistency = "malformed file"
|
||||
|
||||
// regex to determine if a node is trashed or versioned.
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
|
||||
_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$`)
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink
|
||||
_trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$`)
|
||||
)
|
||||
|
||||
// Consistency holds the node and blob data of a storage provider
|
||||
type Consistency struct {
|
||||
// Storing the data like this might take a lot of memory
|
||||
// we might need to optimize this if we run into memory issues
|
||||
Nodes map[string][]Inconsistency
|
||||
LinkedNodes map[string][]Inconsistency
|
||||
BlobReferences map[string][]Inconsistency
|
||||
Blobs map[string][]Inconsistency
|
||||
|
||||
nodeToLink map[string]string
|
||||
blobToNode map[string]string
|
||||
}
|
||||
|
||||
// NewConsistency creates a new Consistency object
|
||||
func NewConsistency() *Consistency {
|
||||
return &Consistency{
|
||||
Nodes: make(map[string][]Inconsistency),
|
||||
LinkedNodes: make(map[string][]Inconsistency),
|
||||
BlobReferences: make(map[string][]Inconsistency),
|
||||
Blobs: make(map[string][]Inconsistency),
|
||||
|
||||
nodeToLink: make(map[string]string),
|
||||
blobToNode: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// CheckProviderConsistency checks the consistency of a space
|
||||
func CheckProviderConsistency(storagepath string, lbs ListBlobstore, fail bool) error {
|
||||
fsys := os.DirFS(storagepath)
|
||||
|
||||
p := NewProvider(fsys, storagepath, lbs)
|
||||
if err := p.ProduceData(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c := NewConsistency()
|
||||
c.GatherData(p.Events)
|
||||
|
||||
return c.PrintResults(storagepath, fail)
|
||||
}
|
||||
|
||||
// GatherData gathers and evaluates data produced by the DataProvider
|
||||
func (c *Consistency) GatherData(events <-chan interface{}) {
|
||||
for ev := range events {
|
||||
switch d := ev.(type) {
|
||||
case NodeData:
|
||||
// does it have inconsistencies?
|
||||
if len(d.Inconsistencies) != 0 {
|
||||
c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...)
|
||||
}
|
||||
// is it linked?
|
||||
if _, ok := c.LinkedNodes[d.NodePath]; ok {
|
||||
deleteInconsistency(c.LinkedNodes, d.NodePath)
|
||||
} else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil {
|
||||
c.Nodes[d.NodePath] = []Inconsistency{}
|
||||
}
|
||||
// does it have a blob?
|
||||
if d.BlobPath != "" {
|
||||
if _, ok := c.Blobs[d.BlobPath]; ok {
|
||||
deleteInconsistency(c.Blobs, d.BlobPath)
|
||||
} else {
|
||||
c.BlobReferences[d.BlobPath] = []Inconsistency{}
|
||||
c.blobToNode[d.BlobPath] = d.NodePath
|
||||
}
|
||||
}
|
||||
case LinkData:
|
||||
// does it have a node?
|
||||
if _, ok := c.Nodes[d.NodePath]; ok {
|
||||
deleteInconsistency(c.Nodes, d.NodePath)
|
||||
} else {
|
||||
c.LinkedNodes[d.NodePath] = []Inconsistency{}
|
||||
c.nodeToLink[d.NodePath] = d.LinkPath
|
||||
}
|
||||
case BlobData:
|
||||
// does it have a reference?
|
||||
if _, ok := c.BlobReferences[d.BlobPath]; ok {
|
||||
deleteInconsistency(c.BlobReferences, d.BlobPath)
|
||||
} else {
|
||||
c.Blobs[d.BlobPath] = []Inconsistency{}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
for n := range c.Nodes {
|
||||
if len(c.Nodes[n]) == 0 {
|
||||
c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing)
|
||||
}
|
||||
}
|
||||
for l := range c.LinkedNodes {
|
||||
c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing)
|
||||
}
|
||||
for b := range c.Blobs {
|
||||
c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned)
|
||||
}
|
||||
for b := range c.BlobReferences {
|
||||
c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing)
|
||||
}
|
||||
}
|
||||
|
||||
// PrintResults prints the results of the evaluation
|
||||
func (c *Consistency) PrintResults(discpath string, fail bool) error {
|
||||
if len(c.Nodes) != 0 {
|
||||
fmt.Println("\n🚨 Inconsistent Nodes:")
|
||||
}
|
||||
for n := range c.Nodes {
|
||||
fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n)
|
||||
}
|
||||
if len(c.LinkedNodes) != 0 {
|
||||
fmt.Println("\n🚨 Inconsistent Links:")
|
||||
}
|
||||
for l := range c.LinkedNodes {
|
||||
fmt.Printf("\t👉️ %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l)
|
||||
}
|
||||
if len(c.Blobs) != 0 {
|
||||
fmt.Println("\n🚨 Inconsistent Blobs:")
|
||||
}
|
||||
for b := range c.Blobs {
|
||||
fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b)
|
||||
}
|
||||
if len(c.BlobReferences) != 0 {
|
||||
fmt.Println("\n🚨 Inconsistent BlobReferences:")
|
||||
}
|
||||
for b := range c.BlobReferences {
|
||||
fmt.Printf("\t👉️ %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b])
|
||||
}
|
||||
if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 {
|
||||
fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", discpath)
|
||||
} else if fail {
|
||||
os.Exit(1)
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func deleteInconsistency(incs map[string][]Inconsistency, path string) {
|
||||
if len(incs[path]) == 0 {
|
||||
delete(incs, path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/backup"
|
||||
"github.com/test-go/testify/require"
|
||||
)
|
||||
|
||||
func TestGatherData(t *testing.T) {
|
||||
testcases := []struct {
|
||||
Name string
|
||||
Events []interface{}
|
||||
Expected *backup.Consistency
|
||||
}{
|
||||
{
|
||||
Name: "no symlinks - no blobs",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", true),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
node(c, "nodepath", backup.InconsistencySymlinkMissing)
|
||||
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "symlink not required - no blobs",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", false),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "no inconsistencies",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", true),
|
||||
linkData("linkpath", "nodepath"),
|
||||
blobData("blobpath"),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "orphaned blob",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", true),
|
||||
linkData("linkpath", "nodepath"),
|
||||
blobData("blobpath"),
|
||||
blobData("anotherpath"),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
blob(c, "anotherpath", backup.InconsistencyBlobOrphaned)
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "missing node",
|
||||
Events: []interface{}{
|
||||
linkData("linkpath", "nodepath"),
|
||||
blobData("blobpath"),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
linkedNode(c, "nodepath", backup.InconsistencyNodeMissing)
|
||||
blob(c, "blobpath", backup.InconsistencyBlobOrphaned)
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "corrupt metadata",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
|
||||
linkData("linkpath", "nodepath"),
|
||||
blobData("blobpath"),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
node(c, "nodepath", backup.InconsistencyMetadataMissing)
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "corrupt metadata, no blob",
|
||||
Events: []interface{}{
|
||||
nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
|
||||
linkData("linkpath", "nodepath"),
|
||||
},
|
||||
Expected: consistency(func(c *backup.Consistency) {
|
||||
node(c, "nodepath", backup.InconsistencyMetadataMissing)
|
||||
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
events := make(chan interface{})
|
||||
|
||||
go func() {
|
||||
for _, ev := range tc.Events {
|
||||
switch e := ev.(type) {
|
||||
case backup.NodeData:
|
||||
events <- e
|
||||
case backup.LinkData:
|
||||
events <- e
|
||||
case backup.BlobData:
|
||||
events <- e
|
||||
}
|
||||
}
|
||||
close(events)
|
||||
}()
|
||||
|
||||
c := backup.NewConsistency()
|
||||
c.GatherData(events)
|
||||
|
||||
require.Equal(t, tc.Expected.Nodes, c.Nodes)
|
||||
require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes)
|
||||
require.Equal(t, tc.Expected.Blobs, c.Blobs)
|
||||
require.Equal(t, tc.Expected.BlobReferences, c.BlobReferences)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func nodeData(nodePath, blobPath string, requiresSymlink bool, incs ...backup.Inconsistency) backup.NodeData {
|
||||
return backup.NodeData{
|
||||
NodePath: nodePath,
|
||||
BlobPath: blobPath,
|
||||
RequiresSymlink: requiresSymlink,
|
||||
Inconsistencies: incs,
|
||||
}
|
||||
}
|
||||
|
||||
func linkData(linkPath, nodePath string) backup.LinkData {
|
||||
return backup.LinkData{
|
||||
LinkPath: linkPath,
|
||||
NodePath: nodePath,
|
||||
}
|
||||
}
|
||||
|
||||
func blobData(blobPath string) backup.BlobData {
|
||||
return backup.BlobData{
|
||||
BlobPath: blobPath,
|
||||
}
|
||||
}
|
||||
|
||||
func consistency(f func(*backup.Consistency)) *backup.Consistency {
|
||||
c := backup.NewConsistency()
|
||||
f(c)
|
||||
return c
|
||||
}
|
||||
|
||||
func node(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
|
||||
c.Nodes[path] = inc
|
||||
}
|
||||
|
||||
func linkedNode(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
|
||||
c.LinkedNodes[path] = inc
|
||||
}
|
||||
|
||||
func blob(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
|
||||
c.Blobs[path] = inc
|
||||
}
|
||||
|
||||
func blobReference(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
|
||||
c.BlobReferences[path] = inc
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
// ListBlobstore required to check blob consistency
|
||||
type ListBlobstore interface {
|
||||
List() ([]*node.Node, error)
|
||||
Path(node *node.Node) string
|
||||
}
|
||||
|
||||
// DataProvider provides data for the consistency check
|
||||
type DataProvider struct {
|
||||
Events chan interface{}
|
||||
|
||||
fsys fs.FS
|
||||
discpath string
|
||||
lbs ListBlobstore
|
||||
skipBlobs bool
|
||||
}
|
||||
|
||||
// NodeData holds data about the nodes
|
||||
type NodeData struct {
|
||||
NodePath string
|
||||
BlobPath string
|
||||
RequiresSymlink bool
|
||||
Inconsistencies []Inconsistency
|
||||
}
|
||||
|
||||
// LinkData about the symlinks
|
||||
type LinkData struct {
|
||||
LinkPath string
|
||||
NodePath string
|
||||
}
|
||||
|
||||
// BlobData about the blobs in the blobstore
|
||||
type BlobData struct {
|
||||
BlobPath string
|
||||
}
|
||||
|
||||
// NewProvider creates a new DataProvider object
|
||||
func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider {
|
||||
return &DataProvider{
|
||||
Events: make(chan interface{}),
|
||||
|
||||
fsys: fsys,
|
||||
discpath: discpath,
|
||||
lbs: lbs,
|
||||
skipBlobs: lbs == nil,
|
||||
}
|
||||
}
|
||||
|
||||
// ProduceData produces data for the consistency check
|
||||
// Spawns 4 go-routines at the moment. If needed, this can be optimized.
|
||||
func (dp *DataProvider) ProduceData() error {
|
||||
dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(dirs) == 0 {
|
||||
return errors.New("no backup found. Double check storage path")
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
// crawl spaces
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
for _, d := range dirs {
|
||||
dp.evaluateNodeDir(d)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// crawl trash
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
dp.evaluateTrashDir()
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// crawl blobstore
|
||||
if !dp.skipBlobs {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
bs, err := dp.lbs.List()
|
||||
if err != nil {
|
||||
fmt.Println("error listing blobs", err)
|
||||
}
|
||||
|
||||
for _, bn := range bs {
|
||||
dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)}
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
// wait for all crawlers to finish
|
||||
go func() {
|
||||
wg.Wait()
|
||||
dp.quit()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
|
||||
if dp.skipBlobs {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
b, err := fs.ReadFile(dp.fsys, path+".mpk")
|
||||
if err != nil {
|
||||
return "", InconsistencyFilesMissing
|
||||
}
|
||||
|
||||
m := map[string][]byte{}
|
||||
if err := msgpack.Unmarshal(b, &m); err != nil {
|
||||
return "", InconsistencyMalformedFile
|
||||
}
|
||||
|
||||
// FIXME: how to check if metadata is complete?
|
||||
|
||||
if bid := m["user.ocis.blobid"]; string(bid) != "" {
|
||||
spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
|
||||
return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), ""
|
||||
}
|
||||
|
||||
return "", ""
|
||||
}
|
||||
|
||||
func (dp *DataProvider) evaluateNodeDir(d string) {
|
||||
// d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24
|
||||
// we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension
|
||||
entries, err := fs.ReadDir(dp.fsys, d)
|
||||
if err != nil {
|
||||
fmt.Println("error reading dir", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("empty dir", filepath.Join(dp.discpath, d))
|
||||
return
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case e.IsDir():
|
||||
ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name()))
|
||||
if err != nil {
|
||||
fmt.Println("error reading dir", err)
|
||||
continue
|
||||
}
|
||||
for _, l := range ls {
|
||||
linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name())
|
||||
|
||||
r, _ := os.Readlink(linkpath)
|
||||
nodePath := filepath.Join(dp.discpath, d, e.Name(), r)
|
||||
dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath}
|
||||
}
|
||||
fallthrough
|
||||
case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()):
|
||||
np := filepath.Join(dp.discpath, d, e.Name())
|
||||
var inc []Inconsistency
|
||||
if !dp.filesExist(filepath.Join(d, e.Name())) {
|
||||
inc = append(inc, InconsistencyFilesMissing)
|
||||
}
|
||||
bp, i := dp.getBlobPath(filepath.Join(d, e.Name()))
|
||||
if i != "" {
|
||||
inc = append(inc, i)
|
||||
}
|
||||
|
||||
dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dp *DataProvider) evaluateTrashDir() {
|
||||
linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*")
|
||||
if err != nil {
|
||||
fmt.Println("error reading trash", err)
|
||||
}
|
||||
for _, l := range linkpaths {
|
||||
linkpath := filepath.Join(dp.discpath, l)
|
||||
r, _ := os.Readlink(linkpath)
|
||||
p := filepath.Join(dp.discpath, l, "..", r)
|
||||
dp.Events <- LinkData{LinkPath: linkpath, NodePath: p}
|
||||
}
|
||||
}
|
||||
|
||||
func (dp *DataProvider) filesExist(path string) bool {
|
||||
check := func(p string) bool {
|
||||
_, err := fs.Stat(dp.fsys, p)
|
||||
return err == nil
|
||||
}
|
||||
return check(path) && check(path+".mpk")
|
||||
}
|
||||
|
||||
func (dp *DataProvider) quit() {
|
||||
close(dp.Events)
|
||||
}
|
||||
|
||||
func requiresSymlink(path string) bool {
|
||||
spaceID, nodeID := getIDsFromPath(path)
|
||||
if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func getIDsFromPath(path string) (string, string) {
|
||||
rawIDs := strings.Split(path, "/nodes/")
|
||||
if len(rawIDs) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
s := strings.Split(rawIDs[0], "/spaces/")
|
||||
if len(s) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
spaceID := strings.Replace(s[1], "/", "", -1)
|
||||
nodeID := strings.Replace(rawIDs[1], "/", "", -1)
|
||||
return spaceID, nodeID
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
|
||||
s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/backup"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// BackupCommand is the entrypoint for the backup command
|
||||
func BackupCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "backup",
|
||||
Usage: "ocis backup functionality",
|
||||
Subcommands: []*cli.Command{
|
||||
ConsistencyCommand(cfg),
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
Action: func(_ *cli.Context) error {
|
||||
fmt.Println("Read the docs")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ConsistencyCommand is the entrypoint for the consistency Command
|
||||
func ConsistencyCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "consistency",
|
||||
Usage: "check backup consistency",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "basepath",
|
||||
Aliases: []string{"p"},
|
||||
Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)",
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "blobstore",
|
||||
Aliases: []string{"b"},
|
||||
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis",
|
||||
Value: "ocis",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "fail",
|
||||
Usage: "exit with non-zero status if consistency check fails",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
basePath := c.String("basepath")
|
||||
if basePath == "" {
|
||||
fmt.Println("basepath is required")
|
||||
return cli.ShowCommandHelp(c, "consistency")
|
||||
}
|
||||
|
||||
var (
|
||||
bs backup.ListBlobstore
|
||||
err error
|
||||
)
|
||||
switch c.String("blobstore") {
|
||||
case "s3ng":
|
||||
bs, err = s3bs.New(
|
||||
cfg.StorageUsers.Drivers.S3NG.Endpoint,
|
||||
cfg.StorageUsers.Drivers.S3NG.Region,
|
||||
cfg.StorageUsers.Drivers.S3NG.Bucket,
|
||||
cfg.StorageUsers.Drivers.S3NG.AccessKey,
|
||||
cfg.StorageUsers.Drivers.S3NG.SecretKey,
|
||||
s3bs.Options{},
|
||||
)
|
||||
case "ocis":
|
||||
bs, err = ocisbs.New(basePath)
|
||||
case "none":
|
||||
bs = nil
|
||||
default:
|
||||
err = errors.New("blobstore type not supported")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
if err := backup.CheckProviderConsistency(basePath, bs, c.Bool("fail")); err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(BackupCommand)
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tw "github.com/olekukonko/tablewriter"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/version"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// BenchmarkCommand is the entrypoint for the benchmark commands.
|
||||
func BenchmarkCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "benchmark",
|
||||
Usage: "cli tools to test low and high level performance",
|
||||
Category: "benchmark",
|
||||
Subcommands: []*cli.Command{BenchmarkClientCommand(cfg), BenchmarkSyscallsCommand(cfg)},
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkClientCommand is the entrypoint for the benchmark client command.
|
||||
func BenchmarkClientCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "client",
|
||||
|
||||
Usage: "Start a client that continuously makes web requests and prints stats. The options mimic curl, but URL must be at the end.",
|
||||
Flags: []cli.Flag{
|
||||
|
||||
// TODO with v3 'flag.Persistent: true' can be set to make the order of flags no longer relevant \o/
|
||||
// flags mimicing curl
|
||||
&cli.StringFlag{
|
||||
Name: "request",
|
||||
Aliases: []string{"X"},
|
||||
Value: "PROPFIND",
|
||||
Usage: "Specifies a custom request method to use when communicating with the HTTP server.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "user",
|
||||
Aliases: []string{"u"},
|
||||
Value: "admin:admin",
|
||||
Usage: "Specify the user name and password to use for server authentication.",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "insecure",
|
||||
Aliases: []string{"k"},
|
||||
Usage: "Skip the TLS verification step and proceed without checking.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "data",
|
||||
Aliases: []string{"d"},
|
||||
Usage: "Sends the specified data in a request to the HTTP server.",
|
||||
// TODE support multiple data flags, support data-binary, data-raw
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
Name: "header",
|
||||
Aliases: []string{"H"},
|
||||
Usage: "Extra header to include in information sent.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "rate",
|
||||
Usage: `Specify the maximum transfer frequency you allow a client to use - in number of transfer starts per time unit (sometimes called request rate).
|
||||
The request rate is provided as "N/U" where N is an integer number and U is a time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd' /(day, as in a 24 hour unit). The default time unit, if no "/U" is provided, is number of transfers per hour.`,
|
||||
},
|
||||
/*
|
||||
&cli.StringFlag{
|
||||
Name: "oauth2-bearer",
|
||||
Usage: "Specify the Bearer Token for OAUTH 2.0 server authentication.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "user-agent",
|
||||
Aliases: []string{"A"},
|
||||
Value: "admin:admin",
|
||||
Usage: "Specify the User-Agent string to send to the HTTP server.",
|
||||
},
|
||||
*/
|
||||
// other flags
|
||||
&cli.StringFlag{
|
||||
Name: "bearer-token-command",
|
||||
Usage: "Command to execute for a bearer token, e.g. 'oidc-token OCIS'. When set, disables basic auth.",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "every",
|
||||
Usage: "Aggregate stats every time this amount of seconds has passed.",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "jobs",
|
||||
Aliases: []string{"j"},
|
||||
Value: 1,
|
||||
Usage: "Number of parallel clients to start.",
|
||||
},
|
||||
},
|
||||
Category: "benchmark",
|
||||
Action: func(c *cli.Context) error {
|
||||
opt := clientOptions{
|
||||
request: c.String("request"),
|
||||
url: c.Args().First(),
|
||||
insecure: c.Bool("insecure"),
|
||||
jobs: c.Int("jobs"),
|
||||
headers: make(map[string]string),
|
||||
data: []byte(c.String("data")),
|
||||
}
|
||||
if opt.url == "" {
|
||||
log.Fatal(errors.New("no URL specified"))
|
||||
}
|
||||
|
||||
for _, h := range c.StringSlice("headers") {
|
||||
parts := strings.SplitN(h, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
log.Fatal(errors.New("invalid header '" + h + "'"))
|
||||
}
|
||||
opt.headers[parts[0]] = strings.TrimSpace(parts[1])
|
||||
}
|
||||
|
||||
rate := c.String("rate")
|
||||
if rate != "" {
|
||||
parts := strings.SplitN(rate, "/", 2)
|
||||
num, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
unit := time.Hour // default
|
||||
if len(parts) == 2 {
|
||||
switch parts[1] {
|
||||
case "s":
|
||||
unit = time.Second
|
||||
case "m":
|
||||
unit = time.Minute
|
||||
case "d":
|
||||
unit = time.Hour * 24
|
||||
default:
|
||||
log.Fatal(errors.New("unsupported rate unit. Use s, m, h or d"))
|
||||
}
|
||||
}
|
||||
opt.rateDelay = unit / time.Duration(num)
|
||||
}
|
||||
|
||||
user := c.String("user")
|
||||
opt.auth = func() string {
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(user))
|
||||
}
|
||||
|
||||
btc := c.String("bearer-token-command")
|
||||
if btc != "" {
|
||||
parts := strings.SplitN(btc, " ", 2)
|
||||
var cmd *exec.Cmd
|
||||
opt.auth = func() string {
|
||||
if len(parts) > 1 {
|
||||
cmd = exec.Command(parts[0], parts[1])
|
||||
} else {
|
||||
cmd = exec.Command(parts[0])
|
||||
}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
return "Bearer " + string(output)
|
||||
}
|
||||
}
|
||||
|
||||
every := c.Int("every")
|
||||
if every != 0 {
|
||||
opt.ticker = time.NewTicker(time.Second * time.Duration(every))
|
||||
defer opt.ticker.Stop()
|
||||
}
|
||||
|
||||
return client(opt)
|
||||
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type clientOptions struct {
|
||||
request string
|
||||
url string
|
||||
auth func() string
|
||||
insecure bool
|
||||
headers map[string]string
|
||||
rateDelay time.Duration
|
||||
data []byte
|
||||
ticker *time.Ticker
|
||||
jobs int
|
||||
}
|
||||
|
||||
func client(o clientOptions) error {
|
||||
|
||||
type stat struct {
|
||||
job int
|
||||
duration time.Duration
|
||||
status int
|
||||
}
|
||||
stats := make(chan stat)
|
||||
for i := 0; i < o.jobs; i++ {
|
||||
go func(i int) {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: o.insecure,
|
||||
},
|
||||
}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
cookies := map[string]*http.Cookie{}
|
||||
for {
|
||||
req, err := http.NewRequest(o.request, o.url, bytes.NewReader(o.data))
|
||||
if err != nil {
|
||||
log.Printf("client %d: could not create request: %s\n", i, err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("Authorization", strings.TrimSpace(o.auth()))
|
||||
for k, v := range o.headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
for _, cookie := range cookies {
|
||||
req.AddCookie(cookie)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
res, err := client.Do(req)
|
||||
duration := -time.Until(start)
|
||||
if err != nil {
|
||||
log.Printf("client %d: could not create request: %s\n", i, err)
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
res.Body.Close()
|
||||
stats <- stat{
|
||||
job: i,
|
||||
duration: duration,
|
||||
status: res.StatusCode,
|
||||
}
|
||||
for _, c := range res.Cookies() {
|
||||
cookies[c.Name] = c
|
||||
}
|
||||
}
|
||||
time.Sleep(o.rateDelay - duration)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
numRequests := 0
|
||||
if o.ticker == nil {
|
||||
// no ticker, just write every request
|
||||
for {
|
||||
stat := <-stats
|
||||
numRequests++
|
||||
fmt.Printf("req %d took %v and returned status %d\n", numRequests, stat.duration, stat.status)
|
||||
}
|
||||
}
|
||||
|
||||
var duration time.Duration
|
||||
for {
|
||||
select {
|
||||
case stat := <-stats:
|
||||
numRequests++
|
||||
duration += stat.duration
|
||||
case <-o.ticker.C:
|
||||
if numRequests > 0 {
|
||||
fmt.Printf("%d req at %v/req\n", numRequests, duration/time.Duration(numRequests))
|
||||
numRequests = 0
|
||||
duration = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// BenchmarkSyscallsCommand is the entrypoint for the benchmark syscalls command.
|
||||
func BenchmarkSyscallsCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "syscalls",
|
||||
Usage: "test the performance of syscalls",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "path",
|
||||
Usage: "Path to test",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "iterations",
|
||||
Value: "100",
|
||||
Usage: "Number of iterations to execute",
|
||||
},
|
||||
},
|
||||
Category: "benchmark",
|
||||
Action: func(c *cli.Context) error {
|
||||
|
||||
path := c.String("path")
|
||||
if path == "" {
|
||||
f, err := os.CreateTemp("", "ocis-bench-temp-")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
path = f.Name()
|
||||
f.Close()
|
||||
defer os.Remove(path)
|
||||
}
|
||||
|
||||
iterations := c.Int("iterations")
|
||||
|
||||
return benchmark(iterations, path)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func benchmark(iterations int, path string) error {
|
||||
tests := map[string]func() error{
|
||||
"lockedfile open(wo,c,t) close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
lockedFile, err := lockedfile.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockedFile.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"stat": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(ro) close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(wo,t) write close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
h, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.WriteString("1234567890")
|
||||
if err != nil {
|
||||
h.Close()
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(ro) read close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
bytes := make([]byte, 0, 10)
|
||||
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.Read(bytes)
|
||||
if err != nil {
|
||||
h.Close()
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"xattr-set": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
err := xattr.Set(path, "user.test", []byte("123456"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"xattr-get": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
_, err := xattr.Get(path, "user.test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
fmt.Println("Version: " + version.GetString())
|
||||
fmt.Printf("Compiled: %s\n", version.Compiled())
|
||||
fmt.Printf("Path: %s\n", path)
|
||||
fmt.Printf("Iterations: %d\n", iterations)
|
||||
fmt.Println("")
|
||||
|
||||
table := tw.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Test", "Iterations", "dur/it", "total"})
|
||||
table.SetAutoFormatHeaders(false)
|
||||
table.SetColumnAlignment([]int{tw.ALIGN_LEFT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT})
|
||||
table.SetAutoMergeCellsByColumnIndex([]int{2, 3})
|
||||
for _, t := range []string{"lockedfile open(wo,c,t) close", "stat", "fopen(wo,t) write close", "fopen(ro) close", "fopen(ro) read close", "xattr-set", "xattr-get"} {
|
||||
start := time.Now()
|
||||
err := tests[t]()
|
||||
end := time.Now()
|
||||
delta := end.Sub(start)
|
||||
if err != nil {
|
||||
table.Append([]string{t, fmt.Sprintf("%d", iterations), err.Error(), err.Error()})
|
||||
} else {
|
||||
table.Append([]string{t, fmt.Sprintf("%d", iterations), strconv.Itoa(int(delta.Nanoseconds())/iterations) + "ns", delta.String()})
|
||||
}
|
||||
}
|
||||
table.Render()
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(BenchmarkCommand)
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
revactx "github.com/cs3org/reva/v2/pkg/ctx"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/cache"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree"
|
||||
"github.com/cs3org/reva/v2/pkg/storagespace"
|
||||
"github.com/cs3org/reva/v2/pkg/store"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// DecomposedfsCommand is the entrypoint for the groups command.
|
||||
func DecomposedfsCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "decomposedfs",
|
||||
Usage: `cli tools to inspect and manipulate a decomposedfs storage.`,
|
||||
Category: "maintenance",
|
||||
Subcommands: []*cli.Command{
|
||||
metadataCmd(cfg),
|
||||
checkCmd(cfg),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(DecomposedfsCommand)
|
||||
}
|
||||
|
||||
func checkCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "check-treesize",
|
||||
Usage: `cli tool to check the treesize metadata of a Space`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "root",
|
||||
Aliases: []string{"r"},
|
||||
Required: true,
|
||||
Usage: "Path to the root directory of the decomposedfs",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "node",
|
||||
Required: true,
|
||||
Aliases: []string{"n"},
|
||||
Usage: "Space ID of the Space to inspect",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "repair",
|
||||
Usage: "Try to repair nodes with incorrect treesize metadata. IMPORTANT: Only use this while ownCloud Infinite Scale is not running.",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "force",
|
||||
Usage: "Do not prompt for confirmation when running in repair mode.",
|
||||
},
|
||||
},
|
||||
Action: check,
|
||||
}
|
||||
}
|
||||
|
||||
func check(c *cli.Context) error {
|
||||
rootFlag := c.String("root")
|
||||
repairFlag := c.Bool("repair")
|
||||
|
||||
if repairFlag && !c.Bool("force") {
|
||||
answer := strings.ToLower(stringPrompt("IMPORTANT: Only use '--repair' when ownCloud Infinite Scale is not running. Do you want to continue? [yes | no = default]"))
|
||||
if answer != "yes" && answer != "y" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
lu, backend := getBackend(c)
|
||||
o := &options.Options{
|
||||
MetadataBackend: backend.Name(),
|
||||
MaxConcurrency: 100,
|
||||
}
|
||||
bs, err := blobstore.New(rootFlag)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to init blobstore")
|
||||
return err
|
||||
}
|
||||
|
||||
tree := tree.New(lu, bs, o, store.Create(), &zerolog.Logger{})
|
||||
|
||||
nId := c.String("node")
|
||||
n, err := lu.NodeFromSpaceID(context.Background(), nId)
|
||||
if err != nil || !n.Exists {
|
||||
fmt.Println("Can not find node '" + nId + "'")
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Checking treesizes in space: %s (id: %s)\n", n.Name, n.ID)
|
||||
ctx := revactx.ContextSetUser(context.Background(),
|
||||
&userpb.User{
|
||||
Id: &userpb.UserId{
|
||||
OpaqueId: "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
Username: "offline",
|
||||
})
|
||||
|
||||
treeSize, err := walkTree(ctx, tree, lu, n, repairFlag)
|
||||
treesizeFromMetadata, err := n.GetTreeSize(c.Context)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to read treesize of node: %s: %s\n", n.ID, err)
|
||||
}
|
||||
if treesizeFromMetadata != treeSize {
|
||||
fmt.Printf("Tree sizes mismatch for space: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
|
||||
n.Name, n.ID, n.InternalPath(), treeSize, treesizeFromMetadata)
|
||||
if repairFlag {
|
||||
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
|
||||
n.ID, treeSize)
|
||||
n.SetTreeSize(c.Context, treeSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func walkTree(ctx context.Context, tree *tree.Tree, lu *lookup.Lookup, root *node.Node, repair bool) (uint64, error) {
|
||||
if root.Type(ctx) != provider.ResourceType_RESOURCE_TYPE_CONTAINER {
|
||||
return 0, errors.New("can't travers non-container nodes")
|
||||
}
|
||||
children, err := tree.ListFolder(ctx, root)
|
||||
if err != nil {
|
||||
fmt.Println("Can not list children for space'" + root.ID + "'")
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var treesize uint64
|
||||
for _, child := range children {
|
||||
switch child.Type(ctx) {
|
||||
case provider.ResourceType_RESOURCE_TYPE_CONTAINER:
|
||||
subtreesize, err := walkTree(ctx, tree, lu, child, repair)
|
||||
if err != nil {
|
||||
fmt.Printf("error calculating tree size of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
treesizeFromMetadata, err := child.GetTreeSize(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to read tree size of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
if treesizeFromMetadata != subtreesize {
|
||||
origin, err := lu.Path(ctx, child, node.NoCheck)
|
||||
if err != nil {
|
||||
fmt.Printf("error get path: %s\n", err)
|
||||
}
|
||||
fmt.Printf("Tree sizes mismatch for node: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
|
||||
origin, child.ID, child.InternalPath(), subtreesize, treesizeFromMetadata)
|
||||
if repair {
|
||||
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
|
||||
child.ID, subtreesize)
|
||||
child.SetTreeSize(ctx, subtreesize)
|
||||
}
|
||||
}
|
||||
treesize += subtreesize
|
||||
case provider.ResourceType_RESOURCE_TYPE_FILE:
|
||||
blobsize, err := child.GetBlobSize(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("error reading blobsize of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
treesize += blobsize
|
||||
default:
|
||||
fmt.Printf("Ignoring type: %v, node: %s %s\n", child.Type(ctx), child.Name, child.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return treesize, nil
|
||||
}
|
||||
|
||||
func metadataCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "metadata",
|
||||
Usage: `cli tools to inspect and manipulate node metadata`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "root",
|
||||
Aliases: []string{"r"},
|
||||
Required: true,
|
||||
Usage: "Path to the decomposedfs",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "node",
|
||||
Required: true,
|
||||
Aliases: []string{"n"},
|
||||
Usage: "Path to or ID of the node to inspect",
|
||||
},
|
||||
},
|
||||
Subcommands: []*cli.Command{dumpCmd(cfg), getCmd(cfg), setCmd(cfg)},
|
||||
}
|
||||
}
|
||||
|
||||
func dumpCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "dump",
|
||||
Usage: `print the metadata of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
|
||||
Action: func(c *cli.Context) error {
|
||||
lu, backend := getBackend(c)
|
||||
path, err := getPath(c, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attribs, err := backend.All(c.Context, path)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading attributes")
|
||||
return err
|
||||
}
|
||||
printAttribs(attribs, c.String("attribute"))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "get",
|
||||
Usage: `print a specific attribute of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "attribute",
|
||||
Aliases: []string{"a"},
|
||||
Usage: "attribute to inspect",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
lu, backend := getBackend(c)
|
||||
path, err := getPath(c, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attribs, err := backend.All(c.Context, path)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading attributes")
|
||||
return err
|
||||
}
|
||||
printAttribs(attribs, c.String("attribute"))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func setCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "set",
|
||||
Usage: `manipulate metadata of the given node. Binary attributes can be given hex encoded (prefix by '0x') or base64 encoded (prefix by '0s').`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "attribute",
|
||||
Required: true,
|
||||
Aliases: []string{"a"},
|
||||
Usage: "attribute to inspect",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "value",
|
||||
Required: true,
|
||||
Aliases: []string{"v"},
|
||||
Usage: "value to set",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
lu, backend := getBackend(c)
|
||||
path, err := getPath(c, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v := c.String("value")
|
||||
if strings.HasPrefix(v, "0s") {
|
||||
b64, err := base64.StdEncoding.DecodeString(v[2:])
|
||||
if err == nil {
|
||||
v = string(b64)
|
||||
} else {
|
||||
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
|
||||
}
|
||||
} else if strings.HasPrefix(v, "0x") {
|
||||
h, err := hex.DecodeString(v[2:])
|
||||
if err == nil {
|
||||
v = string(h)
|
||||
} else {
|
||||
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
err = backend.Set(c.Context, path, c.String("attribute"), []byte(v))
|
||||
if err != nil {
|
||||
fmt.Println("Error setting attribute")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func backend(root, backend string) metadata.Backend {
|
||||
switch backend {
|
||||
case "xattrs":
|
||||
return metadata.NewXattrsBackend(root, cache.Config{})
|
||||
case "mpk":
|
||||
return metadata.NewMessagePackBackend(root, cache.Config{})
|
||||
}
|
||||
return metadata.NullBackend{}
|
||||
}
|
||||
|
||||
func getBackend(c *cli.Context) (*lookup.Lookup, metadata.Backend) {
|
||||
rootFlag := c.String("root")
|
||||
|
||||
bod := lookup.DetectBackendOnDisk(rootFlag)
|
||||
backend := backend(rootFlag, bod)
|
||||
lu := lookup.New(backend, &options.Options{
|
||||
Root: rootFlag,
|
||||
MetadataBackend: bod,
|
||||
}, &timemanager.Manager{})
|
||||
return lu, backend
|
||||
}
|
||||
|
||||
func getPath(c *cli.Context, lu *lookup.Lookup) (string, error) {
|
||||
nodeFlag := c.String("node")
|
||||
|
||||
path := ""
|
||||
if strings.HasPrefix(nodeFlag, "/") {
|
||||
path = nodeFlag
|
||||
} else {
|
||||
nId := c.String("node")
|
||||
id, err := storagespace.ParseID(nId)
|
||||
if err != nil {
|
||||
fmt.Println("Invalid node id.")
|
||||
return "", err
|
||||
}
|
||||
n, err := lu.NodeFromID(context.Background(), &id)
|
||||
if err != nil || !n.Exists {
|
||||
fmt.Println("Can not find node '" + nId + "'")
|
||||
return "", err
|
||||
}
|
||||
path = n.InternalPath()
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func printAttribs(attribs map[string][]byte, onlyAttribute string) {
|
||||
if onlyAttribute != "" {
|
||||
fmt.Println(onlyAttribute + `=` + attribToString(attribs[onlyAttribute]))
|
||||
return
|
||||
}
|
||||
|
||||
names := []string{}
|
||||
for k := range attribs {
|
||||
names = append(names, k)
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
|
||||
for _, n := range names {
|
||||
fmt.Println(n + `=` + attribToString(attribs[n]))
|
||||
}
|
||||
}
|
||||
|
||||
func attribToString(attrib []byte) string {
|
||||
for i := 0; i < len(attrib); i++ {
|
||||
if attrib[i] < 32 || attrib[i] >= 127 {
|
||||
return "0s" + base64.StdEncoding.EncodeToString(attrib)
|
||||
}
|
||||
}
|
||||
return `"` + string(attrib) + `"`
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func SubcommandDescription(serviceName string) string {
|
||||
return fmt.Sprintf("%s service commands", serviceName)
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/defaults"
|
||||
ocisinit "github.com/opencloud-eu/opencloud/opencloud/pkg/init"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
cli "github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// InitCommand is the entrypoint for the init command
|
||||
func InitCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "init",
|
||||
Usage: "initialise an ocis config",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "insecure",
|
||||
EnvVars: []string{"OCIS_INSECURE"},
|
||||
Value: "ask",
|
||||
Usage: "Allow insecure oCIS config",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "diff",
|
||||
Aliases: []string{"d"},
|
||||
Usage: "Show the difference between the current config and the new one",
|
||||
Value: false,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "force-overwrite",
|
||||
Aliases: []string{"f"},
|
||||
EnvVars: []string{"OCIS_FORCE_CONFIG_OVERWRITE"},
|
||||
Value: false,
|
||||
Usage: "Force overwrite existing config file",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "config-path",
|
||||
Value: defaults.BaseConfigPath(),
|
||||
Usage: "Config path for the ocis runtime",
|
||||
EnvVars: []string{"OCIS_CONFIG_DIR", "OCIS_BASE_DATA_PATH"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "admin-password",
|
||||
Aliases: []string{"ap"},
|
||||
EnvVars: []string{"ADMIN_PASSWORD", "IDM_ADMIN_PASSWORD"},
|
||||
Usage: "Set admin password instead of using a random generated one",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
insecureFlag := c.String("insecure")
|
||||
insecure := false
|
||||
if insecureFlag == "ask" {
|
||||
answer := strings.ToLower(stringPrompt("Do you want to configure Infinite Scale with certificate checking disabled?\n This is not recommended for public instances! [yes | no = default]"))
|
||||
if answer == "yes" || answer == "y" {
|
||||
insecure = true
|
||||
}
|
||||
} else if insecureFlag == strings.ToLower("true") || insecureFlag == strings.ToLower("yes") || insecureFlag == strings.ToLower("y") {
|
||||
insecure = true
|
||||
}
|
||||
err := ocisinit.CreateConfig(insecure, c.Bool("force-overwrite"), c.Bool("diff"), c.String("config-path"), c.String("admin-password"))
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create config: %s", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(InitCommand)
|
||||
}
|
||||
|
||||
func stringPrompt(label string) string {
|
||||
input := ""
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
for {
|
||||
fmt.Fprint(os.Stderr, label+" ")
|
||||
input, _ = reader.ReadString('\n')
|
||||
if input != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/rpc"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// ListCommand is the entrypoint for the list command.
|
||||
func ListCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "list",
|
||||
Usage: "list oCIS services running in the runtime (supervised mode)",
|
||||
Category: "runtime",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "hostname",
|
||||
Value: "localhost",
|
||||
EnvVars: []string{"OCIS_RUNTIME_HOST"},
|
||||
Destination: &cfg.Runtime.Host,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "port",
|
||||
Value: "9250",
|
||||
EnvVars: []string{"OCIS_RUNTIME_PORT"},
|
||||
Destination: &cfg.Runtime.Port,
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
client, err := rpc.DialHTTP("tcp", net.JoinHostPort(cfg.Runtime.Host, cfg.Runtime.Port))
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to the runtime. Has the runtime been started and did you configure the right runtime address (\"%s\")", cfg.Runtime.Host+":"+cfg.Runtime.Port)
|
||||
}
|
||||
|
||||
var arg1 string
|
||||
|
||||
if err := client.Call("Service.List", struct{}{}, &arg1); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(arg1)
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(ListCommand)
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/cs3org/reva/v2/pkg/publicshare"
|
||||
publicregistry "github.com/cs3org/reva/v2/pkg/publicshare/manager/registry"
|
||||
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/cs3org/reva/v2/pkg/share"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/shareid"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/registry"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/metadata"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
tw "github.com/olekukonko/tablewriter"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
oclog "github.com/opencloud-eu/opencloud/ocis-pkg/log"
|
||||
mregistry "github.com/opencloud-eu/opencloud/ocis-pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/config"
|
||||
sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Migrate is the entrypoint for the Migrate command.
|
||||
func Migrate(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "migrate",
|
||||
Usage: "migrate data from an existing to another instance",
|
||||
Category: "migration",
|
||||
Subcommands: []*cli.Command{
|
||||
MigrateDecomposedfs(cfg),
|
||||
MigrateShares(cfg),
|
||||
MigratePublicShares(cfg),
|
||||
RebuildJSONCS3Indexes(cfg),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(Migrate)
|
||||
}
|
||||
|
||||
// RebuildJSONCS3Indexes rebuilds the share indexes from the shares json
|
||||
func RebuildJSONCS3Indexes(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "rebuild-jsoncs3-indexes",
|
||||
Usage: "rebuild the share indexes from the shares json",
|
||||
Subcommands: []*cli.Command{},
|
||||
Flags: []cli.Flag{},
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
log := logger()
|
||||
ctx := log.WithContext(context.Background())
|
||||
rcfg := revaShareConfig(cfg.Sharing)
|
||||
|
||||
// Initialize registry to make service lookup work
|
||||
_ = mregistry.GetRegistry()
|
||||
|
||||
// Get a jsoncs3 manager to operate its caches
|
||||
type config struct {
|
||||
GatewayAddr string `mapstructure:"gateway_addr"`
|
||||
MaxConcurrency int `mapstructure:"max_concurrency"`
|
||||
ProviderAddr string `mapstructure:"provider_addr"`
|
||||
ServiceUserID string `mapstructure:"service_user_id"`
|
||||
ServiceUserIdp string `mapstructure:"service_user_idp"`
|
||||
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
|
||||
}
|
||||
conf := &config{}
|
||||
if err := mapstructure.Decode(rcfg["jsoncs3"], conf); err != nil {
|
||||
err = errors.Wrap(err, "error creating a new manager")
|
||||
return err
|
||||
}
|
||||
s, err := metadata.NewCS3Storage(conf.GatewayAddr, conf.ProviderAddr, conf.ServiceUserID, conf.ServiceUserIdp, conf.MachineAuthAPIKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = s.Init(ctx, "jsoncs3-share-manager-metadata")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gatewaySelector, err := pool.GatewaySelector(conf.GatewayAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mgr, err := jsoncs3.New(s, gatewaySelector, 0, nil, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Rebuild indexes
|
||||
errorsOccured := false
|
||||
storages, err := s.ReadDir(ctx, "storages")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for iStorage, storage := range storages {
|
||||
fmt.Printf("Scanning storage %s (%d/%d)\n", storage, iStorage+1, len(storages))
|
||||
spaces, err := s.ReadDir(ctx, filepath.Join("storages", storage))
|
||||
if err != nil {
|
||||
fmt.Printf("failed! (%s)\n", err.Error())
|
||||
errorsOccured = true
|
||||
continue
|
||||
}
|
||||
|
||||
for iSpace, space := range spaces {
|
||||
fmt.Printf(" Rebuilding space '%s' %d/%d...", strings.TrimSuffix(space, ".json"), iSpace+1, len(spaces))
|
||||
|
||||
spaceBlob, err := s.SimpleDownload(ctx, filepath.Join("storages", storage, space))
|
||||
if err != nil {
|
||||
fmt.Printf(" failed! (%s)\n", err.Error())
|
||||
errorsOccured = true
|
||||
continue
|
||||
}
|
||||
shares := &providercache.Shares{}
|
||||
err = json.Unmarshal(spaceBlob, shares)
|
||||
if err != nil {
|
||||
fmt.Printf(" failed! (%s)\n", err.Error())
|
||||
errorsOccured = true
|
||||
continue
|
||||
}
|
||||
for _, share := range shares.Shares {
|
||||
err = mgr.Cache.Add(ctx, share.ResourceId.StorageId, share.ResourceId.SpaceId, share.Id.OpaqueId, share)
|
||||
if err != nil {
|
||||
fmt.Printf(" adding share '%s' to the cache failed! (%s)\n", share.Id.OpaqueId, err.Error())
|
||||
errorsOccured = true
|
||||
}
|
||||
err = mgr.CreatedCache.Add(ctx, share.Creator.OpaqueId, share.Id.OpaqueId)
|
||||
if err != nil {
|
||||
fmt.Printf(" adding share '%s' to the created cache failed! (%s)\n", share.Id.OpaqueId, err.Error())
|
||||
errorsOccured = true
|
||||
}
|
||||
|
||||
spaceId := share.ResourceId.StorageId + shareid.IDDelimiter + share.ResourceId.SpaceId
|
||||
switch share.Grantee.Type {
|
||||
case provider.GranteeType_GRANTEE_TYPE_USER:
|
||||
userid := share.Grantee.GetUserId().GetOpaqueId()
|
||||
existingState, err := mgr.UserReceivedStates.Get(ctx, userid, spaceId, share.Id.OpaqueId)
|
||||
if err != nil {
|
||||
fmt.Printf(" retrieving current state of received share '%s' from the user cache failed! (%s)\n", share.Id.OpaqueId, err.Error())
|
||||
errorsOccured = true
|
||||
} else if existingState == nil {
|
||||
rs := &collaboration.ReceivedShare{
|
||||
Share: share,
|
||||
State: collaboration.ShareState_SHARE_STATE_PENDING,
|
||||
}
|
||||
err := mgr.UserReceivedStates.Add(ctx, userid, spaceId, rs)
|
||||
if err != nil {
|
||||
fmt.Printf(" adding share '%s' to the user cache failed! (%s)\n", share.Id.OpaqueId, err.Error())
|
||||
errorsOccured = true
|
||||
}
|
||||
}
|
||||
case provider.GranteeType_GRANTEE_TYPE_GROUP:
|
||||
groupid := share.Grantee.GetGroupId().GetOpaqueId()
|
||||
err := mgr.GroupReceivedCache.Add(ctx, groupid, spaceId)
|
||||
if err != nil {
|
||||
fmt.Printf(" adding share '%s' to the group cache failed! (%s)\n", share.Id.OpaqueId, err.Error())
|
||||
errorsOccured = true
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf(" done\n")
|
||||
}
|
||||
fmt.Printf("done\n")
|
||||
}
|
||||
if errorsOccured {
|
||||
return errors.New("There were errors. Please review the logs or try again.")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// MigrateDecomposedfs is the entrypoint for the decomposedfs migrate command
|
||||
func MigrateDecomposedfs(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "decomposedfs",
|
||||
Usage: "run a decomposedfs migration",
|
||||
Subcommands: []*cli.Command{
|
||||
ListDecomposedfsMigrations(cfg),
|
||||
},
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "direction",
|
||||
Aliases: []string{"d"},
|
||||
Value: "migrate",
|
||||
Usage: "direction of the migration to run ('migrate' or 'rollback')",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "migration",
|
||||
Aliases: []string{"m"},
|
||||
Value: "",
|
||||
Usage: "ID of the migration to run",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "root",
|
||||
Aliases: []string{"r"},
|
||||
Required: true,
|
||||
Usage: "Path to the root directory of the decomposedfs",
|
||||
},
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
log := logger()
|
||||
rootFlag := c.String("root")
|
||||
bod := lookup.DetectBackendOnDisk(rootFlag)
|
||||
backend := backend(rootFlag, bod)
|
||||
lu := lookup.New(backend, &options.Options{
|
||||
Root: rootFlag,
|
||||
MetadataBackend: bod,
|
||||
}, &timemanager.Manager{})
|
||||
|
||||
m := migrator.New(lu, log)
|
||||
|
||||
err := m.RunMigration(c.String("migration"), c.String("direction") == "down")
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ListDecomposedfsMigrations is the entrypoint for the decomposedfs list migrations command
|
||||
func ListDecomposedfsMigrations(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "list",
|
||||
Usage: "list decomposedfs migrations",
|
||||
Action: func(c *cli.Context) error {
|
||||
rootFlag := c.String("root")
|
||||
bod := lookup.DetectBackendOnDisk(rootFlag)
|
||||
backend := backend(rootFlag, bod)
|
||||
lu := lookup.New(backend, &options.Options{
|
||||
Root: rootFlag,
|
||||
MetadataBackend: bod,
|
||||
}, &timemanager.Manager{})
|
||||
|
||||
m := migrator.New(lu, logger())
|
||||
migrationStates, err := m.Migrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrations := []string{}
|
||||
for m := range migrationStates {
|
||||
migrations = append(migrations, m)
|
||||
}
|
||||
sort.Strings(migrations)
|
||||
|
||||
table := tw.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Migration", "State", "Message"})
|
||||
table.SetAutoFormatHeaders(false)
|
||||
for _, migration := range migrations {
|
||||
table.Append([]string{migration, migrationStates[migration].State, migrationStates[migration].Message})
|
||||
}
|
||||
table.Render()
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func MigrateShares(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "shares",
|
||||
Usage: "migrates shares from the previous to the new share manager",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "from",
|
||||
Value: "json",
|
||||
Usage: "Share manager to export the data from",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "to",
|
||||
Value: "jsoncs3",
|
||||
Usage: "Share manager to import the data into",
|
||||
},
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
log := logger()
|
||||
ctx := log.WithContext(context.Background())
|
||||
rcfg := revaShareConfig(cfg.Sharing)
|
||||
oldDriver := c.String("from")
|
||||
newDriver := c.String("to")
|
||||
shareChan := make(chan *collaboration.Share)
|
||||
receivedShareChan := make(chan share.ReceivedShareWithUser)
|
||||
|
||||
f, ok := registry.NewFuncs[oldDriver]
|
||||
if !ok {
|
||||
log.Error().Msg("Unknown share manager type '" + oldDriver + "'")
|
||||
os.Exit(1)
|
||||
}
|
||||
oldMgr, err := f(rcfg[oldDriver].(map[string]interface{}))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to initiate source share manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
dumpMgr, ok := oldMgr.(share.DumpableManager)
|
||||
if !ok {
|
||||
log.Error().Msg("Share manager type '" + oldDriver + "' does not support dumping its shares.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
f, ok = registry.NewFuncs[newDriver]
|
||||
if !ok {
|
||||
log.Error().Msg("Unknown share manager type '" + newDriver + "'")
|
||||
os.Exit(1)
|
||||
}
|
||||
newMgr, err := f(rcfg[newDriver].(map[string]interface{}))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to initiate destination share manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
loadMgr, ok := newMgr.(share.LoadableManager)
|
||||
if !ok {
|
||||
log.Error().Msg("Share manager type '" + newDriver + "' does not support loading a shares dump.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
log.Info().Msg("Migrating shares...")
|
||||
err = loadMgr.Load(ctx, shareChan, receivedShareChan)
|
||||
log.Info().Msg("Finished migrating shares.")
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Error while loading shares")
|
||||
os.Exit(1)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
err = dumpMgr.Dump(ctx, shareChan, receivedShareChan)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Error while dumping shares")
|
||||
os.Exit(1)
|
||||
}
|
||||
close(shareChan)
|
||||
close(receivedShareChan)
|
||||
wg.Done()
|
||||
}()
|
||||
wg.Wait()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func MigratePublicShares(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "publicshares",
|
||||
Usage: "migrates public shares from the previous to the new public share manager",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "from",
|
||||
Value: "json",
|
||||
Usage: "Public share manager to export the data from",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "to",
|
||||
Value: "jsoncs3",
|
||||
Usage: "Public share manager to import the data into",
|
||||
},
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
log := logger()
|
||||
ctx := log.WithContext(context.Background())
|
||||
|
||||
rcfg := revaPublicShareConfig(cfg.Sharing)
|
||||
oldDriver := c.String("from")
|
||||
newDriver := c.String("to")
|
||||
shareChan := make(chan *publicshare.WithPassword)
|
||||
|
||||
f, ok := publicregistry.NewFuncs[oldDriver]
|
||||
if !ok {
|
||||
log.Error().Msg("Unknown public share manager type '" + oldDriver + "'")
|
||||
os.Exit(1)
|
||||
}
|
||||
oldMgr, err := f(rcfg[oldDriver].(map[string]interface{}))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to initiate source public share manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
dumpMgr, ok := oldMgr.(publicshare.DumpableManager)
|
||||
if !ok {
|
||||
log.Error().Msg("Public share manager type '" + oldDriver + "' does not support dumping its public shares.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
f, ok = publicregistry.NewFuncs[newDriver]
|
||||
if !ok {
|
||||
log.Error().Msg("Unknown public share manager type '" + newDriver + "'")
|
||||
os.Exit(1)
|
||||
}
|
||||
newMgr, err := f(rcfg[newDriver].(map[string]interface{}))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to initiate destination public share manager")
|
||||
os.Exit(1)
|
||||
}
|
||||
loadMgr, ok := newMgr.(publicshare.LoadableManager)
|
||||
if !ok {
|
||||
log.Error().Msg("Public share manager type '" + newDriver + "' does not support loading a public shares dump.")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
log.Info().Msg("Migrating public shares...")
|
||||
err = loadMgr.Load(ctx, shareChan)
|
||||
log.Info().Msg("Finished migrating public shares.")
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Error while loading public shares")
|
||||
os.Exit(1)
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
err = dumpMgr.Dump(ctx, shareChan)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Error while dumping public shares")
|
||||
os.Exit(1)
|
||||
}
|
||||
close(shareChan)
|
||||
wg.Done()
|
||||
}()
|
||||
wg.Wait()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func revaShareConfig(cfg *sharing.Config) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"json": map[string]interface{}{
|
||||
"file": cfg.UserSharingDrivers.JSON.File,
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
},
|
||||
"sql": map[string]interface{}{ // cernbox sql
|
||||
"db_username": cfg.UserSharingDrivers.SQL.DBUsername,
|
||||
"db_password": cfg.UserSharingDrivers.SQL.DBPassword,
|
||||
"db_host": cfg.UserSharingDrivers.SQL.DBHost,
|
||||
"db_port": cfg.UserSharingDrivers.SQL.DBPort,
|
||||
"db_name": cfg.UserSharingDrivers.SQL.DBName,
|
||||
"password_hash_cost": cfg.UserSharingDrivers.SQL.PasswordHashCost,
|
||||
"enable_expired_shares_cleanup": cfg.UserSharingDrivers.SQL.EnableExpiredSharesCleanup,
|
||||
"janitor_run_interval": cfg.UserSharingDrivers.SQL.JanitorRunInterval,
|
||||
},
|
||||
"owncloudsql": map[string]interface{}{
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
"storage_mount_id": cfg.UserSharingDrivers.OwnCloudSQL.UserStorageMountID,
|
||||
"db_username": cfg.UserSharingDrivers.OwnCloudSQL.DBUsername,
|
||||
"db_password": cfg.UserSharingDrivers.OwnCloudSQL.DBPassword,
|
||||
"db_host": cfg.UserSharingDrivers.OwnCloudSQL.DBHost,
|
||||
"db_port": cfg.UserSharingDrivers.OwnCloudSQL.DBPort,
|
||||
"db_name": cfg.UserSharingDrivers.OwnCloudSQL.DBName,
|
||||
},
|
||||
"cs3": map[string]interface{}{
|
||||
"gateway_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
|
||||
"provider_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
|
||||
"service_user_id": cfg.UserSharingDrivers.CS3.SystemUserID,
|
||||
"service_user_idp": cfg.UserSharingDrivers.CS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.UserSharingDrivers.CS3.SystemUserAPIKey,
|
||||
},
|
||||
"jsoncs3": map[string]interface{}{
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
"provider_addr": cfg.UserSharingDrivers.JSONCS3.ProviderAddr,
|
||||
"service_user_id": cfg.UserSharingDrivers.JSONCS3.SystemUserID,
|
||||
"service_user_idp": cfg.UserSharingDrivers.JSONCS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.UserSharingDrivers.JSONCS3.SystemUserAPIKey,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func revaPublicShareConfig(cfg *sharing.Config) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"json": map[string]interface{}{
|
||||
"file": cfg.PublicSharingDrivers.JSON.File,
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
},
|
||||
"jsoncs3": map[string]interface{}{
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
"provider_addr": cfg.PublicSharingDrivers.JSONCS3.ProviderAddr,
|
||||
"service_user_id": cfg.PublicSharingDrivers.JSONCS3.SystemUserID,
|
||||
"service_user_idp": cfg.PublicSharingDrivers.JSONCS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.PublicSharingDrivers.JSONCS3.SystemUserAPIKey,
|
||||
},
|
||||
"sql": map[string]interface{}{
|
||||
"db_username": cfg.PublicSharingDrivers.SQL.DBUsername,
|
||||
"db_password": cfg.PublicSharingDrivers.SQL.DBPassword,
|
||||
"db_host": cfg.PublicSharingDrivers.SQL.DBHost,
|
||||
"db_port": cfg.PublicSharingDrivers.SQL.DBPort,
|
||||
"db_name": cfg.PublicSharingDrivers.SQL.DBName,
|
||||
"password_hash_cost": cfg.PublicSharingDrivers.SQL.PasswordHashCost,
|
||||
"enable_expired_shares_cleanup": cfg.PublicSharingDrivers.SQL.EnableExpiredSharesCleanup,
|
||||
"janitor_run_interval": cfg.PublicSharingDrivers.SQL.JanitorRunInterval,
|
||||
},
|
||||
"cs3": map[string]interface{}{
|
||||
"gateway_addr": cfg.PublicSharingDrivers.CS3.ProviderAddr,
|
||||
"provider_addr": cfg.PublicSharingDrivers.CS3.ProviderAddr,
|
||||
"service_user_id": cfg.PublicSharingDrivers.CS3.SystemUserID,
|
||||
"service_user_idp": cfg.PublicSharingDrivers.CS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.PublicSharingDrivers.CS3.SystemUserAPIKey,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func logger() *zerolog.Logger {
|
||||
log := oclog.NewLogger(
|
||||
oclog.Name("migrate"),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
return &log
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
|
||||
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
|
||||
s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
|
||||
"github.com/cs3org/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/revisions"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
// _nodesGlobPattern is the glob pattern to find all nodes
|
||||
_nodesGlobPattern = "spaces/*/*/nodes/"
|
||||
)
|
||||
|
||||
// RevisionsCommand is the entrypoint for the revisions command.
|
||||
func RevisionsCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "revisions",
|
||||
Usage: "ocis revisions functionality",
|
||||
Subcommands: []*cli.Command{
|
||||
PurgeRevisionsCommand(cfg),
|
||||
},
|
||||
Before: func(_ *cli.Context) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
Action: func(_ *cli.Context) error {
|
||||
fmt.Println("Read the docs")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// PurgeRevisionsCommand allows removing all revisions from a storage provider.
|
||||
func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "purge",
|
||||
Usage: "purge revisions",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "basepath",
|
||||
Aliases: []string{"p"},
|
||||
Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)",
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "blobstore",
|
||||
Aliases: []string{"b"},
|
||||
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service",
|
||||
Value: "ocis",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "dry-run",
|
||||
Usage: "do not delete anything, just print what would be deleted",
|
||||
Value: true,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
Aliases: []string{"v"},
|
||||
Usage: "print verbose output",
|
||||
Value: false,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "resource-id",
|
||||
Aliases: []string{"r"},
|
||||
Usage: "purge all revisions of this file/space. If not set, all revisions will be purged",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "glob-mechanism",
|
||||
Usage: "the glob mechanism to find all nodes. Can be 'glob', 'list' or 'workers'. 'glob' uses globbing with a single worker. 'workers' spawns multiple go routines, accelatering the command drastically but causing high cpu and ram usage. 'list' looks for references by listing directories with multiple workers. Default is 'glob'",
|
||||
Value: "glob",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
basePath := c.String("basepath")
|
||||
if basePath == "" {
|
||||
fmt.Println("basepath is required")
|
||||
return cli.ShowCommandHelp(c, "revisions")
|
||||
}
|
||||
|
||||
var (
|
||||
bs revisions.DelBlobstore
|
||||
err error
|
||||
)
|
||||
switch c.String("blobstore") {
|
||||
case "s3ng":
|
||||
bs, err = s3bs.New(
|
||||
cfg.StorageUsers.Drivers.S3NG.Endpoint,
|
||||
cfg.StorageUsers.Drivers.S3NG.Region,
|
||||
cfg.StorageUsers.Drivers.S3NG.Bucket,
|
||||
cfg.StorageUsers.Drivers.S3NG.AccessKey,
|
||||
cfg.StorageUsers.Drivers.S3NG.SecretKey,
|
||||
s3bs.Options{},
|
||||
)
|
||||
case "ocis":
|
||||
bs, err = ocisbs.New(basePath)
|
||||
case "none":
|
||||
bs = nil
|
||||
default:
|
||||
err = errors.New("blobstore type not supported")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
var rid *provider.ResourceId
|
||||
resid, err := storagespace.ParseID(c.String("resource-id"))
|
||||
if err == nil {
|
||||
rid = &resid
|
||||
}
|
||||
|
||||
mechanism := c.String("glob-mechanism")
|
||||
if rid.GetOpaqueId() != "" {
|
||||
mechanism = "glob"
|
||||
}
|
||||
|
||||
var ch <-chan string
|
||||
switch mechanism {
|
||||
default:
|
||||
fallthrough
|
||||
case "glob":
|
||||
p := generatePath(basePath, rid)
|
||||
if rid.GetOpaqueId() == "" {
|
||||
p = filepath.Join(p, "*/*/*/*/*")
|
||||
}
|
||||
ch = revisions.Glob(p)
|
||||
case "workers":
|
||||
p := generatePath(basePath, rid)
|
||||
ch = revisions.GlobWorkers(p, "/*", "/*/*/*/*")
|
||||
case "list":
|
||||
p := filepath.Join(basePath, "spaces")
|
||||
if rid != nil {
|
||||
p = generatePath(basePath, rid)
|
||||
}
|
||||
ch = revisions.List(p, 10)
|
||||
}
|
||||
|
||||
files, blobs, revisions := revisions.PurgeRevisions(ch, bs, c.Bool("dry-run"), c.Bool("verbose"))
|
||||
printResults(files, blobs, revisions, c.Bool("dry-run"))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func printResults(countFiles, countBlobs, countRevisions int, dryRun bool) {
|
||||
switch {
|
||||
case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
|
||||
fmt.Println("❎ No revisions found. Storage provider is clean.")
|
||||
case !dryRun:
|
||||
fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
default:
|
||||
fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
}
|
||||
}
|
||||
|
||||
func generatePath(basePath string, rid *provider.ResourceId) string {
|
||||
if rid == nil {
|
||||
return filepath.Join(basePath, _nodesGlobPattern)
|
||||
}
|
||||
|
||||
sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
|
||||
if sid == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
|
||||
if nid == "" {
|
||||
return filepath.Join(basePath, "spaces", sid, "nodes")
|
||||
}
|
||||
|
||||
return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*")
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(RevisionsCommand)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/clihelper"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Execute is the entry point for the ocis command.
|
||||
func Execute() error {
|
||||
cfg := config.DefaultConfig()
|
||||
|
||||
app := clihelper.DefaultApp(&cli.App{
|
||||
Name: "opencloud",
|
||||
Usage: "opencloud",
|
||||
})
|
||||
|
||||
for _, fn := range register.Commands {
|
||||
app.Commands = append(
|
||||
app.Commands,
|
||||
fn(cfg),
|
||||
)
|
||||
}
|
||||
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
return app.RunContext(ctx, os.Args)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/runtime"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "server",
|
||||
Usage: "start a fullstack server (runtime and all services in supervised mode)",
|
||||
Category: "fullstack",
|
||||
Before: func(c *cli.Context) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, false))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
// Prefer the in-memory registry as the default when running in single-binary mode
|
||||
r := runtime.New(cfg)
|
||||
return r.Start(c.Context)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(Server)
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/command/helper"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
activitylog "github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
|
||||
antivirus "github.com/opencloud-eu/opencloud/services/antivirus/pkg/command"
|
||||
appprovider "github.com/opencloud-eu/opencloud/services/app-provider/pkg/command"
|
||||
appregistry "github.com/opencloud-eu/opencloud/services/app-registry/pkg/command"
|
||||
audit "github.com/opencloud-eu/opencloud/services/audit/pkg/command"
|
||||
authapp "github.com/opencloud-eu/opencloud/services/auth-app/pkg/command"
|
||||
authbasic "github.com/opencloud-eu/opencloud/services/auth-basic/pkg/command"
|
||||
authbearer "github.com/opencloud-eu/opencloud/services/auth-bearer/pkg/command"
|
||||
authmachine "github.com/opencloud-eu/opencloud/services/auth-machine/pkg/command"
|
||||
authservice "github.com/opencloud-eu/opencloud/services/auth-service/pkg/command"
|
||||
clientlog "github.com/opencloud-eu/opencloud/services/clientlog/pkg/command"
|
||||
collaboration "github.com/opencloud-eu/opencloud/services/collaboration/pkg/command"
|
||||
eventhistory "github.com/opencloud-eu/opencloud/services/eventhistory/pkg/command"
|
||||
frontend "github.com/opencloud-eu/opencloud/services/frontend/pkg/command"
|
||||
gateway "github.com/opencloud-eu/opencloud/services/gateway/pkg/command"
|
||||
graph "github.com/opencloud-eu/opencloud/services/graph/pkg/command"
|
||||
groups "github.com/opencloud-eu/opencloud/services/groups/pkg/command"
|
||||
idm "github.com/opencloud-eu/opencloud/services/idm/pkg/command"
|
||||
idp "github.com/opencloud-eu/opencloud/services/idp/pkg/command"
|
||||
invitations "github.com/opencloud-eu/opencloud/services/invitations/pkg/command"
|
||||
nats "github.com/opencloud-eu/opencloud/services/nats/pkg/command"
|
||||
notifications "github.com/opencloud-eu/opencloud/services/notifications/pkg/command"
|
||||
ocdav "github.com/opencloud-eu/opencloud/services/ocdav/pkg/command"
|
||||
ocm "github.com/opencloud-eu/opencloud/services/ocm/pkg/command"
|
||||
ocs "github.com/opencloud-eu/opencloud/services/ocs/pkg/command"
|
||||
policies "github.com/opencloud-eu/opencloud/services/policies/pkg/command"
|
||||
postprocessing "github.com/opencloud-eu/opencloud/services/postprocessing/pkg/command"
|
||||
proxy "github.com/opencloud-eu/opencloud/services/proxy/pkg/command"
|
||||
search "github.com/opencloud-eu/opencloud/services/search/pkg/command"
|
||||
settings "github.com/opencloud-eu/opencloud/services/settings/pkg/command"
|
||||
sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/command"
|
||||
sse "github.com/opencloud-eu/opencloud/services/sse/pkg/command"
|
||||
storagepubliclink "github.com/opencloud-eu/opencloud/services/storage-publiclink/pkg/command"
|
||||
storageshares "github.com/opencloud-eu/opencloud/services/storage-shares/pkg/command"
|
||||
storagesystem "github.com/opencloud-eu/opencloud/services/storage-system/pkg/command"
|
||||
storageusers "github.com/opencloud-eu/opencloud/services/storage-users/pkg/command"
|
||||
thumbnails "github.com/opencloud-eu/opencloud/services/thumbnails/pkg/command"
|
||||
userlog "github.com/opencloud-eu/opencloud/services/userlog/pkg/command"
|
||||
users "github.com/opencloud-eu/opencloud/services/users/pkg/command"
|
||||
web "github.com/opencloud-eu/opencloud/services/web/pkg/command"
|
||||
webdav "github.com/opencloud-eu/opencloud/services/webdav/pkg/command"
|
||||
webfinger "github.com/opencloud-eu/opencloud/services/webfinger/pkg/command"
|
||||
)
|
||||
|
||||
var svccmds = []register.Command{
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Activitylog.Service.Name, activitylog.GetCommands(cfg.Activitylog), func(c *config.Config) {
|
||||
cfg.Activitylog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Antivirus.Service.Name, antivirus.GetCommands(cfg.Antivirus), func(c *config.Config) {
|
||||
// cfg.Antivirus.Commons = cfg.Commons // antivirus needs no commons atm
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AppProvider.Service.Name, appprovider.GetCommands(cfg.AppProvider), func(c *config.Config) {
|
||||
cfg.AppProvider.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AppRegistry.Service.Name, appregistry.GetCommands(cfg.AppRegistry), func(c *config.Config) {
|
||||
cfg.AppRegistry.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Audit.Service.Name, audit.GetCommands(cfg.Audit), func(c *config.Config) {
|
||||
cfg.Audit.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthApp.Service.Name, authapp.GetCommands(cfg.AuthApp), func(_ *config.Config) {
|
||||
cfg.AuthApp.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthBasic.Service.Name, authbasic.GetCommands(cfg.AuthBasic), func(c *config.Config) {
|
||||
cfg.AuthBasic.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthBearer.Service.Name, authbearer.GetCommands(cfg.AuthBearer), func(c *config.Config) {
|
||||
cfg.AuthBearer.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthMachine.Service.Name, authmachine.GetCommands(cfg.AuthMachine), func(c *config.Config) {
|
||||
cfg.AuthMachine.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthService.Service.Name, authservice.GetCommands(cfg.AuthService), func(c *config.Config) {
|
||||
cfg.AuthService.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Clientlog.Service.Name, clientlog.GetCommands(cfg.Clientlog), func(c *config.Config) {
|
||||
cfg.Clientlog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Collaboration.Service.Name, collaboration.GetCommands(cfg.Collaboration), func(c *config.Config) {
|
||||
cfg.Collaboration.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.EventHistory.Service.Name, eventhistory.GetCommands(cfg.EventHistory), func(c *config.Config) {
|
||||
cfg.EventHistory.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Frontend.Service.Name, frontend.GetCommands(cfg.Frontend), func(c *config.Config) {
|
||||
cfg.Frontend.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Gateway.Service.Name, gateway.GetCommands(cfg.Gateway), func(c *config.Config) {
|
||||
cfg.Gateway.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Graph.Service.Name, graph.GetCommands(cfg.Graph), func(c *config.Config) {
|
||||
cfg.Graph.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Groups.Service.Name, groups.GetCommands(cfg.Groups), func(c *config.Config) {
|
||||
cfg.Groups.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.IDM.Service.Name, idm.GetCommands(cfg.IDM), func(c *config.Config) {
|
||||
cfg.IDM.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.IDP.Service.Name, idp.GetCommands(cfg.IDP), func(c *config.Config) {
|
||||
cfg.IDP.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Invitations.Service.Name, invitations.GetCommands(cfg.Invitations), func(c *config.Config) {
|
||||
cfg.Invitations.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Nats.Service.Name, nats.GetCommands(cfg.Nats), func(c *config.Config) {
|
||||
cfg.Nats.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Notifications.Service.Name, notifications.GetCommands(cfg.Notifications), func(c *config.Config) {
|
||||
cfg.Notifications.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.OCDav.Service.Name, ocdav.GetCommands(cfg.OCDav), func(c *config.Config) {
|
||||
cfg.OCDav.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.OCM.Service.Name, ocm.GetCommands(cfg.OCM), func(c *config.Config) {
|
||||
cfg.OCM.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.OCS.Service.Name, ocs.GetCommands(cfg.OCS), func(c *config.Config) {
|
||||
cfg.OCS.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Policies.Service.Name, policies.GetCommands(cfg.Policies), func(c *config.Config) {
|
||||
cfg.Policies.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Postprocessing.Service.Name, postprocessing.GetCommands(cfg.Postprocessing), func(c *config.Config) {
|
||||
cfg.Postprocessing.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Proxy.Service.Name, proxy.GetCommands(cfg.Proxy), func(c *config.Config) {
|
||||
cfg.Proxy.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Search.Service.Name, search.GetCommands(cfg.Search), func(c *config.Config) {
|
||||
cfg.Search.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Settings.Service.Name, settings.GetCommands(cfg.Settings), func(c *config.Config) {
|
||||
cfg.Settings.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Sharing.Service.Name, sharing.GetCommands(cfg.Sharing), func(c *config.Config) {
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.SSE.Service.Name, sse.GetCommands(cfg.SSE), func(c *config.Config) {
|
||||
cfg.SSE.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.StoragePublicLink.Service.Name, storagepubliclink.GetCommands(cfg.StoragePublicLink), func(c *config.Config) {
|
||||
cfg.StoragePublicLink.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageShares.Service.Name, storageshares.GetCommands(cfg.StorageShares), func(c *config.Config) {
|
||||
cfg.StorageShares.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageSystem.Service.Name, storagesystem.GetCommands(cfg.StorageSystem), func(c *config.Config) {
|
||||
cfg.StorageSystem.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageUsers.Service.Name, storageusers.GetCommands(cfg.StorageUsers), func(c *config.Config) {
|
||||
cfg.StorageUsers.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Thumbnails.Service.Name, thumbnails.GetCommands(cfg.Thumbnails), func(c *config.Config) {
|
||||
cfg.Thumbnails.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Userlog.Service.Name, userlog.GetCommands(cfg.Userlog), func(c *config.Config) {
|
||||
cfg.Userlog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Users.Service.Name, users.GetCommands(cfg.Users), func(c *config.Config) {
|
||||
cfg.Users.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Web.Service.Name, web.GetCommands(cfg.Web), func(c *config.Config) {
|
||||
cfg.Web.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.WebDAV.Service.Name, webdav.GetCommands(cfg.WebDAV), func(c *config.Config) {
|
||||
cfg.WebDAV.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cli.Command {
|
||||
return ServiceCommand(cfg, cfg.Webfinger.Service.Name, webfinger.GetCommands(cfg.Webfinger), func(c *config.Config) {
|
||||
cfg.Webfinger.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// ServiceCommand is the entry point for the all service commands.
|
||||
func ServiceCommand(cfg *config.Config, serviceName string, subcommands []*cli.Command, f func(*config.Config)) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: serviceName,
|
||||
Usage: helper.SubcommandDescription(serviceName),
|
||||
Category: "services",
|
||||
Before: func(c *cli.Context) error {
|
||||
configlog.Error(parser.ParseConfig(cfg, true))
|
||||
f(cfg)
|
||||
return nil
|
||||
},
|
||||
Subcommands: subcommands,
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
for _, c := range svccmds {
|
||||
register.AddCommand(c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3"
|
||||
"github.com/cs3org/reva/v2/pkg/share/manager/registry"
|
||||
"github.com/cs3org/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
mregistry "github.com/opencloud-eu/opencloud/ocis-pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser"
|
||||
)
|
||||
|
||||
// SharesCommand is the entrypoint for the groups command.
|
||||
func SharesCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "shares",
|
||||
Usage: `cli tools to manage entries in the share manager.`,
|
||||
Category: "maintenance",
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
Subcommands: []*cli.Command{
|
||||
cleanupCmd(cfg),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(SharesCommand)
|
||||
}
|
||||
|
||||
func cleanupCmd(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "cleanup",
|
||||
Usage: `clean up stale entries in the share manager.`,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "service-account-id",
|
||||
Value: "",
|
||||
Usage: "Name of the service account to use for the cleanup",
|
||||
EnvVars: []string{"OCIS_SERVICE_ACCOUNT_ID"},
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "service-account-secret",
|
||||
Value: "",
|
||||
Usage: "Secret for the service account",
|
||||
EnvVars: []string{"OCIS_SERVICE_ACCOUNT_SECRET"},
|
||||
Required: true,
|
||||
},
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
return cleanup(c, cfg)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func cleanup(c *cli.Context, cfg *config.Config) error {
|
||||
driver := cfg.Sharing.UserSharingDriver
|
||||
// cleanup is only implemented for the jsoncs3 share manager
|
||||
if driver != "jsoncs3" {
|
||||
return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager"))
|
||||
}
|
||||
|
||||
rcfg := revaShareConfig(cfg.Sharing)
|
||||
f, ok := registry.NewFuncs[driver]
|
||||
if !ok {
|
||||
return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'"))
|
||||
}
|
||||
mgr, err := f(rcfg[driver].(map[string]interface{}))
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Initialize registry to make service lookup work
|
||||
_ = mregistry.GetRegistry()
|
||||
|
||||
// get an authenticated context
|
||||
gatewaySelector, err := pool.GatewaySelector(cfg.Sharing.Reva.Address)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
client, err := gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
serviceUserCtx, err := utils.GetServiceUserContext(c.String("service-account-id"), client, c.String("service-account-secret"))
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
l := logger()
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
serviceUserCtx = l.WithContext(serviceUserCtx)
|
||||
|
||||
mgr.(*jsoncs3.Manager).CleanupStaleShares(serviceUserCtx)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/trash"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
func TrashCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "trash",
|
||||
Usage: "ocis trash functionality",
|
||||
Subcommands: []*cli.Command{
|
||||
TrashPurgeEmptyDirsCommand(cfg),
|
||||
},
|
||||
Before: func(c *cli.Context) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
Action: func(_ *cli.Context) error {
|
||||
fmt.Println("Read the docs")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TrashPurgeEmptyDirsCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "purge-empty-dirs",
|
||||
Usage: "purge empty directories",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "basepath",
|
||||
Aliases: []string{"p"},
|
||||
Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)",
|
||||
Required: true,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "dry-run",
|
||||
Usage: "do not delete anything, just print what would be deleted",
|
||||
Value: true,
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
basePath := c.String("basepath")
|
||||
if basePath == "" {
|
||||
fmt.Println("basepath is required")
|
||||
return cli.ShowCommandHelp(c, "trash")
|
||||
}
|
||||
|
||||
if err := trash.PurgeTrashEmptyPaths(basePath, c.Bool("dry-run")); err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(TrashCommand)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
tw "github.com/olekukonko/tablewriter"
|
||||
"github.com/urfave/cli/v2"
|
||||
mreg "go-micro.dev/v4/registry"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/version"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
)
|
||||
|
||||
const (
|
||||
_skipServiceListingFlagName = "skip-services"
|
||||
)
|
||||
|
||||
// VersionCommand is the entrypoint for the version command.
|
||||
func VersionCommand(cfg *config.Config) *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "version",
|
||||
Usage: "print the version of this binary and all running service instances",
|
||||
Flags: []cli.Flag{
|
||||
&cli.BoolFlag{
|
||||
Name: _skipServiceListingFlagName,
|
||||
Usage: "skip service listing",
|
||||
},
|
||||
},
|
||||
Category: "info",
|
||||
Action: func(c *cli.Context) error {
|
||||
fmt.Println("Version: " + version.GetString())
|
||||
fmt.Printf("Compiled: %s\n", version.Compiled())
|
||||
|
||||
if c.Bool(_skipServiceListingFlagName) {
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Print("\n")
|
||||
|
||||
reg := registry.GetRegistry()
|
||||
serviceList, err := reg.ListServices()
|
||||
if err != nil {
|
||||
fmt.Printf("could not list services: %v\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var services []*mreg.Service
|
||||
for _, s := range serviceList {
|
||||
s, err := reg.GetService(s.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("could not get service: %v\n", err)
|
||||
return err
|
||||
}
|
||||
services = append(services, s...)
|
||||
}
|
||||
|
||||
if len(services) == 0 {
|
||||
fmt.Println("No running services found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
table := tw.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Version", "Address", "Id"})
|
||||
table.SetAutoFormatHeaders(false)
|
||||
for _, s := range services {
|
||||
for _, n := range s.Nodes {
|
||||
table.Append([]string{s.Version, n.Address, n.Id})
|
||||
}
|
||||
}
|
||||
table.Render()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(VersionCommand)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package init
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
func checkConfigPath(configPath string) error {
|
||||
targetPath := path.Join(configPath, configFilename)
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
return fmt.Errorf("config in %s already exists", targetPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func configExists(configPath string) bool {
|
||||
targetPath := path.Join(configPath, configFilename)
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func backupOcisConfigFile(configPath string) (string, error) {
|
||||
sourceConfig := path.Join(configPath, configFilename)
|
||||
targetBackupConfig := path.Join(configPath, configFilename+"."+time.Now().Format("2006-01-02-15-04-05")+".backup")
|
||||
source, err := os.Open(sourceConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not read %s (%s)", sourceConfig, err)
|
||||
}
|
||||
defer source.Close()
|
||||
target, err := os.Create(targetBackupConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not generate backup %s (%s)", targetBackupConfig, err)
|
||||
}
|
||||
defer target.Close()
|
||||
_, err = io.Copy(target, source)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not write backup %s (%s)", targetBackupConfig, err)
|
||||
}
|
||||
return targetBackupConfig, nil
|
||||
}
|
||||
|
||||
// printBanner prints the generated OCIS config banner.
|
||||
func printBanner(targetPath, ocisAdminServicePassword, targetBackupConfig string) {
|
||||
fmt.Printf(
|
||||
"\n=========================================\n"+
|
||||
" generated OCIS Config\n"+
|
||||
"=========================================\n"+
|
||||
" configpath : %s\n"+
|
||||
" user : admin\n"+
|
||||
" password : %s\n\n",
|
||||
targetPath, ocisAdminServicePassword)
|
||||
if targetBackupConfig != "" {
|
||||
fmt.Printf("\n=========================================\n"+
|
||||
"An older config file has been backuped to\n %s\n\n",
|
||||
targetBackupConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// writeConfig writes the config to the target path and prints a banner
|
||||
func writeConfig(configPath, ocisAdminServicePassword, targetBackupConfig string, yamlOutput []byte) error {
|
||||
targetPath := path.Join(configPath, configFilename)
|
||||
err := os.WriteFile(targetPath, yamlOutput, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printBanner(targetPath, ocisAdminServicePassword, targetBackupConfig)
|
||||
return nil
|
||||
}
|
||||
|
||||
// writePatch writes the diff to a file
|
||||
func writePatch(configPath string, yamlOutput []byte) error {
|
||||
fmt.Println("running in diff mode")
|
||||
tmpFile := path.Join(configPath, "ocis.yaml.tmp")
|
||||
err := os.WriteFile(tmpFile, yamlOutput, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("diff -u " + path.Join(configPath, configFilename) + " " + tmpFile)
|
||||
cmd := exec.Command("diff", "-u", path.Join(configPath, configFilename), tmpFile)
|
||||
stdout, err := cmd.Output()
|
||||
if err == nil {
|
||||
err = os.Remove(tmpFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("no changes, your config is up to date")
|
||||
return nil
|
||||
}
|
||||
fmt.Println(string(stdout))
|
||||
err = os.Remove(tmpFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
patchPath := path.Join(configPath, "ocis.config.patch")
|
||||
err = os.WriteFile(patchPath, stdout, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("diff written to %s\n", patchPath)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package init
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/gofrs/uuid"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/generators"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
configFilename = "ocis.yaml"
|
||||
passwordLength = 32
|
||||
)
|
||||
|
||||
var (
|
||||
_insecureService = InsecureService{Insecure: true}
|
||||
_insecureEvents = Events{TLSInsecure: true}
|
||||
)
|
||||
|
||||
// CreateConfig creates a config file with random passwords at configPath
|
||||
func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword string) error {
|
||||
if diff && forceOverwrite {
|
||||
return fmt.Errorf("diff and force-overwrite flags are mutually exclusive")
|
||||
}
|
||||
if diff && adminPassword != "" {
|
||||
return fmt.Errorf("diff and admin-password flags are mutually exclusive")
|
||||
}
|
||||
|
||||
if configExists(configPath) && !forceOverwrite && !diff {
|
||||
return fmt.Errorf("config file already exists, use --force-overwrite to overwrite or --diff to show diff")
|
||||
}
|
||||
|
||||
err := checkConfigPath(configPath)
|
||||
if err != nil && (!forceOverwrite && !diff) {
|
||||
fmt.Println("off")
|
||||
return err
|
||||
}
|
||||
targetBackupConfig := ""
|
||||
if err != nil {
|
||||
targetBackupConfig, err = backupOcisConfigFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = os.MkdirAll(configPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Load old config
|
||||
var oldCfg OcisConfig
|
||||
if diff {
|
||||
fp, err := os.ReadFile(path.Join(configPath, configFilename))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = yaml.Unmarshal(fp, &oldCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
systemUserID, adminUserID, graphApplicationID, storageUsersMountID, serviceAccountID string
|
||||
idmServicePassword, idpServicePassword, ocisAdminServicePassword, revaServicePassword string
|
||||
tokenManagerJwtSecret, collaborationWOPISecret, machineAuthAPIKey, systemUserAPIKey string
|
||||
revaTransferSecret, thumbnailsTransferSecret, serviceAccountSecret string
|
||||
)
|
||||
|
||||
if diff {
|
||||
systemUserID = oldCfg.SystemUserID
|
||||
adminUserID = oldCfg.AdminUserID
|
||||
graphApplicationID = oldCfg.Graph.Application.ID
|
||||
storageUsersMountID = oldCfg.Gateway.StorageRegistry.StorageUsersMountID
|
||||
serviceAccountID = oldCfg.Graph.ServiceAccount.ServiceAccountID
|
||||
|
||||
idmServicePassword = oldCfg.Idm.ServiceUserPasswords.IdmPassword
|
||||
idpServicePassword = oldCfg.Idm.ServiceUserPasswords.IdpPassword
|
||||
ocisAdminServicePassword = oldCfg.Idm.ServiceUserPasswords.AdminPassword
|
||||
revaServicePassword = oldCfg.Idm.ServiceUserPasswords.RevaPassword
|
||||
tokenManagerJwtSecret = oldCfg.TokenManager.JWTSecret
|
||||
collaborationWOPISecret = oldCfg.Collaboration.WopiApp.Secret
|
||||
if collaborationWOPISecret == "" {
|
||||
collaborationWOPISecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random wopi secret for collaboration service: %s", err)
|
||||
}
|
||||
}
|
||||
machineAuthAPIKey = oldCfg.MachineAuthAPIKey
|
||||
systemUserAPIKey = oldCfg.SystemUserAPIKey
|
||||
revaTransferSecret = oldCfg.TransferSecret
|
||||
thumbnailsTransferSecret = oldCfg.Thumbnails.Thumbnail.TransferSecret
|
||||
serviceAccountSecret = oldCfg.Graph.ServiceAccount.ServiceAccountSecret
|
||||
} else {
|
||||
systemUserID = uuid.Must(uuid.NewV4()).String()
|
||||
adminUserID = uuid.Must(uuid.NewV4()).String()
|
||||
graphApplicationID = uuid.Must(uuid.NewV4()).String()
|
||||
storageUsersMountID = uuid.Must(uuid.NewV4()).String()
|
||||
serviceAccountID = uuid.Must(uuid.NewV4()).String()
|
||||
|
||||
idmServicePassword, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for idm: %s", err)
|
||||
}
|
||||
idpServicePassword, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for idp: %s", err)
|
||||
}
|
||||
ocisAdminServicePassword = adminPassword
|
||||
if ocisAdminServicePassword == "" {
|
||||
ocisAdminServicePassword, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for ocis admin: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
revaServicePassword, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for reva: %s", err)
|
||||
}
|
||||
tokenManagerJwtSecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for tokenmanager: %s", err)
|
||||
}
|
||||
collaborationWOPISecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random wopi secret for collaboration service: %s", err)
|
||||
}
|
||||
machineAuthAPIKey, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for machineauthsecret: %s", err)
|
||||
}
|
||||
systemUserAPIKey, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random system user API key: %s", err)
|
||||
}
|
||||
revaTransferSecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for revaTransferSecret: %s", err)
|
||||
}
|
||||
thumbnailsTransferSecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for thumbnailsTransferSecret: %s", err)
|
||||
}
|
||||
serviceAccountSecret, err = generators.GenerateRandomPassword(passwordLength)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not generate random password for thumbnailsTransferSecret: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
serviceAccount := ServiceAccount{
|
||||
ServiceAccountID: serviceAccountID,
|
||||
ServiceAccountSecret: serviceAccountSecret,
|
||||
}
|
||||
|
||||
cfg := OcisConfig{
|
||||
TokenManager: TokenManager{
|
||||
JWTSecret: tokenManagerJwtSecret,
|
||||
},
|
||||
MachineAuthAPIKey: machineAuthAPIKey,
|
||||
SystemUserAPIKey: systemUserAPIKey,
|
||||
TransferSecret: revaTransferSecret,
|
||||
SystemUserID: systemUserID,
|
||||
AdminUserID: adminUserID,
|
||||
Idm: IdmService{
|
||||
ServiceUserPasswords: ServiceUserPasswordsSettings{
|
||||
AdminPassword: ocisAdminServicePassword,
|
||||
IdpPassword: idpServicePassword,
|
||||
RevaPassword: revaServicePassword,
|
||||
IdmPassword: idmServicePassword,
|
||||
},
|
||||
},
|
||||
Idp: LdapBasedService{
|
||||
Ldap: LdapSettings{
|
||||
BindPassword: idpServicePassword,
|
||||
},
|
||||
},
|
||||
AuthBasic: AuthbasicService{
|
||||
AuthProviders: LdapBasedService{
|
||||
Ldap: LdapSettings{
|
||||
BindPassword: revaServicePassword,
|
||||
},
|
||||
},
|
||||
},
|
||||
Collaboration: Collaboration{
|
||||
WopiApp: WopiApp{
|
||||
Secret: collaborationWOPISecret,
|
||||
},
|
||||
},
|
||||
Groups: UsersAndGroupsService{
|
||||
Drivers: LdapBasedService{
|
||||
Ldap: LdapSettings{
|
||||
BindPassword: revaServicePassword,
|
||||
},
|
||||
},
|
||||
},
|
||||
Users: UsersAndGroupsService{
|
||||
Drivers: LdapBasedService{
|
||||
Ldap: LdapSettings{
|
||||
BindPassword: revaServicePassword,
|
||||
},
|
||||
},
|
||||
},
|
||||
Graph: GraphService{
|
||||
Application: GraphApplication{
|
||||
ID: graphApplicationID,
|
||||
},
|
||||
Identity: LdapBasedService{
|
||||
Ldap: LdapSettings{
|
||||
BindPassword: idmServicePassword,
|
||||
},
|
||||
},
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Thumbnails: ThumbnailService{
|
||||
Thumbnail: ThumbnailSettings{
|
||||
TransferSecret: thumbnailsTransferSecret,
|
||||
},
|
||||
},
|
||||
Gateway: Gateway{
|
||||
StorageRegistry: StorageRegistry{
|
||||
StorageUsersMountID: storageUsersMountID,
|
||||
},
|
||||
},
|
||||
StorageUsers: StorageUsers{
|
||||
MountID: storageUsersMountID,
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Userlog: Userlog{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
AuthService: AuthService{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Search: Search{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Notifications: Notifications{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Frontend: FrontendService{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Ocm: OcmService{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Clientlog: Clientlog{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Proxy: ProxyService{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
Settings: SettingsService{
|
||||
ServiceAccountIDs: []string{serviceAccount.ServiceAccountID},
|
||||
},
|
||||
Activitylog: Activitylog{
|
||||
ServiceAccount: serviceAccount,
|
||||
},
|
||||
}
|
||||
|
||||
if insecure {
|
||||
cfg.AuthBearer = AuthbearerService{
|
||||
AuthProviders: AuthProviderSettings{Oidc: _insecureService},
|
||||
}
|
||||
cfg.Collaboration.App.Insecure = true
|
||||
cfg.Frontend.AppHandler = _insecureService
|
||||
cfg.Frontend.Archiver = _insecureService
|
||||
cfg.Graph.Spaces = _insecureService
|
||||
cfg.Graph.Events = _insecureEvents
|
||||
cfg.Notifications.Notifications.Events = _insecureEvents
|
||||
cfg.Search.Events = _insecureEvents
|
||||
cfg.Audit.Events = _insecureEvents
|
||||
cfg.Sharing.Events = _insecureEvents
|
||||
cfg.StorageUsers.Events = _insecureEvents
|
||||
cfg.Nats.Nats.TLSSkipVerifyClientCert = true
|
||||
cfg.Ocdav = _insecureService
|
||||
cfg.Proxy = ProxyService{
|
||||
InsecureBackends: true,
|
||||
OIDC: InsecureProxyOIDC{
|
||||
Insecure: true,
|
||||
},
|
||||
ServiceAccount: serviceAccount,
|
||||
}
|
||||
|
||||
cfg.Thumbnails.Thumbnail.WebdavAllowInsecure = true
|
||||
cfg.Thumbnails.Thumbnail.Cs3AllowInsecure = true
|
||||
}
|
||||
yamlOutput, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not marshall config into yaml: %s", err)
|
||||
}
|
||||
if diff {
|
||||
return writePatch(configPath, yamlOutput)
|
||||
}
|
||||
return writeConfig(configPath, ocisAdminServicePassword, targetBackupConfig, yamlOutput)
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package init
|
||||
|
||||
// TODO: use the oCIS config struct instead of this custom struct
|
||||
// We can't use it right now, because it would need "omitempty" on
|
||||
// all elements, in order to produce a slim config file with `ocis init`.
|
||||
// We can't just add these "omitempty" tags, since we want to generate
|
||||
// full example configuration files with that struct, too.
|
||||
// Proposed solution to get rid of this temporary solution:
|
||||
// - use the oCIS config struct
|
||||
// - set the needed values like below
|
||||
// - marshal it to yaml
|
||||
// - unmarshal it into yaml.Node
|
||||
// - recurse through the nodes and delete empty / default ones
|
||||
// - marshal it to yaml
|
||||
|
||||
// OcisConfig is the configuration for the oCIS services
|
||||
type OcisConfig struct {
|
||||
TokenManager TokenManager `yaml:"token_manager"`
|
||||
MachineAuthAPIKey string `yaml:"machine_auth_api_key"`
|
||||
SystemUserAPIKey string `yaml:"system_user_api_key"`
|
||||
TransferSecret string `yaml:"transfer_secret"`
|
||||
SystemUserID string `yaml:"system_user_id"`
|
||||
AdminUserID string `yaml:"admin_user_id"`
|
||||
Graph GraphService `yaml:"graph"`
|
||||
Idp LdapBasedService `yaml:"idp"`
|
||||
Idm IdmService `yaml:"idm"`
|
||||
Collaboration Collaboration `yaml:"collaboration"`
|
||||
Proxy ProxyService `yaml:"proxy"`
|
||||
Frontend FrontendService `yaml:"frontend"`
|
||||
AuthBasic AuthbasicService `yaml:"auth_basic"`
|
||||
AuthBearer AuthbearerService `yaml:"auth_bearer"`
|
||||
Users UsersAndGroupsService `yaml:"users"`
|
||||
Groups UsersAndGroupsService `yaml:"groups"`
|
||||
Ocdav InsecureService `yaml:"ocdav"`
|
||||
Ocm OcmService `yaml:"ocm"`
|
||||
Thumbnails ThumbnailService `yaml:"thumbnails"`
|
||||
Search Search `yaml:"search"`
|
||||
Audit Audit `yaml:"audit"`
|
||||
Settings SettingsService `yaml:"settings"`
|
||||
Sharing Sharing `yaml:"sharing"`
|
||||
StorageUsers StorageUsers `yaml:"storage_users"`
|
||||
Notifications Notifications `yaml:"notifications"`
|
||||
Nats Nats `yaml:"nats"`
|
||||
Gateway Gateway `yaml:"gateway"`
|
||||
Userlog Userlog `yaml:"userlog"`
|
||||
AuthService AuthService `yaml:"auth_service"`
|
||||
Clientlog Clientlog `yaml:"clientlog"`
|
||||
Activitylog Activitylog `yaml:"activitylog"`
|
||||
}
|
||||
|
||||
// Activitylog is the configuration for the activitylog service
|
||||
type Activitylog struct {
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// App is the configuration for the collaboration service
|
||||
type App struct {
|
||||
Insecure bool `yaml:"insecure"`
|
||||
}
|
||||
|
||||
// Audit is the configuration for the audit service
|
||||
type Audit struct {
|
||||
Events Events
|
||||
}
|
||||
|
||||
// AuthbasicService is the configuration for the authbasic service
|
||||
type AuthbasicService struct {
|
||||
AuthProviders LdapBasedService `yaml:"auth_providers"`
|
||||
}
|
||||
|
||||
// AuthbearerService is the configuration for the authbearer service
|
||||
type AuthbearerService struct {
|
||||
AuthProviders AuthProviderSettings `yaml:"auth_providers"`
|
||||
}
|
||||
|
||||
// AuthProviderSettings is the configuration for the auth provider settings
|
||||
type AuthProviderSettings struct {
|
||||
Oidc InsecureService
|
||||
}
|
||||
|
||||
// AuthService is the configuration for the auth service
|
||||
type AuthService struct {
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// Clientlog is the configuration for the clientlog service
|
||||
type Clientlog struct {
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// Collaboration is the configuration for the collaboration service
|
||||
type Collaboration struct {
|
||||
WopiApp WopiApp `yaml:"wopi"`
|
||||
App App `yaml:"app"`
|
||||
}
|
||||
|
||||
// Events is the configuration for events
|
||||
type Events struct {
|
||||
TLSInsecure bool `yaml:"tls_insecure"`
|
||||
}
|
||||
|
||||
// FrontendService is the configuration for the frontend service
|
||||
type FrontendService struct {
|
||||
AppHandler InsecureService `yaml:"app_handler"`
|
||||
Archiver InsecureService
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// Gateway is the configuration for the gateway
|
||||
type Gateway struct {
|
||||
StorageRegistry StorageRegistry `yaml:"storage_registry"`
|
||||
}
|
||||
|
||||
// GraphApplication is the configuration for the graph application
|
||||
type GraphApplication struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
|
||||
// GraphService is the configuration for the graph service
|
||||
type GraphService struct {
|
||||
Application GraphApplication
|
||||
Events Events
|
||||
Spaces InsecureService
|
||||
Identity LdapBasedService
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// IdmService is the configuration for the IDM service
|
||||
type IdmService struct {
|
||||
ServiceUserPasswords ServiceUserPasswordsSettings `yaml:"service_user_passwords"`
|
||||
}
|
||||
|
||||
// InsecureProxyOIDC is the configuration for the insecure proxy OIDC
|
||||
type InsecureProxyOIDC struct {
|
||||
Insecure bool `yaml:"insecure"`
|
||||
}
|
||||
|
||||
// InsecureService is the configuration for services that can be insecure
|
||||
type InsecureService struct {
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
// LdapBasedService is the configuration for LDAP based services
|
||||
type LdapBasedService struct {
|
||||
Ldap LdapSettings
|
||||
}
|
||||
|
||||
// LdapSettings is the configuration for LDAP settings
|
||||
type LdapSettings struct {
|
||||
BindPassword string `yaml:"bind_password"`
|
||||
}
|
||||
|
||||
// Nats is the configuration for the nats service
|
||||
type Nats struct {
|
||||
// The nats config has a field called nats
|
||||
Nats struct {
|
||||
TLSSkipVerifyClientCert bool `yaml:"tls_skip_verify_client_cert"`
|
||||
}
|
||||
}
|
||||
|
||||
// Notifications is the configuration for the notifications service
|
||||
type Notifications struct {
|
||||
Notifications struct{ Events Events } // The notifications config has a field called notifications
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// OcmService is the configuration for the OCM service
|
||||
type OcmService struct {
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// ProxyService is the configuration for the proxy service
|
||||
type ProxyService struct {
|
||||
OIDC InsecureProxyOIDC `yaml:"oidc"`
|
||||
InsecureBackends bool `yaml:"insecure_backends"`
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// Search is the configuration for the search service
|
||||
type Search struct {
|
||||
Events Events
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// ServiceAccount is the configuration for the used service account
|
||||
type ServiceAccount struct {
|
||||
ServiceAccountID string `yaml:"service_account_id"`
|
||||
ServiceAccountSecret string `yaml:"service_account_secret"`
|
||||
}
|
||||
|
||||
// ServiceUserPasswordsSettings is the configuration for service user passwords
|
||||
type ServiceUserPasswordsSettings struct {
|
||||
AdminPassword string `yaml:"admin_password"`
|
||||
IdmPassword string `yaml:"idm_password"`
|
||||
RevaPassword string `yaml:"reva_password"`
|
||||
IdpPassword string `yaml:"idp_password"`
|
||||
}
|
||||
|
||||
// SettingsService is the configuration for the settings service
|
||||
type SettingsService struct {
|
||||
ServiceAccountIDs []string `yaml:"service_account_ids"`
|
||||
}
|
||||
|
||||
// Sharing is the configuration for the sharing service
|
||||
type Sharing struct {
|
||||
Events Events
|
||||
}
|
||||
|
||||
// StorageRegistry is the configuration for the storage registry
|
||||
type StorageRegistry struct {
|
||||
StorageUsersMountID string `yaml:"storage_users_mount_id"`
|
||||
}
|
||||
|
||||
// StorageUsers is the configuration for the storage users
|
||||
type StorageUsers struct {
|
||||
Events Events
|
||||
MountID string `yaml:"mount_id"`
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// ThumbnailSettings is the configuration for the thumbnail settings
|
||||
type ThumbnailSettings struct {
|
||||
TransferSecret string `yaml:"transfer_secret"`
|
||||
WebdavAllowInsecure bool `yaml:"webdav_allow_insecure"`
|
||||
Cs3AllowInsecure bool `yaml:"cs3_allow_insecure"`
|
||||
}
|
||||
|
||||
// ThumbnailService is the configuration for the thumbnail service
|
||||
type ThumbnailService struct {
|
||||
Thumbnail ThumbnailSettings
|
||||
}
|
||||
|
||||
// TokenManager is the configuration for the token manager
|
||||
type TokenManager struct {
|
||||
JWTSecret string `yaml:"jwt_secret"`
|
||||
}
|
||||
|
||||
// UsersAndGroupsService is the configuration for the users and groups service
|
||||
type UsersAndGroupsService struct {
|
||||
Drivers LdapBasedService
|
||||
}
|
||||
|
||||
// Userlog is the configuration for the userlog service
|
||||
type Userlog struct {
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
}
|
||||
|
||||
// WopiApp is the configuration for the WOPI app
|
||||
type WopiApp struct {
|
||||
Secret string `yaml:"secret"`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package register
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
// Commands defines the slice of commands.
|
||||
Commands = []Command{}
|
||||
)
|
||||
|
||||
// Command defines the register command.
|
||||
type Command func(*config.Config) *cli.Command
|
||||
|
||||
// AddCommand appends a command to Commands.
|
||||
func AddCommand(cmd Command) {
|
||||
Commands = append(
|
||||
Commands,
|
||||
cmd,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// Package revisions allows manipulating revisions in a storage provider.
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
// regex to determine if a node versioned. Examples:
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk
|
||||
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock
|
||||
_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`)
|
||||
)
|
||||
|
||||
// DelBlobstore is the interface for a blobstore that can delete blobs.
|
||||
type DelBlobstore interface {
|
||||
Delete(node *node.Node) error
|
||||
}
|
||||
|
||||
// Glob uses globbing to find all revision nodes in a storage provider.
|
||||
func Glob(pattern string) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
nodes, err := filepath.Glob(filepath.Join(pattern))
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", pattern, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
fmt.Println("no nodes found. Double check storage path")
|
||||
return
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
if _versionRegex.MatchString(n) {
|
||||
ch <- n
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// GlobWorkers uses multiple go routine to glob all revision nodes in a storage provider.
|
||||
func GlobWorkers(pattern string, depth string, remainder string) <-chan string {
|
||||
wg := sync.WaitGroup{}
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
nodes, err := filepath.Glob(pattern + depth)
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", pattern, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
fmt.Println("no nodes found. Double check storage path")
|
||||
return
|
||||
}
|
||||
|
||||
for _, node := range nodes {
|
||||
wg.Add(1)
|
||||
go func(node string) {
|
||||
defer wg.Done()
|
||||
nodes, err := filepath.Glob(node + remainder)
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", node, err)
|
||||
return
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if _versionRegex.MatchString(n) {
|
||||
ch <- n
|
||||
}
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Walk walks the storage provider to find all revision nodes.
|
||||
func Walk(base string) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if !_versionRegex.MatchString(info.Name()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
ch <- path
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return
|
||||
}
|
||||
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
// List uses directory listing to find all revision nodes in a storage provider.
|
||||
func List(base string, workers int) <-chan string {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
if err := listFolder(base, ch, make(chan struct{}, workers)); err != nil {
|
||||
fmt.Println("error listing", base, err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// PurgeRevisions removes all revisions from a storage provider.
|
||||
func PurgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) {
|
||||
countFiles := 0
|
||||
countBlobs := 0
|
||||
countRevisions := 0
|
||||
|
||||
var err error
|
||||
for d := range nodes {
|
||||
if !_versionRegex.MatchString(d) {
|
||||
continue
|
||||
}
|
||||
|
||||
var blobID string
|
||||
e := filepath.Ext(d)
|
||||
switch e {
|
||||
case ".mpk":
|
||||
blobID, err = getBlobID(d)
|
||||
if err != nil {
|
||||
fmt.Printf("error getting blobID from %s: %v\n", d, err)
|
||||
continue
|
||||
}
|
||||
|
||||
countBlobs++
|
||||
case ".mlock":
|
||||
// no extra action on .mlock files
|
||||
default:
|
||||
countRevisions++
|
||||
}
|
||||
|
||||
if !dryRun {
|
||||
if blobID != "" {
|
||||
// TODO: needs spaceID for s3ng
|
||||
if err := bs.Delete(&node.Node{BlobID: blobID}); err != nil {
|
||||
fmt.Printf("error deleting blob %s: %v\n", blobID, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Remove(d); err != nil {
|
||||
fmt.Printf("error removing %s: %v\n", d, err)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
countFiles++
|
||||
|
||||
if verbose {
|
||||
spaceID, nodeID := getIDsFromPath(d)
|
||||
if dryRun {
|
||||
fmt.Println("Would delete")
|
||||
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
|
||||
fmt.Println("\tSpaceID:", spaceID)
|
||||
fmt.Println("\tPath:", d)
|
||||
if blobID != "" {
|
||||
fmt.Println("\tBlob:", blobID)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Deleted")
|
||||
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
|
||||
fmt.Println("\tSpaceID:", spaceID)
|
||||
fmt.Println("\tPath:", d)
|
||||
if blobID != "" {
|
||||
fmt.Println("\tBlob:", blobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return countFiles, countBlobs, countRevisions
|
||||
}
|
||||
|
||||
func listFolder(path string, ch chan<- string, workers chan struct{}) error {
|
||||
workers <- struct{}{}
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
children, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
<-workers
|
||||
return err
|
||||
}
|
||||
|
||||
for _, child := range children {
|
||||
if child.IsDir() {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := listFolder(filepath.Join(path, child.Name()), ch, workers); err != nil {
|
||||
fmt.Println("error listing", path, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if _versionRegex.MatchString(child.Name()) {
|
||||
ch <- filepath.Join(path, child.Name())
|
||||
}
|
||||
|
||||
}
|
||||
<-workers
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBlobID(path string) (string, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
m := map[string][]byte{}
|
||||
if err := msgpack.Unmarshal(b, &m); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if bid := m["user.ocis.blobid"]; string(bid) != "" {
|
||||
return string(bid), nil
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func getIDsFromPath(path string) (string, string) {
|
||||
rawIDs := strings.Split(path, "/nodes/")
|
||||
if len(rawIDs) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
s := strings.Split(rawIDs[0], "/spaces/")
|
||||
if len(s) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
n := strings.Split(rawIDs[1], ".REV.")
|
||||
if len(n) != 2 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
spaceID := strings.Replace(s[1], "/", "", -1)
|
||||
nodeID := strings.Replace(n[0], "/", "", -1)
|
||||
return spaceID, filepath.Base(nodeID)
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/google/uuid"
|
||||
"github.com/test-go/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
_basePath = "/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/"
|
||||
)
|
||||
|
||||
// func TestInit(t *testing.T) {
|
||||
// initialize(10, 2)
|
||||
// defer os.RemoveAll("test_temp")
|
||||
// }
|
||||
|
||||
func TestGlob30(t *testing.T) { test(t, 10, 2, glob) }
|
||||
func TestGlob80(t *testing.T) { test(t, 20, 3, glob) }
|
||||
func TestGlob250(t *testing.T) { test(t, 50, 4, glob) }
|
||||
func TestGlob600(t *testing.T) { test(t, 100, 5, glob) }
|
||||
|
||||
func TestWalk30(t *testing.T) { test(t, 10, 2, walk) }
|
||||
func TestWalk80(t *testing.T) { test(t, 20, 3, walk) }
|
||||
func TestWalk250(t *testing.T) { test(t, 50, 4, walk) }
|
||||
func TestWalk600(t *testing.T) { test(t, 100, 5, walk) }
|
||||
|
||||
func TestList30(t *testing.T) { test(t, 10, 2, list2) }
|
||||
func TestList80(t *testing.T) { test(t, 20, 3, list10) }
|
||||
func TestList250(t *testing.T) { test(t, 50, 4, list20) }
|
||||
func TestList600(t *testing.T) { test(t, 100, 5, list2) }
|
||||
|
||||
func TestGlobWorkers30(t *testing.T) { test(t, 10, 2, globWorkersD1) }
|
||||
func TestGlobWorkers80(t *testing.T) { test(t, 20, 3, globWorkersD2) }
|
||||
func TestGlobWorkers250(t *testing.T) { test(t, 50, 4, globWorkersD4) }
|
||||
func TestGlobWorkers600(t *testing.T) { test(t, 100, 5, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob30(b *testing.B) { benchmark(b, 10, 2, glob) }
|
||||
func BenchmarkWalk30(b *testing.B) { benchmark(b, 10, 2, walk) }
|
||||
func BenchmarkList30(b *testing.B) { benchmark(b, 10, 2, list2) }
|
||||
func BenchmarkGlobWorkers30(b *testing.B) { benchmark(b, 10, 2, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob80(b *testing.B) { benchmark(b, 20, 3, glob) }
|
||||
func BenchmarkWalk80(b *testing.B) { benchmark(b, 20, 3, walk) }
|
||||
func BenchmarkList80(b *testing.B) { benchmark(b, 20, 3, list2) }
|
||||
func BenchmarkGlobWorkers80(b *testing.B) { benchmark(b, 20, 3, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlob250(b *testing.B) { benchmark(b, 50, 4, glob) }
|
||||
func BenchmarkWalk250(b *testing.B) { benchmark(b, 50, 4, walk) }
|
||||
func BenchmarkList250(b *testing.B) { benchmark(b, 50, 4, list2) }
|
||||
func BenchmarkGlobWorkers250(b *testing.B) { benchmark(b, 50, 4, globWorkersD2) }
|
||||
|
||||
func BenchmarkGlobAT600(b *testing.B) { benchmark(b, 100, 5, glob) }
|
||||
func BenchmarkWalkAT600(b *testing.B) { benchmark(b, 100, 5, walk) }
|
||||
func BenchmarkList2AT600(b *testing.B) { benchmark(b, 100, 5, list2) }
|
||||
func BenchmarkList10AT600(b *testing.B) { benchmark(b, 100, 5, list10) }
|
||||
func BenchmarkList20AT600(b *testing.B) { benchmark(b, 100, 5, list20) }
|
||||
func BenchmarkGlobWorkersD1AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD1) }
|
||||
func BenchmarkGlobWorkersD2AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD2) }
|
||||
func BenchmarkGlobWorkersD4AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD4) }
|
||||
|
||||
func BenchmarkGlobAT22000(b *testing.B) { benchmark(b, 2000, 10, glob) }
|
||||
func BenchmarkWalkAT22000(b *testing.B) { benchmark(b, 2000, 10, walk) }
|
||||
func BenchmarkList2AT22000(b *testing.B) { benchmark(b, 2000, 10, list2) }
|
||||
func BenchmarkList10AT22000(b *testing.B) { benchmark(b, 2000, 10, list10) }
|
||||
func BenchmarkList20AT22000(b *testing.B) { benchmark(b, 2000, 10, list20) }
|
||||
func BenchmarkGlobWorkersD1AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD1) }
|
||||
func BenchmarkGlobWorkersD2AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD2) }
|
||||
func BenchmarkGlobWorkersD4AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD4) }
|
||||
|
||||
func BenchmarkGlob110000(b *testing.B) { benchmark(b, 10000, 10, glob) }
|
||||
func BenchmarkWalk110000(b *testing.B) { benchmark(b, 10000, 10, walk) }
|
||||
func BenchmarkList110000(b *testing.B) { benchmark(b, 10000, 10, list2) }
|
||||
func BenchmarkGlobWorkers110000(b *testing.B) { benchmark(b, 10000, 10, globWorkersD2) }
|
||||
|
||||
func benchmark(b *testing.B, numNodes int, numRevisions int, f func(string) <-chan string) {
|
||||
base := initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll(base)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
ch := f(base)
|
||||
PurgeRevisions(ch, nil, false, false)
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func test(t *testing.T, numNodes int, numRevisions int, f func(string) <-chan string) {
|
||||
base := initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll(base)
|
||||
|
||||
ch := f(base)
|
||||
_, _, revisions := PurgeRevisions(ch, nil, false, false)
|
||||
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
|
||||
}
|
||||
|
||||
func glob(base string) <-chan string {
|
||||
return Glob(base + _basePath + "*/*/*/*/*")
|
||||
}
|
||||
|
||||
func walk(base string) <-chan string {
|
||||
return Walk(base + _basePath)
|
||||
}
|
||||
|
||||
func list2(base string) <-chan string {
|
||||
return List(base+_basePath, 2)
|
||||
}
|
||||
|
||||
func list10(base string) <-chan string {
|
||||
return List(base+_basePath, 10)
|
||||
}
|
||||
|
||||
func list20(base string) <-chan string {
|
||||
return List(base+_basePath, 20)
|
||||
}
|
||||
|
||||
func globWorkersD1(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*", "/*/*/*/*")
|
||||
}
|
||||
|
||||
func globWorkersD2(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*/*", "/*/*/*")
|
||||
}
|
||||
|
||||
func globWorkersD4(base string) <-chan string {
|
||||
return GlobWorkers(base+_basePath, "*/*/*/*", "/*")
|
||||
}
|
||||
|
||||
func initialize(numNodes int, numRevisions int) string {
|
||||
base := "test_temp_" + uuid.New().String()
|
||||
if err := os.Mkdir(base, os.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// create base path
|
||||
if err := os.MkdirAll(base+_basePath, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating base path", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
path := lookup.Pathify(uuid.New().String(), 4, 2)
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(base+_basePath+dir, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if _, err := os.Create(base + _basePath + path); err != nil {
|
||||
fmt.Println("Error creating file", err)
|
||||
os.RemoveAll(base)
|
||||
os.Exit(1)
|
||||
}
|
||||
for i := 0; i < numRevisions; i++ {
|
||||
os.Create(base + _basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z")
|
||||
}
|
||||
}
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
# ownCloud Infinite Scale: Runtime
|
||||
|
||||
Pman is a slim utility library for supervising long-running processes. It can be [embedded](https://github.com/owncloud/OCIS/blob/ea2a2b328e7261ed72e65adf48359c0a44e14b40/OCIS/pkg/runtime/runtime.go#L84) or used as a cli command.
|
||||
|
||||
When used as a CLI command it relays actions to a running runtime.
|
||||
|
||||
## Usage
|
||||
|
||||
Start a runtime
|
||||
|
||||
```go
|
||||
package main
|
||||
import "github.com/owncloud/ocis/ocis/pkg/runtime/service"
|
||||
|
||||
func main() {
|
||||
service.Start()
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
Start sending messages
|
||||
|
||||
|
||||
## Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/owncloud/ocis/ocis/pkg/runtime/process"
|
||||
"github.com/owncloud/ocis/ocis/pkg/runtime/service"
|
||||
"github.com/rs/zerolog/log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
s := service.NewService()
|
||||
var c = make(chan os.Signal, 1)
|
||||
var o int
|
||||
|
||||
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
|
||||
if err := s.Start(process.NewProcEntry("ocs", nil, "ocs"), &o); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
time.AfterFunc(3*time.Second, func() {
|
||||
var acc = "ocs"
|
||||
fmt.Printf(fmt.Sprintf("shutting down service: %s", acc))
|
||||
if err := s.Controller.Kill(&acc); err != nil {
|
||||
log.Fatal()
|
||||
}
|
||||
os.Exit(0)
|
||||
})
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Run the example above with `RUNTIME_KEEP_ALIVE=true` and with no `RUNTIME_KEEP_ALIVE` set to see its behavior. It requires an [oCIS binary](https://github.com/owncloud/ocis/releases) present in your `$PATH` for it to work.
|
||||
@@ -0,0 +1,34 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/rpc"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/runtime/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// List running service.
|
||||
func List(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "list",
|
||||
Aliases: []string{"r"},
|
||||
Short: "List running services",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
client, err := rpc.DialHTTP("tcp", net.JoinHostPort(cfg.Hostname, cfg.Port))
|
||||
if err != nil {
|
||||
log.Fatal("dialing:", err)
|
||||
}
|
||||
|
||||
var arg1 string
|
||||
|
||||
if err := client.Call("Service.List", struct{}{}, &arg1); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(arg1)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package config
|
||||
|
||||
// Config determines behavior across the tool.
|
||||
type Config struct {
|
||||
// Hostname where the runtime is running. When using PMAN in cli mode, it determines where the host runtime is.
|
||||
// Default is localhost.
|
||||
Hostname string
|
||||
|
||||
// Port configures the port where a runtime is available. It defaults to 10666.
|
||||
Port string
|
||||
|
||||
// KeepAlive configures if restart attempts are made if the process supervised terminates. Default is false.
|
||||
KeepAlive bool
|
||||
}
|
||||
|
||||
var (
|
||||
defaultHostname = "localhost"
|
||||
defaultPort = "10666"
|
||||
)
|
||||
|
||||
// NewConfig returns a new config with a set of defaults.
|
||||
func NewConfig() *Config {
|
||||
return &Config{
|
||||
Hostname: defaultHostname,
|
||||
Port: defaultPort,
|
||||
KeepAlive: false,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/log"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
// Options is a runtime option
|
||||
type Options struct {
|
||||
Services []string
|
||||
Logger log.Logger
|
||||
Context *cli.Context
|
||||
}
|
||||
|
||||
// Option undocumented
|
||||
type Option func(o *Options)
|
||||
|
||||
// Services option
|
||||
func Services(s []string) Option {
|
||||
return func(o *Options) {
|
||||
o.Services = append(o.Services, s...)
|
||||
}
|
||||
}
|
||||
|
||||
// Context option
|
||||
func Context(c *cli.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = c
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/runtime/service"
|
||||
)
|
||||
|
||||
// Runtime represents an oCIS runtime environment.
|
||||
type Runtime struct {
|
||||
c *config.Config
|
||||
}
|
||||
|
||||
// New creates a new oCIS + micro runtime
|
||||
func New(cfg *config.Config) Runtime {
|
||||
return Runtime{
|
||||
c: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// Start rpc runtime
|
||||
func (r *Runtime) Start(ctx context.Context) error {
|
||||
return service.Start(ctx, service.WithConfig(r.c))
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
)
|
||||
|
||||
// Log configures a structure logger.
|
||||
type Log struct {
|
||||
Pretty bool
|
||||
}
|
||||
|
||||
// Options are the configurable options for a Service.
|
||||
type Options struct {
|
||||
Log *Log
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// Option represents an option.
|
||||
type Option func(o *Options)
|
||||
|
||||
// NewOptions returns a new Options struct.
|
||||
func NewOptions() *Options {
|
||||
return &Options{
|
||||
Log: &Log{},
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogPretty sets Controller config.
|
||||
func WithLogPretty(pretty bool) Option {
|
||||
return func(o *Options) {
|
||||
o.Log.Pretty = pretty
|
||||
}
|
||||
}
|
||||
|
||||
// WithConfig sets Controller config.
|
||||
func WithConfig(cfg *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = cfg
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/rpc"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
authapp "github.com/opencloud-eu/opencloud/services/auth-app/pkg/command"
|
||||
|
||||
"github.com/cenkalti/backoff"
|
||||
"github.com/cs3org/reva/v2/pkg/events/stream"
|
||||
"github.com/cs3org/reva/v2/pkg/logger"
|
||||
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/mohae/deepcopy"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
notifications "github.com/opencloud-eu/opencloud/services/notifications/pkg/command"
|
||||
"github.com/thejerf/suture/v4"
|
||||
|
||||
ociscfg "github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/log"
|
||||
ogrpc "github.com/opencloud-eu/opencloud/ocis-pkg/service/grpc"
|
||||
"github.com/opencloud-eu/opencloud/ocis-pkg/shared"
|
||||
activitylog "github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
|
||||
antivirus "github.com/opencloud-eu/opencloud/services/antivirus/pkg/command"
|
||||
appProvider "github.com/opencloud-eu/opencloud/services/app-provider/pkg/command"
|
||||
appRegistry "github.com/opencloud-eu/opencloud/services/app-registry/pkg/command"
|
||||
audit "github.com/opencloud-eu/opencloud/services/audit/pkg/command"
|
||||
authbasic "github.com/opencloud-eu/opencloud/services/auth-basic/pkg/command"
|
||||
authmachine "github.com/opencloud-eu/opencloud/services/auth-machine/pkg/command"
|
||||
authservice "github.com/opencloud-eu/opencloud/services/auth-service/pkg/command"
|
||||
clientlog "github.com/opencloud-eu/opencloud/services/clientlog/pkg/command"
|
||||
eventhistory "github.com/opencloud-eu/opencloud/services/eventhistory/pkg/command"
|
||||
frontend "github.com/opencloud-eu/opencloud/services/frontend/pkg/command"
|
||||
gateway "github.com/opencloud-eu/opencloud/services/gateway/pkg/command"
|
||||
graph "github.com/opencloud-eu/opencloud/services/graph/pkg/command"
|
||||
groups "github.com/opencloud-eu/opencloud/services/groups/pkg/command"
|
||||
idm "github.com/opencloud-eu/opencloud/services/idm/pkg/command"
|
||||
idp "github.com/opencloud-eu/opencloud/services/idp/pkg/command"
|
||||
invitations "github.com/opencloud-eu/opencloud/services/invitations/pkg/command"
|
||||
nats "github.com/opencloud-eu/opencloud/services/nats/pkg/command"
|
||||
ocdav "github.com/opencloud-eu/opencloud/services/ocdav/pkg/command"
|
||||
ocm "github.com/opencloud-eu/opencloud/services/ocm/pkg/command"
|
||||
ocs "github.com/opencloud-eu/opencloud/services/ocs/pkg/command"
|
||||
policies "github.com/opencloud-eu/opencloud/services/policies/pkg/command"
|
||||
postprocessing "github.com/opencloud-eu/opencloud/services/postprocessing/pkg/command"
|
||||
proxy "github.com/opencloud-eu/opencloud/services/proxy/pkg/command"
|
||||
search "github.com/opencloud-eu/opencloud/services/search/pkg/command"
|
||||
settings "github.com/opencloud-eu/opencloud/services/settings/pkg/command"
|
||||
sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/command"
|
||||
sse "github.com/opencloud-eu/opencloud/services/sse/pkg/command"
|
||||
storagepublic "github.com/opencloud-eu/opencloud/services/storage-publiclink/pkg/command"
|
||||
storageshares "github.com/opencloud-eu/opencloud/services/storage-shares/pkg/command"
|
||||
storageSystem "github.com/opencloud-eu/opencloud/services/storage-system/pkg/command"
|
||||
storageusers "github.com/opencloud-eu/opencloud/services/storage-users/pkg/command"
|
||||
thumbnails "github.com/opencloud-eu/opencloud/services/thumbnails/pkg/command"
|
||||
userlog "github.com/opencloud-eu/opencloud/services/userlog/pkg/command"
|
||||
users "github.com/opencloud-eu/opencloud/services/users/pkg/command"
|
||||
web "github.com/opencloud-eu/opencloud/services/web/pkg/command"
|
||||
webdav "github.com/opencloud-eu/opencloud/services/webdav/pkg/command"
|
||||
webfinger "github.com/opencloud-eu/opencloud/services/webfinger/pkg/command"
|
||||
)
|
||||
|
||||
var (
|
||||
// runset keeps track of which services to start supervised.
|
||||
runset map[string]struct{}
|
||||
|
||||
// wait funcs run after the service group has been started.
|
||||
_waitFuncs = []func(*ociscfg.Config) error{pingNats, pingGateway, nil, wait(time.Second), nil}
|
||||
)
|
||||
|
||||
type serviceFuncMap map[string]func(*ociscfg.Config) suture.Service
|
||||
|
||||
// Service represents a RPC service.
|
||||
type Service struct {
|
||||
Supervisor *suture.Supervisor
|
||||
Services []serviceFuncMap
|
||||
Additional serviceFuncMap
|
||||
Log log.Logger
|
||||
|
||||
serviceToken map[string][]suture.ServiceToken
|
||||
context context.Context
|
||||
cancel context.CancelFunc
|
||||
cfg *ociscfg.Config
|
||||
}
|
||||
|
||||
// NewService returns a configured service with a controller and a default logger.
|
||||
// When used as a library, flags are not parsed, and in order to avoid introducing a global state with init functions
|
||||
// calls are done explicitly to loadFromEnv().
|
||||
// Since this is the public constructor, options need to be added, at the moment only logging options
|
||||
// are supported in order to match the running OwnCloud services structured log.
|
||||
func NewService(ctx context.Context, options ...Option) (*Service, error) {
|
||||
opts := NewOptions()
|
||||
|
||||
for _, f := range options {
|
||||
f(opts)
|
||||
}
|
||||
|
||||
l := log.NewLogger(
|
||||
log.Color(opts.Config.Log.Color),
|
||||
log.Pretty(opts.Config.Log.Pretty),
|
||||
log.Level(opts.Config.Log.Level),
|
||||
)
|
||||
|
||||
globalCtx, cancelGlobal := context.WithCancel(ctx)
|
||||
|
||||
s := &Service{
|
||||
Services: make([]serviceFuncMap, len(_waitFuncs)),
|
||||
Additional: make(serviceFuncMap),
|
||||
Log: l,
|
||||
|
||||
serviceToken: make(map[string][]suture.ServiceToken),
|
||||
context: globalCtx,
|
||||
cancel: cancelGlobal,
|
||||
cfg: opts.Config,
|
||||
}
|
||||
|
||||
// populate services
|
||||
reg := func(priority int, name string, exec func(context.Context, *ociscfg.Config) error) {
|
||||
if s.Services[priority] == nil {
|
||||
s.Services[priority] = make(serviceFuncMap)
|
||||
}
|
||||
s.Services[priority][name] = NewSutureServiceBuilder(exec)
|
||||
}
|
||||
|
||||
// nats is in priority group 0. It needs to start before all other services
|
||||
reg(0, opts.Config.Nats.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Nats.Context = ctx
|
||||
cfg.Nats.Commons = cfg.Commons
|
||||
return nats.Execute(cfg.Nats)
|
||||
})
|
||||
|
||||
// gateway is in priority group 1. It needs to start before the reva services
|
||||
reg(1, opts.Config.Gateway.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Gateway.Context = ctx
|
||||
cfg.Gateway.Commons = cfg.Commons
|
||||
return gateway.Execute(cfg.Gateway)
|
||||
})
|
||||
|
||||
// priority group 2 is empty for now
|
||||
|
||||
// most services are in priority group 3
|
||||
reg(3, opts.Config.Activitylog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Activitylog.Context = ctx
|
||||
cfg.Activitylog.Commons = cfg.Commons
|
||||
return activitylog.Execute(cfg.Activitylog)
|
||||
})
|
||||
reg(3, opts.Config.AppProvider.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AppProvider.Context = ctx
|
||||
cfg.AppProvider.Commons = cfg.Commons
|
||||
return appProvider.Execute(cfg.AppProvider)
|
||||
})
|
||||
reg(3, opts.Config.AppRegistry.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AppRegistry.Context = ctx
|
||||
cfg.AppRegistry.Commons = cfg.Commons
|
||||
return appRegistry.Execute(cfg.AppRegistry)
|
||||
})
|
||||
reg(3, opts.Config.AuthBasic.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AuthBasic.Context = ctx
|
||||
cfg.AuthBasic.Commons = cfg.Commons
|
||||
return authbasic.Execute(cfg.AuthBasic)
|
||||
})
|
||||
reg(3, opts.Config.AuthMachine.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AuthMachine.Context = ctx
|
||||
cfg.AuthMachine.Commons = cfg.Commons
|
||||
return authmachine.Execute(cfg.AuthMachine)
|
||||
})
|
||||
reg(3, opts.Config.AuthService.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AuthService.Context = ctx
|
||||
cfg.AuthService.Commons = cfg.Commons
|
||||
return authservice.Execute(cfg.AuthService)
|
||||
})
|
||||
reg(3, opts.Config.Clientlog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Clientlog.Context = ctx
|
||||
cfg.Clientlog.Commons = cfg.Commons
|
||||
return clientlog.Execute(cfg.Clientlog)
|
||||
})
|
||||
reg(3, opts.Config.EventHistory.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.EventHistory.Context = ctx
|
||||
cfg.EventHistory.Commons = cfg.Commons
|
||||
return eventhistory.Execute(cfg.EventHistory)
|
||||
})
|
||||
reg(3, opts.Config.Graph.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Graph.Context = ctx
|
||||
cfg.Graph.Commons = cfg.Commons
|
||||
return graph.Execute(cfg.Graph)
|
||||
})
|
||||
reg(3, opts.Config.Groups.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Groups.Context = ctx
|
||||
cfg.Groups.Commons = cfg.Commons
|
||||
return groups.Execute(cfg.Groups)
|
||||
})
|
||||
reg(3, opts.Config.IDM.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.IDM.Context = ctx
|
||||
cfg.IDM.Commons = cfg.Commons
|
||||
return idm.Execute(cfg.IDM)
|
||||
})
|
||||
reg(3, opts.Config.OCDav.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.OCDav.Context = ctx
|
||||
cfg.OCDav.Commons = cfg.Commons
|
||||
return ocdav.Execute(cfg.OCDav)
|
||||
})
|
||||
reg(3, opts.Config.OCS.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.OCS.Context = ctx
|
||||
cfg.OCS.Commons = cfg.Commons
|
||||
return ocs.Execute(cfg.OCS)
|
||||
})
|
||||
reg(3, opts.Config.Postprocessing.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Postprocessing.Context = ctx
|
||||
cfg.Postprocessing.Commons = cfg.Commons
|
||||
return postprocessing.Execute(cfg.Postprocessing)
|
||||
})
|
||||
reg(3, opts.Config.Search.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Search.Context = ctx
|
||||
cfg.Search.Commons = cfg.Commons
|
||||
return search.Execute(cfg.Search)
|
||||
})
|
||||
reg(3, opts.Config.Settings.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Settings.Context = ctx
|
||||
cfg.Settings.Commons = cfg.Commons
|
||||
return settings.Execute(cfg.Settings)
|
||||
})
|
||||
reg(3, opts.Config.StoragePublicLink.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.StoragePublicLink.Context = ctx
|
||||
cfg.StoragePublicLink.Commons = cfg.Commons
|
||||
return storagepublic.Execute(cfg.StoragePublicLink)
|
||||
})
|
||||
reg(3, opts.Config.StorageShares.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.StorageShares.Context = ctx
|
||||
cfg.StorageShares.Commons = cfg.Commons
|
||||
return storageshares.Execute(cfg.StorageShares)
|
||||
})
|
||||
reg(3, opts.Config.StorageSystem.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.StorageSystem.Context = ctx
|
||||
cfg.StorageSystem.Commons = cfg.Commons
|
||||
return storageSystem.Execute(cfg.StorageSystem)
|
||||
})
|
||||
reg(3, opts.Config.StorageUsers.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.StorageUsers.Context = ctx
|
||||
cfg.StorageUsers.Commons = cfg.Commons
|
||||
return storageusers.Execute(cfg.StorageUsers)
|
||||
})
|
||||
reg(3, opts.Config.Thumbnails.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Thumbnails.Context = ctx
|
||||
cfg.Thumbnails.Commons = cfg.Commons
|
||||
return thumbnails.Execute(cfg.Thumbnails)
|
||||
})
|
||||
reg(3, opts.Config.Userlog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Userlog.Context = ctx
|
||||
cfg.Userlog.Commons = cfg.Commons
|
||||
return userlog.Execute(cfg.Userlog)
|
||||
})
|
||||
reg(3, opts.Config.Users.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Users.Context = ctx
|
||||
cfg.Users.Commons = cfg.Commons
|
||||
return users.Execute(cfg.Users)
|
||||
})
|
||||
reg(3, opts.Config.Web.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Web.Context = ctx
|
||||
cfg.Web.Commons = cfg.Commons
|
||||
return web.Execute(cfg.Web)
|
||||
})
|
||||
reg(3, opts.Config.WebDAV.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.WebDAV.Context = ctx
|
||||
cfg.WebDAV.Commons = cfg.Commons
|
||||
return webdav.Execute(cfg.WebDAV)
|
||||
})
|
||||
reg(3, opts.Config.Webfinger.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Webfinger.Context = ctx
|
||||
cfg.Webfinger.Commons = cfg.Commons
|
||||
return webfinger.Execute(cfg.Webfinger)
|
||||
})
|
||||
reg(3, opts.Config.IDP.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.IDP.Context = ctx
|
||||
cfg.IDP.Commons = cfg.Commons
|
||||
return idp.Execute(cfg.IDP)
|
||||
})
|
||||
reg(3, opts.Config.Proxy.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Proxy.Context = ctx
|
||||
cfg.Proxy.Commons = cfg.Commons
|
||||
return proxy.Execute(cfg.Proxy)
|
||||
})
|
||||
reg(3, opts.Config.Sharing.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Sharing.Context = ctx
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return sharing.Execute(cfg.Sharing)
|
||||
})
|
||||
reg(3, opts.Config.SSE.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.SSE.Context = ctx
|
||||
cfg.SSE.Commons = cfg.Commons
|
||||
return sse.Execute(cfg.SSE)
|
||||
})
|
||||
reg(3, opts.Config.OCM.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.OCM.Context = ctx
|
||||
cfg.OCM.Commons = cfg.Commons
|
||||
return ocm.Execute(cfg.OCM)
|
||||
})
|
||||
|
||||
// out of some unknown reason ci gets angry when frontend service starts in priority group 3
|
||||
// this is not reproducible locally, it can start when nats and gateway are already running
|
||||
// FIXME: find out why
|
||||
reg(4, opts.Config.Frontend.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Frontend.Context = ctx
|
||||
cfg.Frontend.Commons = cfg.Commons
|
||||
return frontend.Execute(cfg.Frontend)
|
||||
})
|
||||
|
||||
// populate optional services
|
||||
areg := func(name string, exec func(context.Context, *ociscfg.Config) error) {
|
||||
s.Additional[name] = NewSutureServiceBuilder(exec)
|
||||
}
|
||||
areg(opts.Config.Antivirus.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Antivirus.Context = ctx
|
||||
// cfg.Antivirus.Commons = cfg.Commons // antivirus holds no Commons atm
|
||||
return antivirus.Execute(cfg.Antivirus)
|
||||
})
|
||||
areg(opts.Config.Audit.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Audit.Context = ctx
|
||||
cfg.Audit.Commons = cfg.Commons
|
||||
return audit.Execute(cfg.Audit)
|
||||
})
|
||||
areg(opts.Config.AuthApp.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.AuthApp.Context = ctx
|
||||
cfg.AuthApp.Commons = cfg.Commons
|
||||
return authapp.Execute(cfg.AuthApp)
|
||||
})
|
||||
areg(opts.Config.Policies.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Policies.Context = ctx
|
||||
cfg.Policies.Commons = cfg.Commons
|
||||
return policies.Execute(cfg.Policies)
|
||||
})
|
||||
areg(opts.Config.Invitations.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Invitations.Context = ctx
|
||||
cfg.Invitations.Commons = cfg.Commons
|
||||
return invitations.Execute(cfg.Invitations)
|
||||
})
|
||||
areg(opts.Config.Notifications.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error {
|
||||
cfg.Notifications.Context = ctx
|
||||
cfg.Notifications.Commons = cfg.Commons
|
||||
return notifications.Execute(cfg.Notifications)
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Start a rpc service. By default, the package scope Start will run all default services to provide with a working
|
||||
// oCIS instance.
|
||||
func Start(ctx context.Context, o ...Option) error {
|
||||
// Start the runtime. Most likely this was called ONLY by the `ocis server` subcommand, but since we cannot protect
|
||||
// from the caller, the previous statement holds truth.
|
||||
|
||||
// prepare a new rpc Service struct.
|
||||
s, err := NewService(ctx, o...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get a cancel function to stop the service
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
// tolerance controls backoff cycles from the supervisor.
|
||||
tolerance := 5
|
||||
totalBackoff := 0
|
||||
|
||||
// Start creates its own supervisor. Running services under `ocis server` will create its own supervision tree.
|
||||
s.Supervisor = suture.New("ocis", suture.Spec{
|
||||
EventHook: func(e suture.Event) {
|
||||
if e.Type() == suture.EventTypeBackoff {
|
||||
totalBackoff++
|
||||
if totalBackoff == tolerance {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
s.Log.Info().Str("event", e.String()).Msg(fmt.Sprintf("supervisor: %v", e.Map()["supervisor_name"]))
|
||||
},
|
||||
FailureThreshold: 5,
|
||||
FailureBackoff: 3 * time.Second,
|
||||
})
|
||||
|
||||
if s.cfg.Commons == nil {
|
||||
s.cfg.Commons = &shared.Commons{
|
||||
Log: &shared.Log{},
|
||||
}
|
||||
}
|
||||
|
||||
if err = rpc.Register(s); err != nil {
|
||||
if s != nil {
|
||||
s.Log.Fatal().Err(err).Msg("could not register rpc service")
|
||||
}
|
||||
}
|
||||
rpc.HandleHTTP()
|
||||
|
||||
l, err := net.Listen("tcp", net.JoinHostPort(s.cfg.Runtime.Host, s.cfg.Runtime.Port))
|
||||
if err != nil {
|
||||
s.Log.Fatal().Err(err).Msg("could not start listener")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
reason := strings.Builder{}
|
||||
if _, err = net.Dial("tcp", net.JoinHostPort(s.cfg.Runtime.Host, s.cfg.Runtime.Port)); err != nil {
|
||||
reason.WriteString("runtime address already in use")
|
||||
}
|
||||
|
||||
fmt.Println(reason.String())
|
||||
}
|
||||
}()
|
||||
|
||||
// prepare the set of services to run
|
||||
s.generateRunSet(s.cfg)
|
||||
|
||||
// there are reasons not to do this, but we have race conditions ourselves. Until we resolve them, mind the following disclaimer:
|
||||
// Calling ServeBackground will CORRECTLY start the supervisor running in a new goroutine. It is risky to directly run
|
||||
// go supervisor.Serve()
|
||||
// because that will briefly create a race condition as it starts up, if you try to .Add() services immediately afterward.
|
||||
// https://pkg.go.dev/github.com/thejerf/suture/v4@v4.0.0#Supervisor
|
||||
go s.Supervisor.ServeBackground(s.context)
|
||||
|
||||
// trap will block on context done channel for interruptions.
|
||||
go trap(s, ctx)
|
||||
|
||||
for i, service := range s.Services {
|
||||
scheduleServiceTokens(s, service)
|
||||
if _waitFuncs[i] != nil {
|
||||
if err := _waitFuncs[i](s.cfg); err != nil {
|
||||
s.Log.Fatal().Err(err).Msg("wait func failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule services that are optional
|
||||
scheduleServiceTokens(s, s.Additional)
|
||||
|
||||
return http.Serve(l, nil)
|
||||
}
|
||||
|
||||
// scheduleServiceTokens adds service tokens to the service supervisor.
|
||||
func scheduleServiceTokens(s *Service, funcSet serviceFuncMap) {
|
||||
for name := range runset {
|
||||
if _, ok := funcSet[name]; !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
swap := deepcopy.Copy(s.cfg)
|
||||
s.serviceToken[name] = append(s.serviceToken[name], s.Supervisor.Add(funcSet[name](swap.(*ociscfg.Config))))
|
||||
}
|
||||
}
|
||||
|
||||
// generateRunSet interprets the cfg.Runtime.Services config option to cherry-pick which services to start using
|
||||
// the runtime.
|
||||
func (s *Service) generateRunSet(cfg *ociscfg.Config) {
|
||||
runset = make(map[string]struct{})
|
||||
if cfg.Runtime.Services != nil {
|
||||
for _, name := range cfg.Runtime.Services {
|
||||
runset[name] = struct{}{}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for _, service := range s.Services {
|
||||
for name := range service {
|
||||
runset[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// add additional services if explicitly added by config
|
||||
for _, name := range cfg.Runtime.Additional {
|
||||
runset[name] = struct{}{}
|
||||
}
|
||||
|
||||
// remove services if explicitly excluded by config
|
||||
for _, name := range cfg.Runtime.Disabled {
|
||||
delete(runset, name)
|
||||
}
|
||||
}
|
||||
|
||||
// List running processes for the Service Controller.
|
||||
func (s *Service) List(_ struct{}, reply *string) error {
|
||||
tableString := &strings.Builder{}
|
||||
table := tablewriter.NewWriter(tableString)
|
||||
table.SetHeader([]string{"Service"})
|
||||
|
||||
names := []string{}
|
||||
for t := range s.serviceToken {
|
||||
if len(s.serviceToken[t]) > 0 {
|
||||
names = append(names, t)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
|
||||
for n := range names {
|
||||
table.Append([]string{names[n]})
|
||||
}
|
||||
|
||||
table.Render()
|
||||
*reply = tableString.String()
|
||||
return nil
|
||||
}
|
||||
|
||||
// trap blocks on halt channel. When the runtime is interrupted it
|
||||
// signals the controller to stop any supervised process.
|
||||
func trap(s *Service, ctx context.Context) {
|
||||
<-ctx.Done()
|
||||
for sName := range s.serviceToken {
|
||||
for i := range s.serviceToken[sName] {
|
||||
if err := s.Supervisor.Remove(s.serviceToken[sName][i]); err != nil {
|
||||
s.Log.Error().Err(err).Str("service", "runtime service").Msgf("terminating with signal: %v", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.Log.Debug().Str("service", "runtime service").Msgf("terminating with signal: %v", s)
|
||||
time.Sleep(3 * time.Second) // give the services time to deregister
|
||||
os.Exit(0) // FIXME this cause an early exit that prevents services from shitting down properly
|
||||
}
|
||||
|
||||
// pingNats will attempt to connect to nats, blocking until a connection is established
|
||||
func pingNats(cfg *ociscfg.Config) error {
|
||||
// We need to get a natsconfig from somewhere. We can use any one.
|
||||
evcfg := cfg.Postprocessing.Postprocessing.Events
|
||||
_, err := stream.NatsFromConfig("initial", true, stream.NatsConfig(evcfg))
|
||||
return err
|
||||
}
|
||||
|
||||
func pingGateway(cfg *ociscfg.Config) error {
|
||||
// init grpc connection
|
||||
_, err := ogrpc.NewClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b := backoff.NewExponentialBackOff()
|
||||
o := func() error {
|
||||
n := b.NextBackOff()
|
||||
_, err := pool.GetGatewayServiceClient(cfg.Reva.Address)
|
||||
if err != nil && n > time.Second {
|
||||
logger.New().Error().Err(err).Msgf("can't connect to gateway service, retrying in %s", n)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
err = backoff.Retry(o, b)
|
||||
return err
|
||||
}
|
||||
|
||||
func wait(d time.Duration) func(cfg *ociscfg.Config) error {
|
||||
return func(cfg *ociscfg.Config) error {
|
||||
time.Sleep(d)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
ociscfg "github.com/opencloud-eu/opencloud/ocis-pkg/config"
|
||||
"github.com/thejerf/suture/v4"
|
||||
)
|
||||
|
||||
// SutureService allows for the settings command to be embedded and supervised by a suture supervisor tree.
|
||||
type SutureService struct {
|
||||
exec func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NewSutureServiceBuilder creates a new suture service
|
||||
func NewSutureServiceBuilder(f func(context.Context, *ociscfg.Config) error) func(*ociscfg.Config) suture.Service {
|
||||
return func(cfg *ociscfg.Config) suture.Service {
|
||||
return SutureService{
|
||||
exec: func(ctx context.Context) error {
|
||||
return f(ctx, cfg)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Serve to fullfil Server interface
|
||||
func (s SutureService) Serve(ctx context.Context) error {
|
||||
return s.exec(ctx)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package trash
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const (
|
||||
// _trashGlobPattern is the glob pattern to find all trash items
|
||||
_trashGlobPattern = "spaces/*/*/trash/*/*/*/*"
|
||||
)
|
||||
|
||||
// PurgeTrashEmptyPaths purges empty paths in the trash
|
||||
func PurgeTrashEmptyPaths(p string, dryRun bool) error {
|
||||
// we have all trash nodes in all spaces now
|
||||
dirs, err := filepath.Glob(filepath.Join(p, _trashGlobPattern))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(dirs) == 0 {
|
||||
return errors.New("no trash found. Double check storage path")
|
||||
}
|
||||
|
||||
for _, d := range dirs {
|
||||
if err := removeEmptyFolder(d, dryRun); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeEmptyFolder(path string, dryRun bool) error {
|
||||
if dryRun {
|
||||
f, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(f) < 1 {
|
||||
fmt.Println("would remove", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
// we do not really care about the error here
|
||||
// if the folder is not empty we will get an error,
|
||||
// this is our signal to break out of the recursion
|
||||
return nil
|
||||
}
|
||||
nd := filepath.Dir(path)
|
||||
if filepath.Base(nd) == "trash" {
|
||||
return nil
|
||||
}
|
||||
return removeEmptyFolder(nd, dryRun)
|
||||
}
|
||||
Reference in New Issue
Block a user