Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+175
View File
@@ -0,0 +1,175 @@
// Package backup contains КуСфера backup functionality.
package backup
import (
"fmt"
"os"
"regexp"
)
// Inconsistency describes the type of inconsistency
type Inconsistency string
var (
// InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore
InconsistencyBlobMissing Inconsistency = "blob missing"
// InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference
InconsistencyBlobOrphaned Inconsistency = "blob orphaned"
// InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node
InconsistencyNodeMissing Inconsistency = "node missing"
// InconsistencyMetadataMissing is an inconsistency where a node is missing metadata
InconsistencyMetadataMissing Inconsistency = "metadata missing"
// InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink
InconsistencySymlinkMissing Inconsistency = "symlink missing"
// InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock
InconsistencyFilesMissing Inconsistency = "files missing"
// InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file
InconsistencyMalformedFile Inconsistency = "malformed file"
// regex to determine if a node is trashed or versioned.
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z(\.\d+)?$`)
// 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink
_trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$`)
)
// Consistency holds the node and blob data of a storage provider
type Consistency struct {
// Storing the data like this might take a lot of memory
// we might need to optimize this if we run into memory issues
Nodes map[string][]Inconsistency
LinkedNodes map[string][]Inconsistency
BlobReferences map[string][]Inconsistency
Blobs map[string][]Inconsistency
nodeToLink map[string]string
blobToNode map[string]string
}
// NewConsistency creates a new Consistency object
func NewConsistency() *Consistency {
return &Consistency{
Nodes: make(map[string][]Inconsistency),
LinkedNodes: make(map[string][]Inconsistency),
BlobReferences: make(map[string][]Inconsistency),
Blobs: make(map[string][]Inconsistency),
nodeToLink: make(map[string]string),
blobToNode: make(map[string]string),
}
}
// CheckProviderConsistency checks the consistency of a space
func CheckProviderConsistency(storagepath string, lbs ListBlobstore, fail bool) error {
fsys := os.DirFS(storagepath)
p := NewProvider(fsys, storagepath, lbs)
if err := p.ProduceData(); err != nil {
return err
}
c := NewConsistency()
c.GatherData(p.Events)
return c.PrintResults(storagepath, fail)
}
// GatherData gathers and evaluates data produced by the DataProvider
func (c *Consistency) GatherData(events <-chan any) {
for ev := range events {
switch d := ev.(type) {
case NodeData:
// does it have inconsistencies?
if len(d.Inconsistencies) != 0 {
c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...)
}
// is it linked?
if _, ok := c.LinkedNodes[d.NodePath]; ok {
deleteInconsistency(c.LinkedNodes, d.NodePath)
} else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil {
c.Nodes[d.NodePath] = []Inconsistency{}
}
// does it have a blob?
if d.BlobPath != "" {
if _, ok := c.Blobs[d.BlobPath]; ok {
deleteInconsistency(c.Blobs, d.BlobPath)
} else {
c.BlobReferences[d.BlobPath] = []Inconsistency{}
c.blobToNode[d.BlobPath] = d.NodePath
}
}
case LinkData:
// does it have a node?
if _, ok := c.Nodes[d.NodePath]; ok {
deleteInconsistency(c.Nodes, d.NodePath)
} else {
c.LinkedNodes[d.NodePath] = []Inconsistency{}
c.nodeToLink[d.NodePath] = d.LinkPath
}
case BlobData:
// does it have a reference?
if _, ok := c.BlobReferences[d.BlobPath]; ok {
deleteInconsistency(c.BlobReferences, d.BlobPath)
} else {
c.Blobs[d.BlobPath] = []Inconsistency{}
}
}
}
for n := range c.Nodes {
if len(c.Nodes[n]) == 0 {
c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing)
}
}
for l := range c.LinkedNodes {
c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing)
}
for b := range c.Blobs {
c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned)
}
for b := range c.BlobReferences {
c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing)
}
}
// PrintResults prints the results of the evaluation
func (c *Consistency) PrintResults(discpath string, fail bool) error {
if len(c.Nodes) != 0 {
fmt.Println("\n🚨 Inconsistent Nodes:")
}
for n := range c.Nodes {
fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n)
}
if len(c.LinkedNodes) != 0 {
fmt.Println("\n🚨 Inconsistent Links:")
}
for l := range c.LinkedNodes {
fmt.Printf("\t👉️ %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l)
}
if len(c.Blobs) != 0 {
fmt.Println("\n🚨 Inconsistent Blobs:")
}
for b := range c.Blobs {
fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b)
}
if len(c.BlobReferences) != 0 {
fmt.Println("\n🚨 Inconsistent BlobReferences:")
}
for b := range c.BlobReferences {
fmt.Printf("\t👉️ %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b])
}
if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 {
fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", discpath)
} else if fail {
os.Exit(1)
}
return nil
}
func deleteInconsistency(incs map[string][]Inconsistency, path string) {
if len(incs[path]) == 0 {
delete(incs, path)
}
}
+162
View File
@@ -0,0 +1,162 @@
package backup_test
import (
"testing"
"github.com/qsfera/server/qsfera/pkg/backup"
"github.com/test-go/testify/require"
)
func TestGatherData(t *testing.T) {
testcases := []struct {
Name string
Events []any
Expected *backup.Consistency
}{
{
Name: "no symlinks - no blobs",
Events: []any{
nodeData("nodepath", "blobpath", true),
},
Expected: consistency(func(c *backup.Consistency) {
node(c, "nodepath", backup.InconsistencySymlinkMissing)
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
}),
},
{
Name: "symlink not required - no blobs",
Events: []any{
nodeData("nodepath", "blobpath", false),
},
Expected: consistency(func(c *backup.Consistency) {
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
}),
},
{
Name: "no inconsistencies",
Events: []any{
nodeData("nodepath", "blobpath", true),
linkData("linkpath", "nodepath"),
blobData("blobpath"),
},
Expected: consistency(func(c *backup.Consistency) {
}),
},
{
Name: "orphaned blob",
Events: []any{
nodeData("nodepath", "blobpath", true),
linkData("linkpath", "nodepath"),
blobData("blobpath"),
blobData("anotherpath"),
},
Expected: consistency(func(c *backup.Consistency) {
blob(c, "anotherpath", backup.InconsistencyBlobOrphaned)
}),
},
{
Name: "missing node",
Events: []any{
linkData("linkpath", "nodepath"),
blobData("blobpath"),
},
Expected: consistency(func(c *backup.Consistency) {
linkedNode(c, "nodepath", backup.InconsistencyNodeMissing)
blob(c, "blobpath", backup.InconsistencyBlobOrphaned)
}),
},
{
Name: "corrupt metadata",
Events: []any{
nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
linkData("linkpath", "nodepath"),
blobData("blobpath"),
},
Expected: consistency(func(c *backup.Consistency) {
node(c, "nodepath", backup.InconsistencyMetadataMissing)
}),
},
{
Name: "corrupt metadata, no blob",
Events: []any{
nodeData("nodepath", "blobpath", true, backup.InconsistencyMetadataMissing),
linkData("linkpath", "nodepath"),
},
Expected: consistency(func(c *backup.Consistency) {
node(c, "nodepath", backup.InconsistencyMetadataMissing)
blobReference(c, "blobpath", backup.InconsistencyBlobMissing)
}),
},
}
for _, tc := range testcases {
events := make(chan any)
go func() {
for _, ev := range tc.Events {
switch e := ev.(type) {
case backup.NodeData:
events <- e
case backup.LinkData:
events <- e
case backup.BlobData:
events <- e
}
}
close(events)
}()
c := backup.NewConsistency()
c.GatherData(events)
require.Equal(t, tc.Expected.Nodes, c.Nodes)
require.Equal(t, tc.Expected.LinkedNodes, c.LinkedNodes)
require.Equal(t, tc.Expected.Blobs, c.Blobs)
require.Equal(t, tc.Expected.BlobReferences, c.BlobReferences)
}
}
func nodeData(nodePath, blobPath string, requiresSymlink bool, incs ...backup.Inconsistency) backup.NodeData {
return backup.NodeData{
NodePath: nodePath,
BlobPath: blobPath,
RequiresSymlink: requiresSymlink,
Inconsistencies: incs,
}
}
func linkData(linkPath, nodePath string) backup.LinkData {
return backup.LinkData{
LinkPath: linkPath,
NodePath: nodePath,
}
}
func blobData(blobPath string) backup.BlobData {
return backup.BlobData{
BlobPath: blobPath,
}
}
func consistency(f func(*backup.Consistency)) *backup.Consistency {
c := backup.NewConsistency()
f(c)
return c
}
func node(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
c.Nodes[path] = inc
}
func linkedNode(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
c.LinkedNodes[path] = inc
}
func blob(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
c.Blobs[path] = inc
}
func blobReference(c *backup.Consistency, path string, inc ...backup.Inconsistency) {
c.BlobReferences[path] = inc
}
+231
View File
@@ -0,0 +1,231 @@
package backup
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/vmihailenco/msgpack/v5"
)
// ListBlobstore required to check blob consistency
type ListBlobstore interface {
List() ([]*node.Node, error)
Path(node *node.Node) string
}
// DataProvider provides data for the consistency check
type DataProvider struct {
Events chan any
fsys fs.FS
discpath string
lbs ListBlobstore
skipBlobs bool
}
// NodeData holds data about the nodes
type NodeData struct {
NodePath string
BlobPath string
RequiresSymlink bool
Inconsistencies []Inconsistency
}
// LinkData about the symlinks
type LinkData struct {
LinkPath string
NodePath string
}
// BlobData about the blobs in the blobstore
type BlobData struct {
BlobPath string
}
// NewProvider creates a new DataProvider object
func NewProvider(fsys fs.FS, discpath string, lbs ListBlobstore) *DataProvider {
return &DataProvider{
Events: make(chan any),
fsys: fsys,
discpath: discpath,
lbs: lbs,
skipBlobs: lbs == nil,
}
}
// ProduceData produces data for the consistency check
// Spawns 4 go-routines at the moment. If needed, this can be optimized.
func (dp *DataProvider) ProduceData() error {
dirs, err := fs.Glob(dp.fsys, "spaces/*/*/nodes/*/*/*/*")
if err != nil {
return err
}
if len(dirs) == 0 {
return errors.New("no backup found. Double check storage path")
}
wg := sync.WaitGroup{}
// crawl spaces
wg.Go(func() {
for _, d := range dirs {
dp.evaluateNodeDir(d)
}
})
// crawl trash
wg.Go(func() {
dp.evaluateTrashDir()
})
// crawl blobstore
if !dp.skipBlobs {
wg.Go(func() {
bs, err := dp.lbs.List()
if err != nil {
fmt.Println("error listing blobs", err)
}
for _, bn := range bs {
dp.Events <- BlobData{BlobPath: dp.lbs.Path(bn)}
}
})
}
// wait for all crawlers to finish
go func() {
wg.Wait()
dp.quit()
}()
return nil
}
func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
if dp.skipBlobs {
return "", ""
}
b, err := fs.ReadFile(dp.fsys, path+".mpk")
if err != nil {
return "", InconsistencyFilesMissing
}
m := map[string][]byte{}
if err := msgpack.Unmarshal(b, &m); err != nil {
return "", InconsistencyMalformedFile
}
// FIXME: how to check if metadata is complete?
if bid := m["user.oc.blobid"]; string(bid) != "" {
spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
return dp.lbs.Path(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: string(bid)}), ""
}
return "", ""
}
func (dp *DataProvider) evaluateNodeDir(d string) {
// d is something like spaces/a8/e5d981-41e4-4468-b532-258d5fb457d3/nodes/2d/08/8d/24
// we could have multiple nodes under this, but we are only interested in one file per node - the one with "" extension
entries, err := fs.ReadDir(dp.fsys, d)
if err != nil {
fmt.Println("error reading dir", err)
return
}
if len(entries) == 0 {
fmt.Println("empty dir", filepath.Join(dp.discpath, d))
return
}
for _, e := range entries {
switch {
case e.IsDir():
ls, err := fs.ReadDir(dp.fsys, filepath.Join(d, e.Name()))
if err != nil {
fmt.Println("error reading dir", err)
continue
}
for _, l := range ls {
linkpath := filepath.Join(dp.discpath, d, e.Name(), l.Name())
r, _ := os.Readlink(linkpath)
nodePath := filepath.Join(dp.discpath, d, e.Name(), r)
dp.Events <- LinkData{LinkPath: linkpath, NodePath: nodePath}
}
fallthrough
case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()):
np := filepath.Join(dp.discpath, d, e.Name())
var inc []Inconsistency
if !dp.filesExist(filepath.Join(d, e.Name())) {
inc = append(inc, InconsistencyFilesMissing)
}
bp, i := dp.getBlobPath(filepath.Join(d, e.Name()))
if i != "" {
inc = append(inc, i)
}
dp.Events <- NodeData{NodePath: np, BlobPath: bp, RequiresSymlink: requiresSymlink(np), Inconsistencies: inc}
}
}
}
func (dp *DataProvider) evaluateTrashDir() {
linkpaths, err := fs.Glob(dp.fsys, "spaces/*/*/trash/*/*/*/*/*")
if err != nil {
fmt.Println("error reading trash", err)
}
for _, l := range linkpaths {
linkpath := filepath.Join(dp.discpath, l)
r, _ := os.Readlink(linkpath)
p := filepath.Join(dp.discpath, l, "..", r)
dp.Events <- LinkData{LinkPath: linkpath, NodePath: p}
}
}
func (dp *DataProvider) filesExist(path string) bool {
check := func(p string) bool {
_, err := fs.Stat(dp.fsys, p)
return err == nil
}
return check(path) && check(path+".mpk")
}
func (dp *DataProvider) quit() {
close(dp.Events)
}
func requiresSymlink(path string) bool {
spaceID, nodeID := getIDsFromPath(path)
if nodeID != "" && spaceID != "" && (spaceID == nodeID || _versionRegex.MatchString(nodeID)) {
return false
}
return true
}
func getIDsFromPath(path string) (string, string) {
rawIDs := strings.Split(path, "/nodes/")
if len(rawIDs) != 2 {
return "", ""
}
s := strings.Split(rawIDs[0], "/spaces/")
if len(s) != 2 {
return "", ""
}
spaceID := strings.Replace(s[1], "/", "", -1)
nodeID := strings.Replace(rawIDs[1], "/", "", -1)
return spaceID, nodeID
}
+82
View File
@@ -0,0 +1,82 @@
package command
import (
"errors"
"fmt"
"github.com/spf13/cobra"
"github.com/qsfera/server/qsfera/pkg/backup"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
decomposedbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
decomposeds3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/blobstore"
)
// BackupCommand is the entrypoint for the backup command
func BackupCommand(cfg *config.Config) *cobra.Command {
bckCmd := &cobra.Command{
Use: "backup",
Short: "КуСфера backup functionality",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg, true))
},
}
bckCmd.AddCommand(ConsistencyCommand(cfg))
return bckCmd
}
// ConsistencyCommand is the entrypoint for the consistency Command
func ConsistencyCommand(cfg *config.Config) *cobra.Command {
consCmd := &cobra.Command{
Use: "consistency",
Short: "check backup consistency",
RunE: func(cmd *cobra.Command, args []string) error {
var (
bs backup.ListBlobstore
err error
)
basePath, _ := cmd.Flags().GetString("basepath")
blobstoreFlag, _ := cmd.Flags().GetString("blobstore")
switch blobstoreFlag {
case "decomposeds3":
bs, err = decomposeds3bs.New(
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
decomposeds3bs.Options{},
)
case "decomposed":
bs, err = decomposedbs.New(basePath)
case "none":
bs = nil
default:
err = errors.New("blobstore type not supported")
}
if err != nil {
fmt.Println(err)
return err
}
fail, _ := cmd.Flags().GetBool("fail")
if err := backup.CheckProviderConsistency(basePath, bs, fail); err != nil {
fmt.Println(err)
return err
}
return nil
},
}
consCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/users)")
_ = consCmd.MarkFlagRequired("basepath")
consCmd.Flags().StringP("blobstore", "b", "decomposed", "the blobstore type. Can be (none, decomposed, decomposeds3). Default decomposed")
consCmd.Flags().Bool("fail", false, "exit with non-zero status if consistency check fails")
return consCmd
}
func init() {
register.AddCommand(BackupCommand)
}
+514
View File
@@ -0,0 +1,514 @@
package command
import (
"bytes"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/exec"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/version"
"github.com/pkg/xattr"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/spf13/cobra"
)
// BenchmarkCommand is the entrypoint for the benchmark commands.
func BenchmarkCommand(cfg *config.Config) *cobra.Command {
benchCmd := &cobra.Command{
Use: "benchmark",
Short: "cli tools to test low and high level performance",
}
benchCmd.AddCommand(BenchmarkClientCommand(cfg), BenchmarkSyscallsCommand(cfg))
return benchCmd
}
// BenchmarkClientCommand is the entrypoint for the benchmark client command.
func BenchmarkClientCommand(cfg *config.Config) *cobra.Command {
benchClientCmd := &cobra.Command{
Use: "client",
Short: "Start a client that continuously makes web requests and prints stats. The options mimic curl, but we default to PROPFIND requests.",
RunE: func(cmd *cobra.Command, args []string) error {
jobs, err := cmd.Flags().GetInt("jobs")
if err != nil {
return err
}
insecure, _ := cmd.Flags().GetBool("insecure")
opt := clientOptions{
url: args[0],
insecure: insecure,
jobs: jobs,
headers: make(map[string]string),
}
if d, _ := cmd.Flags().GetString("data-raw"); d != "" {
opt.request = "POST"
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
opt.data = []byte(d)
}
if d, _ := cmd.Flags().GetString("data"); d != "" {
opt.request = "POST"
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
if strings.HasPrefix(d, "@") {
filePath := strings.TrimPrefix(d, "@")
var data []byte
var err error
// read from file or stdin and trim trailing newlines
if filePath == "-" {
data, err = os.ReadFile("/dev/stdin")
} else {
data, err = os.ReadFile(filePath)
}
if err != nil {
log.Fatal(errors.New("could not read data from file '" + filePath + "': " + err.Error()))
}
// clean byte array similar to curl's --data parameter
// It removes leading/trailing whitespace and converts line breaks to spaces
// Trim leading and trailing whitespace
data = bytes.TrimSpace(data)
// Replace newlines and carriage returns with spaces
data = bytes.ReplaceAll(data, []byte("\r\n"), []byte(" "))
data = bytes.ReplaceAll(data, []byte("\n"), []byte(" "))
data = bytes.ReplaceAll(data, []byte("\r"), []byte(" "))
// Replace multiple spaces with single space
for bytes.Contains(data, []byte(" ")) {
data = bytes.ReplaceAll(data, []byte(" "), []byte(" "))
}
opt.data = data
} else {
opt.data = []byte(d)
}
}
if d, _ := cmd.Flags().GetString("data-binary"); d != "" {
opt.request = "POST"
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
if strings.HasPrefix(d, "@") {
filePath := strings.TrimPrefix(d, "@")
var data []byte
var err error
if filePath == "-" {
data, err = os.ReadFile("/dev/stdin")
} else {
data, err = os.ReadFile(filePath)
}
if err != nil {
log.Fatal(errors.New("could not read data from file '" + filePath + "': " + err.Error()))
}
opt.data = data
} else {
opt.data = []byte(d)
}
}
// override method if specified
if request, _ := cmd.Flags().GetString("request"); request != "" {
opt.request = request
}
if opt.url == "" {
log.Fatal(errors.New("no URL specified"))
}
headersSlice, err := cmd.Flags().GetStringSlice("header")
if err != nil {
return err
}
for _, h := range headersSlice {
parts := strings.SplitN(h, ":", 2)
if len(parts) != 2 {
log.Fatal(errors.New("invalid header '" + h + "'"))
}
opt.headers[parts[0]] = strings.TrimSpace(parts[1])
}
rate, _ := cmd.Flags().GetString("rate")
if rate != "" {
parts := strings.SplitN(rate, "/", 2)
num, err := strconv.Atoi(parts[0])
if err != nil {
fmt.Println(err)
}
unit := time.Hour // default
if len(parts) == 2 {
switch parts[1] {
case "s":
unit = time.Second
case "m":
unit = time.Minute
case "d":
unit = time.Hour * 24
default:
log.Fatal(errors.New("unsupported rate unit. Use s, m, h or d"))
}
}
opt.rateDelay = unit / time.Duration(num)
}
user, _ := cmd.Flags().GetString("user")
opt.auth = func() string {
return "Basic " + base64.StdEncoding.EncodeToString([]byte(user))
}
btc, _ := cmd.Flags().GetString("bearer-token-command")
if btc != "" {
parts := strings.SplitN(btc, " ", 2)
var cmd *exec.Cmd
opt.auth = func() string {
if len(parts) > 1 {
cmd = exec.Command(parts[0], parts[1])
} else {
cmd = exec.Command(parts[0])
}
output, err := cmd.CombinedOutput()
if err != nil {
fmt.Println(err)
}
return "Bearer " + string(output)
}
}
every, err := cmd.Flags().GetInt("every")
if err != nil {
return err
}
if every != 0 {
opt.ticker = time.NewTicker(time.Second * time.Duration(every))
defer opt.ticker.Stop()
}
// Set up signal handling for Ctrl+C
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
go func() {
<-sigChan
fmt.Println("\nReceived interrupt signal, shutting down...")
cancel()
}()
return client(ctx, opt)
},
}
// flags mimicing curl
benchClientCmd.Flags().StringP("request", "X", "PROPFIND", "Specifies a custom request method to use when communicating with the HTTP server.")
benchClientCmd.Flags().StringP("user", "u", "admin:admin", "Specify the user name and password to use for server authentication.")
benchClientCmd.Flags().BoolP("insecure", "k", false, "Skip the TLS verification step and proceed without checking.")
benchClientCmd.Flags().StringP("data", "d", "", "Sends the specified data in a POST request to the HTTP server, in the same way that a browser does when a user has filled in an HTML form and presses the submit button. If you start the data with the letter @, the rest should be a file name to read the data from, or - if you want to read the data from stdin. When -d, --data is told to read from a file like that, carriage returns and newlines are stripped out. If you do not want the @ character to have a special interpretation use --data-raw instead.")
benchClientCmd.Flags().StringP("data-raw", "", "", "Sends the specified data in a request to the HTTP server.")
benchClientCmd.Flags().StringP("data-binary", "", "", "This posts data exactly as specified with no extra processing whatsoever. If you start the data with the letter @, the rest should be a file name to read the data from, or - if you want to read the data from stdin.")
benchClientCmd.Flags().StringSliceP("headers", "H", []string{}, "Extra header to include in information sent.")
benchClientCmd.Flags().String("rate", "", "Specify the maximum transfer frequency you allow a client to use - in number of transfer starts per time unit (sometimes called request rate). The request rate is provided as \"N/U\" where N is an integer number and U is a time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd' /(day, as in a 24 hour unit). The default time unit, if no \"/U\" is provided, is number of transfers per hour.")
// other flags
benchClientCmd.Flags().IntP("jobs", "j", 1, "Number of parallel clients to start. Defaults to 1.")
benchClientCmd.Flags().Int("every", 0, "Aggregate stats every time this amount of seconds has passed.")
benchClientCmd.Flags().String("bearer-token-command", "", "Command to execute for a bearer token, e.g. 'oidc-token qsfera'. When set, disables basic auth.")
return benchClientCmd
}
type clientOptions struct {
request string
url string
auth func() string
insecure bool
headers map[string]string
rateDelay time.Duration
data []byte
ticker *time.Ticker
jobs int
}
func client(ctx context.Context, o clientOptions) error {
type stat struct {
job int
duration time.Duration
status int
}
stats := make(chan stat)
for i := 0; i < o.jobs; i++ {
go func(i int) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: o.insecure,
},
}
client := &http.Client{Transport: tr}
cookies := map[string]*http.Cookie{}
for {
// Check if context is cancelled
select {
case <-ctx.Done():
return
default:
}
req, err := http.NewRequest(o.request, o.url, bytes.NewReader(o.data))
if err != nil {
log.Printf("client %d: could not create request: %s\n", i, err)
return
}
req.Header.Set("Authorization", strings.TrimSpace(o.auth()))
for k, v := range o.headers {
req.Header.Set(k, v)
}
for _, cookie := range cookies {
req.AddCookie(cookie)
}
start := time.Now()
res, err := client.Do(req)
duration := -time.Until(start)
if err != nil {
// Check if error is due to context cancellation
if ctx.Err() != nil {
return
}
log.Printf("client %d: could not create request: %s\n", i, err)
time.Sleep(time.Second)
} else {
res.Body.Close()
select {
case stats <- stat{
job: i,
duration: duration,
status: res.StatusCode,
}:
case <-ctx.Done():
return
}
for _, c := range res.Cookies() {
cookies[c.Name] = c
}
}
// Sleep with context awareness
if o.rateDelay > duration {
select {
case <-time.After(o.rateDelay - duration):
case <-ctx.Done():
return
}
}
}
}(i)
}
numRequests := 0
if o.ticker == nil {
// no ticker, just write every request
for {
select {
case stat := <-stats:
numRequests++
fmt.Printf("req %d took %v and returned status %d\n", numRequests, stat.duration, stat.status)
case <-ctx.Done():
fmt.Println("\nShutting down...")
return nil
}
}
}
var duration time.Duration
for {
select {
case stat := <-stats:
numRequests++
duration += stat.duration
case <-o.ticker.C:
if numRequests > 0 {
fmt.Printf("%d req at %v/req\n", numRequests, duration/time.Duration(numRequests))
numRequests = 0
duration = 0
}
case <-ctx.Done():
if numRequests > 0 {
fmt.Printf("\n%d req at %v/req\n", numRequests, duration/time.Duration(numRequests))
}
fmt.Println("Shutting down...")
return nil
}
}
}
// BenchmarkSyscallsCommand is the entrypoint for the benchmark syscalls command.
func BenchmarkSyscallsCommand(cfg *config.Config) *cobra.Command {
benchSysCallCmd := &cobra.Command{
Use: "syscalls",
Short: "test the performance of syscalls",
RunE: func(cmd *cobra.Command, args []string) error {
path, _ := cmd.Flags().GetString("path")
if path == "" {
f, err := os.CreateTemp("", "qsfera-bench-temp-")
if err != nil {
log.Fatal(err)
}
path = f.Name()
f.Close()
defer os.Remove(path)
}
iterations, err := cmd.Flags().GetInt("iterations")
if err != nil {
return err
}
return benchmark(iterations, path)
},
}
benchSysCallCmd.Flags().String("path", "", "Path to test")
benchSysCallCmd.Flags().Int("iterations", 100, "Number of iterations to execute")
return benchSysCallCmd
}
func benchmark(iterations int, path string) error {
tests := map[string]func() error{
"lockedfile open(wo,c,t) close": func() error {
for i := 0; i < iterations; i++ {
lockedFile, err := lockedfile.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
lockedFile.Close()
}
return nil
},
"stat": func() error {
for i := 0; i < iterations; i++ {
_, err := os.Stat(path)
if err != nil {
return err
}
}
return nil
},
"fopen(ro) close": func() error {
for i := 0; i < iterations; i++ {
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
}
h.Close()
}
return nil
},
"fopen(wo,t) write close": func() error {
for i := 0; i < iterations; i++ {
h, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
}
_, err = h.WriteString("1234567890")
if err != nil {
h.Close()
return err
}
h.Close()
}
return nil
},
"fopen(ro) read close": func() error {
for i := 0; i < iterations; i++ {
bytes := make([]byte, 0, 10)
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
return err
}
_, err = h.Read(bytes)
if err != nil {
h.Close()
return err
}
h.Close()
}
return nil
},
"xattr-set": func() error {
for i := 0; i < iterations; i++ {
err := xattr.Set(path, "user.test", []byte("123456"))
if err != nil {
return err
}
}
return nil
},
"xattr-get": func() error {
for i := 0; i < iterations; i++ {
_, err := xattr.Get(path, "user.test")
if err != nil {
return err
}
}
return nil
},
}
fmt.Println("Version: " + version.GetString())
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Printf("Path: %s\n", path)
fmt.Printf("Iterations: %d\n", iterations)
fmt.Println("")
cfg := tablewriter.Config{
Header: tw.CellConfig{
Formatting: tw.CellFormatting{
AutoFormat: tw.Off,
},
},
Row: tw.CellConfig{
ColumnAligns: []tw.Align{
tw.AlignLeft,
tw.AlignRight,
tw.AlignRight,
tw.AlignRight,
},
},
}
table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfg))
table.Header([]string{"Test", "Iterations", "dur/it", "total"})
for _, t := range []string{"lockedfile open(wo,c,t) close", "stat", "fopen(wo,t) write close", "fopen(ro) close", "fopen(ro) read close", "xattr-set", "xattr-get"} {
start := time.Now()
err := tests[t]()
end := time.Now()
delta := end.Sub(start)
if err != nil {
table.Append([]string{t, fmt.Sprintf("%d", iterations), err.Error(), err.Error()})
} else {
table.Append([]string{t, fmt.Sprintf("%d", iterations), strconv.Itoa(int(delta.Nanoseconds())/iterations) + "ns", delta.String()})
}
}
table.Render()
return nil
}
func init() {
register.AddCommand(BenchmarkCommand)
}
+7
View File
@@ -0,0 +1,7 @@
package command
const (
CommandGroupServer = "Server"
CommandGroupServices = "Service"
CommandGroupStorage = "Storage"
)
+342
View File
@@ -0,0 +1,342 @@
package command
import (
"context"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/store"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/rs/zerolog"
"github.com/spf13/cobra"
)
// DecomposedfsCommand is the entrypoint for the groups command.
func DecomposedfsCommand(cfg *config.Config) *cobra.Command {
decomposedCmd := &cobra.Command{
Use: "decomposedfs",
Short: `cli tools to inspect and manipulate a decomposedfs storage.`,
GroupID: CommandGroupStorage,
}
decomposedCmd.AddCommand(metadataCmd(cfg), checkCmd(cfg))
return decomposedCmd
}
func init() {
register.AddCommand(DecomposedfsCommand)
}
func checkCmd(_ *config.Config) *cobra.Command {
cCmd := &cobra.Command{
Use: "check-treesize",
Short: `cli tool to check the treesize metadata of a Space`,
RunE: check,
}
cCmd.Flags().StringP("root", "r", "", "Path to the root directory of the decomposedfs")
_ = cCmd.MarkFlagRequired("root")
cCmd.Flags().StringP("node", "n", "", "Space ID of the Space to inspect")
_ = cCmd.MarkFlagRequired("node")
cCmd.Flags().Bool("repair", false, "Try to repair nodes with incorrect treesize metadata. IMPORTANT: Only use this while КуСфера is not running.")
cCmd.Flags().Bool("force", false, "Do not prompt for confirmation when running in repair mode.")
return cCmd
}
func check(cmd *cobra.Command, args []string) error {
rootFlag, _ := cmd.Flags().GetString("root")
repairFlag, _ := cmd.Flags().GetBool("repair")
forceFlag, _ := cmd.Flags().GetBool("force")
if repairFlag && !forceFlag {
answer := strings.ToLower(stringPrompt("IMPORTANT: Only use '--repair' when КуСфера is not running. Do you want to continue? [yes | no = default]"))
if answer != "yes" && answer != "y" {
return nil
}
}
lu, backend := getBackend(cmd)
o := &options.Options{
MetadataBackend: backend.Name(),
MaxConcurrency: 100,
}
bs, err := blobstore.New(rootFlag)
if err != nil {
fmt.Println("Failed to init blobstore")
return err
}
tree := tree.New(lu, bs, o, permissions.Permissions{}, store.Create(), &zerolog.Logger{})
nId, _ := cmd.Flags().GetString("node")
n, err := lu.NodeFromSpaceID(context.Background(), nId)
if err != nil || !n.Exists {
fmt.Println("Can not find node '" + nId + "'")
return err
}
fmt.Printf("Checking treesizes in space: %s (id: %s)\n", n.Name, n.ID)
ctx := revactx.ContextSetUser(context.Background(),
&userpb.User{
Id: &userpb.UserId{
OpaqueId: "00000000-0000-0000-0000-000000000000",
},
Username: "offline",
})
treeSize, err := walkTree(ctx, tree, lu, n, repairFlag)
if err != nil {
fmt.Printf("failed to walk tree of node %s: %s\n", n.ID, err)
}
treesizeFromMetadata, err := n.GetTreeSize(cmd.Context())
if err != nil {
fmt.Printf("failed to read treesize of node: %s: %s\n", n.ID, err)
}
if treesizeFromMetadata != treeSize {
fmt.Printf("Tree sizes mismatch for space: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
n.Name, n.ID, n.InternalPath(), treeSize, treesizeFromMetadata)
if repairFlag {
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
n.ID, treeSize)
n.SetTreeSize(cmd.Context(), treeSize)
}
}
return nil
}
func walkTree(ctx context.Context, tree *tree.Tree, lu *lookup.Lookup, root *node.Node, repair bool) (uint64, error) {
if root.Type(ctx) != provider.ResourceType_RESOURCE_TYPE_CONTAINER {
return 0, errors.New("can't travers non-container nodes")
}
children, err := tree.ListFolder(ctx, root)
if err != nil {
fmt.Println("Can not list children for space'" + root.ID + "'")
return 0, err
}
var treesize uint64
for _, child := range children {
switch child.Type(ctx) {
case provider.ResourceType_RESOURCE_TYPE_CONTAINER:
subtreesize, err := walkTree(ctx, tree, lu, child, repair)
if err != nil {
fmt.Printf("error calculating tree size of node: %s: %s\n", child.ID, err)
return 0, err
}
treesizeFromMetadata, err := child.GetTreeSize(ctx)
if err != nil {
fmt.Printf("failed to read tree size of node: %s: %s\n", child.ID, err)
return 0, err
}
if treesizeFromMetadata != subtreesize {
origin, err := lu.Path(ctx, child, node.NoCheck)
if err != nil {
fmt.Printf("error get path: %s\n", err)
}
fmt.Printf("Tree sizes mismatch for node: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
origin, child.ID, child.InternalPath(), subtreesize, treesizeFromMetadata)
if repair {
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
child.ID, subtreesize)
child.SetTreeSize(ctx, subtreesize)
}
}
treesize += subtreesize
case provider.ResourceType_RESOURCE_TYPE_FILE:
blobsize, err := child.GetBlobSize(ctx)
if err != nil {
fmt.Printf("error reading blobsize of node: %s: %s\n", child.ID, err)
return 0, err
}
treesize += blobsize
default:
fmt.Printf("Ignoring type: %v, node: %s %s\n", child.Type(ctx), child.Name, child.ID)
}
}
return treesize, nil
}
func metadataCmd(cfg *config.Config) *cobra.Command {
metaCmd := &cobra.Command{
Use: "metadata",
Short: `cli tools to inspect and manipulate node metadata`,
}
metaCmd.AddCommand(dumpCmd(cfg), getCmd(cfg), setCmd(cfg))
metaCmd.Flags().StringP("root", "r", "", "Path to the root directory of the decomposedfs")
_ = metaCmd.MarkFlagRequired("root")
metaCmd.Flags().StringP("node", "n", "", "Path to or ID of the node to inspect")
_ = metaCmd.MarkFlagRequired("node")
return metaCmd
}
func dumpCmd(_ *config.Config) *cobra.Command {
return &cobra.Command{
Use: "dump",
Short: `print the metadata of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
RunE: func(cmd *cobra.Command, args []string) error {
lu, backend := getBackend(cmd)
path, err := getNode(cmd, lu)
if err != nil {
return err
}
attribs, err := backend.All(cmd.Context(), path)
if err != nil {
fmt.Println("Error reading attributes")
return err
}
attributeFlag, _ := cmd.Flags().GetString("attribute")
printAttribs(attribs, attributeFlag)
return nil
},
}
}
func getCmd(_ *config.Config) *cobra.Command {
gCmd := &cobra.Command{
Use: "get",
Short: `print a specific attribute of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
RunE: func(cmd *cobra.Command, args []string) error {
lu, backend := getBackend(cmd)
path, err := getNode(cmd, lu)
if err != nil {
return err
}
attribs, err := backend.All(cmd.Context(), path)
if err != nil {
fmt.Println("Error reading attributes")
return err
}
attributeFlag, _ := cmd.Flags().GetString("attribute")
printAttribs(attribs, attributeFlag)
return nil
},
}
gCmd.Flags().StringP("attribute", "a", "", "attribute to inspect, can be a glob pattern (e.g. 'user.*' will match all attributes starting with 'user.').")
return gCmd
}
func setCmd(_ *config.Config) *cobra.Command {
sCmd := &cobra.Command{
Use: "set",
Short: `manipulate metadata of the given node. Binary attributes can be given hex encoded (prefix by '0x') or base64 encoded (prefix by '0s').`,
RunE: func(cmd *cobra.Command, args []string) error {
lu, backend := getBackend(cmd)
n, err := getNode(cmd, lu)
if err != nil {
return err
}
v, _ := cmd.Flags().GetString("value")
if strings.HasPrefix(v, "0s") {
b64, err := base64.StdEncoding.DecodeString(v[2:])
if err == nil {
v = string(b64)
} else {
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
}
} else if strings.HasPrefix(v, "0x") {
h, err := hex.DecodeString(v[2:])
if err == nil {
v = string(h)
} else {
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
}
}
attributeFlag, _ := cmd.Flags().GetString("attribute")
err = backend.Set(cmd.Context(), n, attributeFlag, []byte(v))
if err != nil {
fmt.Println("Error setting attribute")
return err
}
return nil
},
}
sCmd.Flags().StringP("attribute", "a", "", "attribute to inspect, can be a glob pattern (e.g. 'user.*' will match all attributes starting with 'user.').")
_ = sCmd.MarkFlagRequired("attribute")
sCmd.Flags().StringP("value", "v", "", "value to set")
_ = sCmd.MarkFlagRequired("value")
return sCmd
}
func backend(backend string) metadata.Backend {
switch backend {
case "xattrs":
return metadata.NewXattrsBackend(cache.Config{})
case "mpk":
return metadata.NewMessagePackBackend(cache.Config{})
}
return metadata.NullBackend{}
}
func getBackend(cmd *cobra.Command) (*lookup.Lookup, metadata.Backend) {
rootFlag, _ := cmd.Flags().GetString("root")
bod := lookup.DetectBackendOnDisk(rootFlag)
backend := backend(bod)
lu := lookup.New(backend, &options.Options{
Root: rootFlag,
MetadataBackend: bod,
}, &timemanager.Manager{})
return lu, backend
}
func getNode(cmd *cobra.Command, lu *lookup.Lookup) (*node.Node, error) {
nodeFlag, _ := cmd.Flags().GetString("node")
id, err := storagespace.ParseID(nodeFlag)
if err != nil {
fmt.Println("Invalid node id.")
return nil, err
}
return lu.NodeFromID(context.Background(), &id)
}
func printAttribs(attribs map[string][]byte, onlyAttribute string) {
if onlyAttribute != "" {
fmt.Println(onlyAttribute + `=` + attribToString(attribs[onlyAttribute]))
return
}
names := []string{}
for k := range attribs {
names = append(names, k)
}
sort.Strings(names)
for _, n := range names {
fmt.Println(n + `=` + attribToString(attribs[n]))
}
}
func attribToString(attrib []byte) string {
for i := 0; i < len(attrib); i++ {
if attrib[i] < 32 || attrib[i] >= 127 {
return "0s" + base64.StdEncoding.EncodeToString(attrib)
}
}
return `"` + string(attrib) + `"`
}
+84
View File
@@ -0,0 +1,84 @@
package command
import (
"bufio"
"fmt"
"log"
"os"
"strings"
ocinit "github.com/qsfera/server/qsfera/pkg/init"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/defaults"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// InitCommand is the entrypoint for the init command
func InitCommand(_ *config.Config) *cobra.Command {
initCmd := &cobra.Command{
Use: "init",
Short: "initialise a КуСфера config",
GroupID: CommandGroupServer,
RunE: func(cmd *cobra.Command, args []string) error {
insecureFlag := viper.GetString("insecure")
insecure := false
if insecureFlag == "ask" {
answer := strings.ToLower(stringPrompt("Do you want to configure КуСфера with certificate checking disabled?\n This is not recommended for public instances! [yes | no = default]"))
if answer == "yes" || answer == "y" {
insecure = true
}
} else if insecureFlag == strings.ToLower("true") || insecureFlag == strings.ToLower("yes") || insecureFlag == strings.ToLower("y") {
insecure = true
}
forceOverwriteFlag := viper.GetBool("force-overwrite")
diffFlag, _ := cmd.Flags().GetBool("diff")
configPathFlag := viper.GetString("config-path")
adminPasswordFlag := viper.GetString("admin-password")
err := ocinit.CreateConfig(insecure, forceOverwriteFlag, diffFlag, configPathFlag, adminPasswordFlag)
if err != nil {
log.Fatalf("Could not create config: %s", err)
}
return nil
},
}
initCmd.Flags().String("insecure", "ask", "Allow insecure КуСфера config")
_ = viper.BindEnv("insecure", "OC_INSECURE")
_ = viper.BindPFlag("insecure", initCmd.Flags().Lookup("insecure"))
initCmd.Flags().BoolP("diff", "d", false, "Show the difference between the current config and the new one")
initCmd.Flags().BoolP("force-overwrite", "f", false, "Force overwrite existing config file")
_ = viper.BindEnv("force-overwrite", "OC_FORCE_CONFIG_OVERWRITE")
_ = viper.BindPFlag("force-overwrite", initCmd.Flags().Lookup("force-overwrite"))
initCmd.Flags().String("config-path", defaults.BaseConfigPath(), "Config path for the КуСфера runtime")
_ = viper.BindEnv("config-path", "OC_CONFIG_DIR")
_ = viper.BindEnv("config-path", "OC_BASE_DATA_PATH")
_ = viper.BindPFlag("config-path", initCmd.Flags().Lookup("config-path"))
initCmd.Flags().String("admin-password", "", "Set admin password instead of using a random generated one")
_ = viper.BindEnv("admin-password", "ADMIN_PASSWORD")
_ = viper.BindEnv("admin-password", "IDM_ADMIN_PASSWORD")
_ = viper.BindPFlag("admin-password", initCmd.Flags().Lookup("admin-password"))
return initCmd
}
func init() {
register.AddCommand(InitCommand)
}
func stringPrompt(label string) string {
input := ""
reader := bufio.NewReader(os.Stdin)
for {
_, _ = fmt.Fprint(os.Stderr, label+" ")
input, _ = reader.ReadString('\n')
if input != "" {
break
}
}
return strings.TrimSpace(input)
}
+59
View File
@@ -0,0 +1,59 @@
package command
import (
"fmt"
"log"
"net"
"net/rpc"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// ListCommand is the entrypoint for the list command.
func ListCommand(cfg *config.Config) *cobra.Command {
listCmd := &cobra.Command{
Use: "list",
Short: "list КуСфера services running in the runtime (supervised mode)",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg, true))
},
RunE: func(cmd *cobra.Command, args []string) error {
host := viper.GetString("hostname")
port := viper.GetString("port")
client, err := rpc.DialHTTP("tcp", net.JoinHostPort(host, port))
if err != nil {
log.Fatalf("Failed to connect to the runtime. Has the runtime been started and did you configure the right runtime address (\"%s\")", cfg.Runtime.Host+":"+cfg.Runtime.Port)
}
var arg1 string
if err := client.Call("Service.List", struct{}{}, &arg1); err != nil {
log.Fatal(err)
}
fmt.Println(arg1)
return nil
},
}
listCmd.Flags().String("hostname", "localhost", "hostname of the runtime")
_ = viper.BindEnv("hostname", "OC_RUNTIME_HOST")
_ = viper.BindPFlag("hostname", listCmd.Flags().Lookup("hostname"))
listCmd.Flags().String("port", "9250", "port of the runtime")
_ = viper.BindEnv("port", "OC_RUNTIME_PORT")
_ = viper.BindPFlag("port", listCmd.Flags().Lookup("port"))
return listCmd
}
func init() {
register.AddCommand(ListCommand)
}
+380
View File
@@ -0,0 +1,380 @@
package command
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/pkg/xattr"
"github.com/spf13/cobra"
"github.com/theckman/yacspin"
"github.com/vmihailenco/msgpack/v5"
)
// Define the names of the extended attributes we are working with.
const (
parentIDAttrName = "user.oc.parentid"
idAttrName = "user.oc.id"
spaceIDAttrName = "user.oc.space.id"
ownerIDAttrName = "user.oc.owner.id"
)
var (
spinner *yacspin.Spinner
restartRequired = false
)
// EntryInfo holds information about a directory entry.
type EntryInfo struct {
Path string
ModTime time.Time
ParentID string
}
// PosixfsCommand is the entrypoint for the posixfs command.
func PosixfsCommand(cfg *config.Config) *cobra.Command {
posixCmd := &cobra.Command{
Use: "posixfs",
Short: `cli tools to inspect and manipulate a posixfs storage.`,
GroupID: CommandGroupStorage,
}
posixCmd.AddCommand(consistencyCmd(cfg))
return posixCmd
}
func init() {
register.AddCommand(PosixfsCommand)
}
// consistencyCmd returns a command to check the consistency of the posixfs storage.
func consistencyCmd(cfg *config.Config) *cobra.Command {
consCmd := &cobra.Command{
Use: "consistency",
Short: "check the consistency of the posixfs storage",
RunE: func(cmd *cobra.Command, args []string) error {
return checkPosixfsConsistency(cmd, cfg)
},
}
consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage")
_ = consCmd.MarkFlagRequired("root")
return consCmd
}
// checkPosixfsConsistency checks the consistency of the posixfs storage.
func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error {
rootPath, _ := cmd.Flags().GetString("root")
indexesPath := filepath.Join(rootPath, "indexes")
_, err := os.Stat(indexesPath)
if err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath)
}
return fmt.Errorf("error accessing '%s': %w", indexesPath, err)
}
spinnerCfg := yacspin.Config{
Frequency: 100 * time.Millisecond,
CharSet: yacspin.CharSets[11],
StopCharacter: "✓",
StopColors: []string{"fgGreen"},
StopFailCharacter: "✗",
StopFailColors: []string{"fgRed"},
}
spinner, err = yacspin.New(spinnerCfg)
err = spinner.Start()
if err != nil {
return fmt.Errorf("error creating spinner: %w", err)
}
checkSpaces(filepath.Join(rootPath, "users"))
spinner.Suffix(" Personal spaces check ")
spinner.StopMessage("completed\n")
spinner.Stop()
checkSpaces(filepath.Join(rootPath, "projects"))
spinner.Suffix(" Project spaces check ")
spinner.StopMessage("completed")
spinner.Stop()
if restartRequired {
fmt.Println("\n\n ⚠️ Please restart your qsfera instance to apply changes.")
}
return nil
}
func checkSpaces(basePath string) {
dirEntries, err := os.ReadDir(basePath)
if err != nil {
spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath))
spinner.StopFail()
return
}
for _, entry := range dirEntries {
if entry.IsDir() {
fullPath := filepath.Join(basePath, entry.Name())
checkSpace(fullPath)
}
}
}
func checkSpace(spacePath string) {
spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath))
info, err := os.Stat(spacePath)
if err != nil {
logFailure("Error accessing path '%s': %v", spacePath, err)
return
}
if !info.IsDir() {
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
return
}
spaceID, err := xattr.Get(spacePath, spaceIDAttrName)
if err != nil || len(spaceID) == 0 {
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName)
return
}
checkSpaceID(spacePath)
}
func checkSpaceID(spacePath string) {
spinner.Message("checking space ID uniqueness")
entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath)
if err != nil {
logFailure("Failed to gather attributes: %v", err)
return
}
if len(entries) == 0 {
logSuccess("(empty space)")
return
}
if len(uniqueIDs) > 1 {
spinner.Pause()
fmt.Println("\n ⚠ Multiple space IDs found:")
for id := range uniqueIDs {
fmt.Printf(" - %s\n", id)
}
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
targetID := oldestEntry.ParentID
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
reader := bufio.NewReader(os.Stdin)
input, _ := reader.ReadString('\n')
input = strings.TrimSpace(strings.ToLower(input))
if input != "y" {
spinner.Unpause()
logFailure("Operation cancelled by user.")
return
}
restartRequired = true
obsoleteIDs := []string{}
for id := range uniqueIDs {
if id != targetID {
obsoleteIDs = append(obsoleteIDs, id)
}
}
fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
spinner.Unpause()
} else {
logSuccess("")
}
}
func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
// Set all parentid attributes to the proper space ID
err := setAllParentIDAttributes(entries, targetID)
if err != nil {
logFailure("an error occurred during file attribute update: %v", err)
return
}
// Update space ID itself
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID)
err = xattr.Set(spacePath, idAttrName, []byte(targetID))
if err != nil {
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
return
}
err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID))
if err != nil {
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
return
}
// update the index
err = updateOwnerIndexFile(spacePath, obsoleteIDs)
if err != nil {
logFailure("Could not update the owner index file: %v", err)
}
}
func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
dirEntries, err := os.ReadDir(path)
if err != nil {
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
}
var allEntries []EntryInfo
uniqueIDs := make(map[string]struct{})
var oldestEntry EntryInfo
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
for _, entry := range dirEntries {
fullPath := filepath.Join(path, entry.Name())
info, err := os.Stat(fullPath)
if err != nil {
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
continue
}
parentID, err := xattr.Get(fullPath, parentIDAttrName)
if err != nil {
continue // Skip if attribute doesn't exist or can't be read
}
entryInfo := EntryInfo{
Path: fullPath,
ModTime: info.ModTime(),
ParentID: string(parentID),
}
allEntries = append(allEntries, entryInfo)
uniqueIDs[string(parentID)] = struct{}{}
if entryInfo.ModTime.Before(oldestTime) {
oldestTime = entryInfo.ModTime
oldestEntry = entryInfo
}
}
return allEntries, uniqueIDs, oldestEntry, nil
}
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
for _, entry := range entries {
if entry.ParentID == targetID {
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
continue
}
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
if err != nil {
return fmt.Errorf("error walking path '%s': %w", path, err)
}
// Remove all attributes from the file.
if err := removeAttributes(path); err != nil {
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
}
return nil
})
}
return nil
}
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
fmt.Printf(" Rewriting index file '%s'\n", basePath)
ownerID, err := xattr.Get(basePath, ownerIDAttrName)
if err != nil {
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
}
indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk")
indexPath = filepath.Clean(indexPath)
// Read the MessagePack file
fileData, err := os.ReadFile(indexPath)
if err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("index file does not exist, skipping update")
}
return fmt.Errorf("could not read index file: %w", err)
}
var indexMap map[string]string
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
}
// Remove obsolete IDs from the map
itemsRemoved := 0
for _, id := range obsoleteIDs {
if _, exists := indexMap[id]; exists {
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
delete(indexMap, id)
itemsRemoved++
} else {
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
}
}
if itemsRemoved == 0 {
return nil
}
// Write the data back to the file
updatedData, err := msgpack.Marshal(&indexMap)
if err != nil {
return fmt.Errorf("failed to marshal updated index map: %w", err)
}
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
return fmt.Errorf("failed to write updated index file: %w", err)
}
logSuccess("Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
return nil
}
func removeAttributes(path string) error {
attrNames, err := xattr.List(path)
if err != nil {
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
}
for _, attrName := range attrNames {
if err := xattr.Remove(path, attrName); err != nil {
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
}
}
return nil
}
func logFailure(message string, args ...any) {
spinner.StopFailMessage(fmt.Sprintf(message, args...))
spinner.StopFail()
spinner.Start()
}
func logSuccess(message string, args ...any) {
spinner.StopMessage(fmt.Sprintf(message, args...))
spinner.Stop()
spinner.Start()
}
+167
View File
@@ -0,0 +1,167 @@
package command
import (
"errors"
"fmt"
"path/filepath"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/qsfera/pkg/revisions"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
decomposedbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
decomposeds3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/spf13/cobra"
)
var (
// _nodesGlobPattern is the glob pattern to find all nodes
_nodesGlobPattern = "spaces/*/*/nodes/"
)
// RevisionsCommand is the entrypoint for the revisions command.
func RevisionsCommand(cfg *config.Config) *cobra.Command {
revCmd := &cobra.Command{
Use: "revisions",
Short: "КуСфера revisions functionality",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg, true))
},
}
revCmd.AddCommand(PurgeRevisionsCommand(cfg))
return revCmd
}
// PurgeRevisionsCommand allows removing all revisions from a storage provider.
func PurgeRevisionsCommand(cfg *config.Config) *cobra.Command {
revCmd := &cobra.Command{
Use: "purge",
Short: "purge revisions",
RunE: func(cmd *cobra.Command, args []string) error {
basePath, _ := cmd.Flags().GetString("basepath")
var (
bs revisions.DelBlobstore
err error
)
blobstoreFlag, _ := cmd.Flags().GetString("blobstore")
switch blobstoreFlag {
case "decomposeds3":
bs, err = decomposeds3bs.New(
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
decomposeds3bs.Options{},
)
case "decomposed":
bs, err = decomposedbs.New(basePath)
case "none":
bs = nil
default:
err = errors.New("blobstore type not supported")
}
if err != nil {
fmt.Println(err)
return err
}
var rid *provider.ResourceId
resourceIDFlag, _ := cmd.Flags().GetString("resource-id")
resid, err := storagespace.ParseID(resourceIDFlag)
if err == nil {
rid = &resid
}
mechanism, _ := cmd.Flags().GetString("glob-mechanism")
if rid.GetOpaqueId() != "" {
mechanism = "glob"
}
var ch <-chan string
switch mechanism {
default:
fallthrough
case "glob":
p := generatePath(basePath, rid)
if rid.GetOpaqueId() == "" {
p = filepath.Join(p, "*/*/*/*/*")
}
ch = revisions.Glob(p)
case "workers":
p := generatePath(basePath, rid)
ch = revisions.GlobWorkers(p, "/*", "/*/*/*/*")
case "list":
p := filepath.Join(basePath, "spaces")
if rid != nil {
p = generatePath(basePath, rid)
}
ch = revisions.List(p, 10)
}
flagDryRun, err := cmd.Flags().GetBool("dry-run")
if err != nil {
return err
}
flagVerbose, err := cmd.Flags().GetBool("verbose")
if err != nil {
return err
}
files, blobs, revisionResults := revisions.PurgeRevisions(ch, bs, flagDryRun, flagVerbose)
printResults(files, blobs, revisionResults, flagDryRun)
return nil
},
}
revCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/metadata)")
_ = revCmd.MarkFlagRequired("basepath")
revCmd.Flags().StringP("blobstore", "b", "decomposed", "the blobstore type. Can be (none, decomposed, decomposeds3). Default decomposed")
revCmd.Flags().Bool("dry-run", true, "do not delete anything, just print what would be deleted")
revCmd.Flags().BoolP("verbose", "v", false, "print verbose output")
revCmd.Flags().StringP("resource-id", "r", "", "purge all revisions of this file/space. If not set, all revisions will be purged")
revCmd.Flags().String("glob-mechanism", "glob", "the glob mechanism to find all nodes. Can be 'glob', 'list' or 'workers'. 'glob' uses globbing with a single worker. 'workers' spawns multiple go routines, accelatering the command drastically but causing high cpu and ram usage. 'list' looks for references by listing directories with multiple workers. Default is 'glob'")
return revCmd
}
func printResults(countFiles, countBlobs, countRevisions int, dryRun bool) {
switch {
case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
fmt.Println("❎ No revisions found. Storage provider is clean.")
case !dryRun:
fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
default:
fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
}
}
func generatePath(basePath string, rid *provider.ResourceId) string {
if rid == nil {
return filepath.Join(basePath, _nodesGlobPattern)
}
sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
if sid == "" {
return ""
}
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
if nid == "" {
return filepath.Join(basePath, "spaces", sid, "nodes")
}
return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*")
}
func init() {
register.AddCommand(RevisionsCommand)
}
+40
View File
@@ -0,0 +1,40 @@
package command
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/clihelper"
"github.com/qsfera/server/pkg/config"
"github.com/spf13/cobra"
)
// Execute is the entry point for the qsfera command.
func Execute() error {
cfg := config.DefaultConfig()
app := clihelper.DefaultApp(&cobra.Command{
Use: "qsfera",
Short: "qsfera",
})
for _, commandFactory := range register.Commands {
command := commandFactory(cfg)
if command.GroupID != "" && !app.ContainsGroup(command.GroupID) {
app.AddGroup(&cobra.Group{
ID: command.GroupID,
Title: command.GroupID,
})
}
app.AddCommand(command)
}
app.SetArgs(os.Args[1:])
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
return app.ExecuteContext(ctx)
}
+32
View File
@@ -0,0 +1,32 @@
package command
import (
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/qsfera/pkg/runtime"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
"github.com/spf13/cobra"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "server",
Short: "start a fullstack server (runtime and all services in supervised mode)",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg, false))
},
GroupID: CommandGroupServer,
RunE: func(cmd *cobra.Command, args []string) error {
// Prefer the in-memory registry as the default when running in single-binary mode
r := runtime.New(cfg)
return r.Start(cmd.Context())
},
}
}
func init() {
register.AddCommand(Server)
}
+285
View File
@@ -0,0 +1,285 @@
package command
import (
"fmt"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
activitylog "github.com/qsfera/server/services/activitylog/pkg/command"
antivirus "github.com/qsfera/server/services/antivirus/pkg/command"
appprovider "github.com/qsfera/server/services/app-provider/pkg/command"
appregistry "github.com/qsfera/server/services/app-registry/pkg/command"
audit "github.com/qsfera/server/services/audit/pkg/command"
authapp "github.com/qsfera/server/services/auth-app/pkg/command"
authbasic "github.com/qsfera/server/services/auth-basic/pkg/command"
authbearer "github.com/qsfera/server/services/auth-bearer/pkg/command"
authmachine "github.com/qsfera/server/services/auth-machine/pkg/command"
authservice "github.com/qsfera/server/services/auth-service/pkg/command"
clientlog "github.com/qsfera/server/services/clientlog/pkg/command"
collaboration "github.com/qsfera/server/services/collaboration/pkg/command"
eventhistory "github.com/qsfera/server/services/eventhistory/pkg/command"
frontend "github.com/qsfera/server/services/frontend/pkg/command"
gateway "github.com/qsfera/server/services/gateway/pkg/command"
graph "github.com/qsfera/server/services/graph/pkg/command"
groups "github.com/qsfera/server/services/groups/pkg/command"
idm "github.com/qsfera/server/services/idm/pkg/command"
idp "github.com/qsfera/server/services/idp/pkg/command"
invitations "github.com/qsfera/server/services/invitations/pkg/command"
nats "github.com/qsfera/server/services/nats/pkg/command"
notifications "github.com/qsfera/server/services/notifications/pkg/command"
ocm "github.com/qsfera/server/services/ocm/pkg/command"
ocs "github.com/qsfera/server/services/ocs/pkg/command"
policies "github.com/qsfera/server/services/policies/pkg/command"
postprocessing "github.com/qsfera/server/services/postprocessing/pkg/command"
proxy "github.com/qsfera/server/services/proxy/pkg/command"
search "github.com/qsfera/server/services/search/pkg/command"
settings "github.com/qsfera/server/services/settings/pkg/command"
sharing "github.com/qsfera/server/services/sharing/pkg/command"
sse "github.com/qsfera/server/services/sse/pkg/command"
storagepubliclink "github.com/qsfera/server/services/storage-publiclink/pkg/command"
storageshares "github.com/qsfera/server/services/storage-shares/pkg/command"
storagesystem "github.com/qsfera/server/services/storage-system/pkg/command"
storageusers "github.com/qsfera/server/services/storage-users/pkg/command"
thumbnails "github.com/qsfera/server/services/thumbnails/pkg/command"
userlog "github.com/qsfera/server/services/userlog/pkg/command"
users "github.com/qsfera/server/services/users/pkg/command"
web "github.com/qsfera/server/services/web/pkg/command"
webdav "github.com/qsfera/server/services/webdav/pkg/command"
webfinger "github.com/qsfera/server/services/webfinger/pkg/command"
"github.com/spf13/cobra"
)
var serviceCommands = []register.Command{
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Activitylog.Service.Name, activitylog.GetCommands(cfg.Activitylog), func(c *config.Config) {
cfg.Activitylog.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Antivirus.Service.Name, antivirus.GetCommands(cfg.Antivirus), func(c *config.Config) {
cfg.Antivirus.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AppProvider.Service.Name, appprovider.GetCommands(cfg.AppProvider), func(c *config.Config) {
cfg.AppProvider.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AppRegistry.Service.Name, appregistry.GetCommands(cfg.AppRegistry), func(c *config.Config) {
cfg.AppRegistry.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Audit.Service.Name, audit.GetCommands(cfg.Audit), func(c *config.Config) {
cfg.Audit.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AuthApp.Service.Name, authapp.GetCommands(cfg.AuthApp), func(_ *config.Config) {
cfg.AuthApp.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AuthBasic.Service.Name, authbasic.GetCommands(cfg.AuthBasic), func(c *config.Config) {
cfg.AuthBasic.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AuthBearer.Service.Name, authbearer.GetCommands(cfg.AuthBearer), func(c *config.Config) {
cfg.AuthBearer.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AuthMachine.Service.Name, authmachine.GetCommands(cfg.AuthMachine), func(c *config.Config) {
cfg.AuthMachine.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.AuthService.Service.Name, authservice.GetCommands(cfg.AuthService), func(c *config.Config) {
cfg.AuthService.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Clientlog.Service.Name, clientlog.GetCommands(cfg.Clientlog), func(c *config.Config) {
cfg.Clientlog.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Collaboration.Service.Name, collaboration.GetCommands(cfg.Collaboration), func(c *config.Config) {
cfg.Collaboration.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.EventHistory.Service.Name, eventhistory.GetCommands(cfg.EventHistory), func(c *config.Config) {
cfg.EventHistory.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Frontend.Service.Name, frontend.GetCommands(cfg.Frontend), func(c *config.Config) {
cfg.Frontend.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Gateway.Service.Name, gateway.GetCommands(cfg.Gateway), func(c *config.Config) {
cfg.Gateway.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Graph.Service.Name, graph.GetCommands(cfg.Graph), func(c *config.Config) {
cfg.Graph.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Groups.Service.Name, groups.GetCommands(cfg.Groups), func(c *config.Config) {
cfg.Groups.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.IDM.Service.Name, idm.GetCommands(cfg.IDM), func(c *config.Config) {
cfg.IDM.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.IDP.Service.Name, idp.GetCommands(cfg.IDP), func(c *config.Config) {
cfg.IDP.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Invitations.Service.Name, invitations.GetCommands(cfg.Invitations), func(c *config.Config) {
cfg.Invitations.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Nats.Service.Name, nats.GetCommands(cfg.Nats), func(c *config.Config) {
cfg.Nats.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Notifications.Service.Name, notifications.GetCommands(cfg.Notifications), func(c *config.Config) {
cfg.Notifications.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.OCM.Service.Name, ocm.GetCommands(cfg.OCM), func(c *config.Config) {
cfg.OCM.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.OCS.Service.Name, ocs.GetCommands(cfg.OCS), func(c *config.Config) {
cfg.OCS.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Policies.Service.Name, policies.GetCommands(cfg.Policies), func(c *config.Config) {
cfg.Policies.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Postprocessing.Service.Name, postprocessing.GetCommands(cfg.Postprocessing), func(c *config.Config) {
cfg.Postprocessing.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Proxy.Service.Name, proxy.GetCommands(cfg.Proxy), func(c *config.Config) {
cfg.Proxy.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Search.Service.Name, search.GetCommands(cfg.Search), func(c *config.Config) {
cfg.Search.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Settings.Service.Name, settings.GetCommands(cfg.Settings), func(c *config.Config) {
cfg.Settings.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Sharing.Service.Name, sharing.GetCommands(cfg.Sharing), func(c *config.Config) {
cfg.Sharing.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.SSE.Service.Name, sse.GetCommands(cfg.SSE), func(c *config.Config) {
cfg.SSE.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.StoragePublicLink.Service.Name, storagepubliclink.GetCommands(cfg.StoragePublicLink), func(c *config.Config) {
cfg.StoragePublicLink.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.StorageShares.Service.Name, storageshares.GetCommands(cfg.StorageShares), func(c *config.Config) {
cfg.StorageShares.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.StorageSystem.Service.Name, storagesystem.GetCommands(cfg.StorageSystem), func(c *config.Config) {
cfg.StorageSystem.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.StorageUsers.Service.Name, storageusers.GetCommands(cfg.StorageUsers), func(c *config.Config) {
cfg.StorageUsers.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Thumbnails.Service.Name, thumbnails.GetCommands(cfg.Thumbnails), func(c *config.Config) {
cfg.Thumbnails.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Userlog.Service.Name, userlog.GetCommands(cfg.Userlog), func(c *config.Config) {
cfg.Userlog.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Users.Service.Name, users.GetCommands(cfg.Users), func(c *config.Config) {
cfg.Users.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Web.Service.Name, web.GetCommands(cfg.Web), func(c *config.Config) {
cfg.Web.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.WebDAV.Service.Name, webdav.GetCommands(cfg.WebDAV), func(c *config.Config) {
cfg.WebDAV.Commons = cfg.Commons
})
},
func(cfg *config.Config) *cobra.Command {
return ServiceCommand(cfg, cfg.Webfinger.Service.Name, webfinger.GetCommands(cfg.Webfinger), func(c *config.Config) {
cfg.Webfinger.Commons = cfg.Commons
})
},
}
// ServiceCommand composes a cobra command from the given inputs.
func ServiceCommand(cfg *config.Config, serviceName string, subCommands []*cobra.Command, f func(*config.Config)) *cobra.Command {
command := &cobra.Command{
Use: serviceName,
Short: fmt.Sprintf("%s service commands", serviceName),
GroupID: CommandGroupServices,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
configlog.Error(parser.ParseConfig(cfg, true))
f(cfg)
return nil
},
}
command.AddCommand(subCommands...)
return command
}
func init() {
for _, c := range serviceCommands {
register.AddCommand(c)
}
}
+178
View File
@@ -0,0 +1,178 @@
package command
import (
"errors"
"github.com/spf13/viper"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
oclog "github.com/qsfera/server/pkg/log"
mregistry "github.com/qsfera/server/pkg/registry"
sharing "github.com/qsfera/server/services/sharing/pkg/config"
sharingparser "github.com/qsfera/server/services/sharing/pkg/config/parser"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3"
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/rs/zerolog"
"github.com/spf13/cobra"
)
// SharesCommand is the entrypoint for the groups command.
func SharesCommand(cfg *config.Config) *cobra.Command {
sharesCmd := &cobra.Command{
Use: "shares",
Short: `cli tools to manage entries in the share manager.`,
PreRunE: func(cmd *cobra.Command, args []string) error {
// Parse base config
if err := parser.ParseConfig(cfg, true); err != nil {
return configlog.ReturnError(err)
}
// Parse sharing config
cfg.Sharing.Commons = cfg.Commons
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
},
}
sharesCmd.AddCommand(cleanupCmd(cfg))
return sharesCmd
}
func init() {
register.AddCommand(SharesCommand)
}
func cleanupCmd(cfg *config.Config) *cobra.Command {
cleanCmd := &cobra.Command{
Use: "cleanup",
Short: `clean up stale entries in the share manager.`,
PreRunE: func(cmd *cobra.Command, args []string) error {
// Parse base config
if err := parser.ParseConfig(cfg, true); err != nil {
return configlog.ReturnError(err)
}
// Parse sharing config
cfg.Sharing.Commons = cfg.Commons
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
},
RunE: func(cmd *cobra.Command, args []string) error {
return cleanup(cmd, cfg)
},
}
cleanCmd.Flags().String("service-account-id", "", "Name of the service account to use for the cleanup")
_ = cleanCmd.MarkFlagRequired("service-account-id")
_ = viper.BindEnv("service-account-id", "OC_SERVICE_ACCOUNT_ID")
_ = viper.BindPFlag("service-account-id", cleanCmd.Flags().Lookup("service-account-id"))
cleanCmd.Flags().String("service-account-secret", "", "Secret for the service account")
_ = cleanCmd.MarkFlagRequired("service-account-secret")
_ = viper.BindEnv("service-account-secret", "OC_SERVICE_ACCOUNT_SECRET")
_ = viper.BindPFlag("service-account-secret", cleanCmd.Flags().Lookup("service-account-secret"))
return cleanCmd
}
func cleanup(_ *cobra.Command, cfg *config.Config) error {
driver := cfg.Sharing.UserSharingDriver
// cleanup is only implemented for the jsoncs3 share manager
if driver != "jsoncs3" {
return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager"))
}
l := logger()
zerolog.SetGlobalLevel(zerolog.InfoLevel)
rcfg := revaShareConfig(cfg.Sharing)
f, ok := registry.NewFuncs[driver]
if !ok {
return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'"))
}
mgr, err := f(rcfg[driver].(map[string]any), l)
if err != nil {
return configlog.ReturnError(err)
}
// Initialize registry to make service lookup work
_ = mregistry.GetRegistry()
// get an authenticated context
gatewaySelector, err := pool.GatewaySelector(cfg.Sharing.Reva.Address)
if err != nil {
return configlog.ReturnError(err)
}
client, err := gatewaySelector.Next()
if err != nil {
return configlog.ReturnError(err)
}
serviceAccountIDFlag := viper.GetString("service-account-id")
serviceAccountSecretFlag := viper.GetString("service-account-secret")
serviceUserCtx, err := utils.GetServiceUserContext(serviceAccountIDFlag, client, serviceAccountSecretFlag)
if err != nil {
return configlog.ReturnError(err)
}
serviceUserCtx = l.WithContext(serviceUserCtx)
mgr.(*jsoncs3.Manager).CleanupStaleShares(serviceUserCtx)
return nil
}
func revaShareConfig(cfg *sharing.Config) map[string]any {
return map[string]any{
"json": map[string]any{
"file": cfg.UserSharingDrivers.JSON.File,
"gateway_addr": cfg.Reva.Address,
},
"sql": map[string]any{ // cernbox sql
"db_username": cfg.UserSharingDrivers.SQL.DBUsername,
"db_password": cfg.UserSharingDrivers.SQL.DBPassword,
"db_host": cfg.UserSharingDrivers.SQL.DBHost,
"db_port": cfg.UserSharingDrivers.SQL.DBPort,
"db_name": cfg.UserSharingDrivers.SQL.DBName,
"password_hash_cost": cfg.UserSharingDrivers.SQL.PasswordHashCost,
"enable_expired_shares_cleanup": cfg.UserSharingDrivers.SQL.EnableExpiredSharesCleanup,
"janitor_run_interval": cfg.UserSharingDrivers.SQL.JanitorRunInterval,
},
"owncloudsql": map[string]any{
"gateway_addr": cfg.Reva.Address,
"storage_mount_id": cfg.UserSharingDrivers.OwnCloudSQL.UserStorageMountID,
"db_username": cfg.UserSharingDrivers.OwnCloudSQL.DBUsername,
"db_password": cfg.UserSharingDrivers.OwnCloudSQL.DBPassword,
"db_host": cfg.UserSharingDrivers.OwnCloudSQL.DBHost,
"db_port": cfg.UserSharingDrivers.OwnCloudSQL.DBPort,
"db_name": cfg.UserSharingDrivers.OwnCloudSQL.DBName,
},
"cs3": map[string]any{
"gateway_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
"provider_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
"service_user_id": cfg.UserSharingDrivers.CS3.SystemUserID,
"service_user_idp": cfg.UserSharingDrivers.CS3.SystemUserIDP,
"machine_auth_apikey": cfg.UserSharingDrivers.CS3.SystemUserAPIKey,
},
"jsoncs3": map[string]any{
"gateway_addr": cfg.Reva.Address,
"provider_addr": cfg.UserSharingDrivers.JSONCS3.ProviderAddr,
"service_user_id": cfg.UserSharingDrivers.JSONCS3.SystemUserID,
"service_user_idp": cfg.UserSharingDrivers.JSONCS3.SystemUserIDP,
"machine_auth_apikey": cfg.UserSharingDrivers.JSONCS3.SystemUserAPIKey,
},
}
}
func logger() *zerolog.Logger {
log := oclog.NewLogger(
oclog.Name("migrate"),
oclog.Level("info"),
oclog.Pretty(true),
oclog.Color(true)).Logger
return &log
}
+57
View File
@@ -0,0 +1,57 @@
package command
import (
"fmt"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/qsfera/pkg/trash"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/config/parser"
"github.com/spf13/cobra"
)
func TrashCommand(cfg *config.Config) *cobra.Command {
trashCmd := &cobra.Command{
Use: "trash",
Short: "КуСфера trash functionality",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg, true))
},
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Read the docs")
return nil
},
}
trashCmd.AddCommand(TrashPurgeEmptyDirsCommand(cfg))
return trashCmd
}
func TrashPurgeEmptyDirsCommand(cfg *config.Config) *cobra.Command {
trashPurgeCmd := &cobra.Command{
Use: "purge-empty-dirs",
Short: "purge empty directories",
RunE: func(cmd *cobra.Command, args []string) error {
basePath, _ := cmd.Flags().GetString("basepath")
dryRun, _ := cmd.Flags().GetBool("dry-run")
if err := trash.PurgeTrashEmptyPaths(basePath, dryRun); err != nil {
fmt.Println(err)
return err
}
return nil
},
}
trashPurgeCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/users)")
_ = trashPurgeCmd.MarkFlagRequired("basepath")
trashPurgeCmd.Flags().Bool("dry-run", true, "do not delete anything, just print what would be deleted")
return trashPurgeCmd
}
func init() {
register.AddCommand(TrashCommand)
}
+80
View File
@@ -0,0 +1,80 @@
package command
import (
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/qsfera/server/qsfera/pkg/register"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/registry"
"github.com/qsfera/server/pkg/version"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
mreg "go-micro.dev/v4/registry"
)
const (
_skipServiceListingFlagName = "skip-services"
)
// VersionCommand is the entrypoint for the version command.
func VersionCommand(cfg *config.Config) *cobra.Command {
versionCmd := &cobra.Command{
Use: "version",
Short: "print the version of this binary and all running service instances",
GroupID: CommandGroupServer,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Edition: %s\n", version.Edition)
fmt.Printf("Compiled: %s\n", version.Compiled())
skipServiceListing, _ := cmd.Flags().GetBool(_skipServiceListingFlagName)
if skipServiceListing {
return nil
}
fmt.Print("\n")
reg := registry.GetRegistry()
serviceList, err := reg.ListServices()
if err != nil {
fmt.Printf("could not list services: %v\n", err)
return err
}
var services []*mreg.Service
for _, s := range serviceList {
s, err := reg.GetService(s.Name)
if err != nil {
fmt.Printf("could not get service: %v\n", err)
return err
}
services = append(services, s...)
}
if len(services) == 0 {
fmt.Println("No running services found.")
return nil
}
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
table.Header([]string{"Version", "Address", "Id"})
for _, s := range services {
for _, n := range s.Nodes {
table.Append([]string{s.Version, n.Address, n.Id})
}
}
table.Render()
return nil
},
}
versionCmd.Flags().Bool(_skipServiceListingFlagName, false, "skip service listing")
return versionCmd
}
func init() {
register.AddCommand(VersionCommand)
}
+108
View File
@@ -0,0 +1,108 @@
package init
import (
"fmt"
"io"
"log"
"os"
"os/exec"
"path"
"time"
)
func checkConfigPath(configPath string) error {
targetPath := path.Join(configPath, configFilename)
if _, err := os.Stat(targetPath); err == nil {
return fmt.Errorf("config in %s already exists", targetPath)
}
return nil
}
func configExists(configPath string) bool {
targetPath := path.Join(configPath, configFilename)
if _, err := os.Stat(targetPath); err == nil {
return true
}
return false
}
func backupQsferaConfigFile(configPath string) (string, error) {
sourceConfig := path.Join(configPath, configFilename)
targetBackupConfig := path.Join(configPath, configFilename+"."+time.Now().Format("2006-01-02-15-04-05")+".backup")
source, err := os.Open(sourceConfig)
if err != nil {
log.Fatalf("Could not read %s (%s)", sourceConfig, err)
}
defer source.Close()
target, err := os.Create(targetBackupConfig)
if err != nil {
log.Fatalf("Could not generate backup %s (%s)", targetBackupConfig, err)
}
defer target.Close()
_, err = io.Copy(target, source)
if err != nil {
log.Fatalf("Could not write backup %s (%s)", targetBackupConfig, err)
}
return targetBackupConfig, nil
}
// printBanner prints the generated qsfera config banner.
func printBanner(targetPath, ocAdminServicePassword, targetBackupConfig string) {
fmt.Printf(
"\n=========================================\n"+
" generated КуСфера Config\n"+
"=========================================\n"+
" configpath : %s\n"+
" user : admin\n"+
" password : %s\n\n",
targetPath, ocAdminServicePassword)
if targetBackupConfig != "" {
fmt.Printf("\n=========================================\n"+
"An older config file has been backed up to\n %s\n\n",
targetBackupConfig)
}
}
// writeConfig writes the config to the target path and prints a banner
func writeConfig(configPath, ocAdminServicePassword, targetBackupConfig string, yamlOutput []byte) error {
targetPath := path.Join(configPath, configFilename)
err := os.WriteFile(targetPath, yamlOutput, 0600)
if err != nil {
return err
}
printBanner(targetPath, ocAdminServicePassword, targetBackupConfig)
return nil
}
// writePatch writes the diff to a file
func writePatch(configPath string, yamlOutput []byte) error {
fmt.Println("running in diff mode")
tmpFile := path.Join(configPath, "qsfera.yaml.tmp")
err := os.WriteFile(tmpFile, yamlOutput, 0600)
if err != nil {
return err
}
fmt.Println("diff -u " + path.Join(configPath, configFilename) + " " + tmpFile)
cmd := exec.Command("diff", "-u", path.Join(configPath, configFilename), tmpFile)
stdout, err := cmd.Output()
if err == nil {
err = os.Remove(tmpFile)
if err != nil {
return err
}
fmt.Println("no changes, your config is up to date")
return nil
}
fmt.Println(string(stdout))
err = os.Remove(tmpFile)
if err != nil {
return err
}
patchPath := path.Join(configPath, "qsfera.config.patch")
err = os.WriteFile(patchPath, stdout, 0600)
if err != nil {
return err
}
fmt.Printf("diff written to %s\n", patchPath)
return nil
}
+315
View File
@@ -0,0 +1,315 @@
package init
import (
"fmt"
"os"
"path"
"github.com/google/uuid"
"gopkg.in/yaml.v2"
"github.com/qsfera/server/pkg/generators"
)
const (
configFilename = "qsfera.yaml"
passwordLength = 32
)
var (
_insecureService = InsecureService{Insecure: true}
_insecureEvents = Events{TLSInsecure: true}
)
// CreateConfig creates a config file with random passwords at configPath
func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword string) error {
if diff && forceOverwrite {
return fmt.Errorf("diff and force-overwrite flags are mutually exclusive")
}
if diff && adminPassword != "" {
return fmt.Errorf("diff and admin-password flags are mutually exclusive")
}
if configExists(configPath) && !forceOverwrite && !diff {
return fmt.Errorf("config file already exists, use --force-overwrite to overwrite or --diff to show diff")
}
err := checkConfigPath(configPath)
if err != nil && (!forceOverwrite && !diff) {
fmt.Println("off")
return err
}
targetBackupConfig := ""
if err != nil {
targetBackupConfig, err = backupQsferaConfigFile(configPath)
if err != nil {
return err
}
}
err = os.MkdirAll(configPath, 0700)
if err != nil {
return err
}
// Load old config
var oldCfg QsferaConfig
if diff {
fp, err := os.ReadFile(path.Join(configPath, configFilename))
if err != nil {
return err
}
err = yaml.Unmarshal(fp, &oldCfg)
if err != nil {
return err
}
}
var (
systemUserID, adminUserID, graphApplicationID, storageUsersMountID, serviceAccountID string
idmServicePassword, idpServicePassword, ocAdminServicePassword, revaServicePassword string
tokenManagerJwtSecret, collaborationWOPISecret, machineAuthAPIKey, systemUserAPIKey string
revaTransferSecret, thumbnailsTransferSecret, serviceAccountSecret, urlSigningSecret string
)
if diff {
systemUserID = oldCfg.SystemUserID
adminUserID = oldCfg.AdminUserID
graphApplicationID = oldCfg.Graph.Application.ID
storageUsersMountID = oldCfg.Gateway.StorageRegistry.StorageUsersMountID
serviceAccountID = oldCfg.Graph.ServiceAccount.ServiceAccountID
idmServicePassword = oldCfg.Idm.ServiceUserPasswords.IdmPassword
idpServicePassword = oldCfg.Idm.ServiceUserPasswords.IdpPassword
ocAdminServicePassword = oldCfg.Idm.ServiceUserPasswords.AdminPassword
revaServicePassword = oldCfg.Idm.ServiceUserPasswords.RevaPassword
tokenManagerJwtSecret = oldCfg.TokenManager.JWTSecret
collaborationWOPISecret = oldCfg.Collaboration.WopiApp.Secret
if collaborationWOPISecret == "" {
collaborationWOPISecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random wopi secret for collaboration service: %s", err)
}
}
machineAuthAPIKey = oldCfg.MachineAuthAPIKey
systemUserAPIKey = oldCfg.SystemUserAPIKey
revaTransferSecret = oldCfg.TransferSecret
thumbnailsTransferSecret = oldCfg.Thumbnails.Thumbnail.TransferSecret
serviceAccountSecret = oldCfg.Graph.ServiceAccount.ServiceAccountSecret
urlSigningSecret = oldCfg.URLSigningSecret
if urlSigningSecret == "" {
urlSigningSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random secret for urlSigningSecret: %s", err)
}
}
} else {
systemUserID = uuid.NewString()
adminUserID = uuid.NewString()
graphApplicationID = uuid.NewString()
storageUsersMountID = uuid.NewString()
serviceAccountID = uuid.NewString()
idmServicePassword, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for idm: %s", err)
}
idpServicePassword, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for idp: %s", err)
}
ocAdminServicePassword = adminPassword
if ocAdminServicePassword == "" {
ocAdminServicePassword, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for qsfera admin: %s", err)
}
}
revaServicePassword, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for reva: %s", err)
}
tokenManagerJwtSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for tokenmanager: %s", err)
}
collaborationWOPISecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random wopi secret for collaboration service: %s", err)
}
machineAuthAPIKey, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for machineauthsecret: %s", err)
}
systemUserAPIKey, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random system user API key: %s", err)
}
revaTransferSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for revaTransferSecret: %s", err)
}
urlSigningSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random secret for urlSigningSecret: %s", err)
}
thumbnailsTransferSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random password for thumbnailsTransferSecret: %s", err)
}
serviceAccountSecret, err = generators.GenerateRandomPassword(passwordLength)
if err != nil {
return fmt.Errorf("could not generate random secret for serviceAccountSecret: %s", err)
}
}
serviceAccount := ServiceAccount{
ServiceAccountID: serviceAccountID,
ServiceAccountSecret: serviceAccountSecret,
}
cfg := QsferaConfig{
TokenManager: TokenManager{
JWTSecret: tokenManagerJwtSecret,
},
MachineAuthAPIKey: machineAuthAPIKey,
SystemUserAPIKey: systemUserAPIKey,
TransferSecret: revaTransferSecret,
URLSigningSecret: urlSigningSecret,
SystemUserID: systemUserID,
AdminUserID: adminUserID,
Idm: IdmService{
ServiceUserPasswords: ServiceUserPasswordsSettings{
AdminPassword: ocAdminServicePassword,
IdpPassword: idpServicePassword,
RevaPassword: revaServicePassword,
IdmPassword: idmServicePassword,
},
},
Idp: LdapBasedService{
Ldap: LdapSettings{
BindPassword: idpServicePassword,
},
},
AuthBasic: AuthbasicService{
AuthProviders: LdapBasedService{
Ldap: LdapSettings{
BindPassword: revaServicePassword,
},
},
},
Collaboration: Collaboration{
WopiApp: WopiApp{
Secret: collaborationWOPISecret,
},
},
Groups: UsersAndGroupsService{
Drivers: LdapBasedService{
Ldap: LdapSettings{
BindPassword: revaServicePassword,
},
},
},
Users: UsersAndGroupsService{
Drivers: LdapBasedService{
Ldap: LdapSettings{
BindPassword: revaServicePassword,
},
},
},
Graph: GraphService{
Application: GraphApplication{
ID: graphApplicationID,
},
Identity: LdapBasedService{
Ldap: LdapSettings{
BindPassword: idmServicePassword,
},
},
ServiceAccount: serviceAccount,
},
Thumbnails: ThumbnailService{
Thumbnail: ThumbnailSettings{
TransferSecret: thumbnailsTransferSecret,
},
},
Gateway: Gateway{
StorageRegistry: StorageRegistry{
StorageUsersMountID: storageUsersMountID,
},
},
StorageUsers: StorageUsers{
MountID: storageUsersMountID,
ServiceAccount: serviceAccount,
},
Userlog: Userlog{
ServiceAccount: serviceAccount,
},
AuthService: AuthService{
ServiceAccount: serviceAccount,
},
Search: Search{
ServiceAccount: serviceAccount,
},
Notifications: Notifications{
ServiceAccount: serviceAccount,
},
Frontend: FrontendService{
ServiceAccount: serviceAccount,
},
Ocm: OcmService{
ServiceAccount: serviceAccount,
},
Clientlog: Clientlog{
ServiceAccount: serviceAccount,
},
Proxy: ProxyService{
ServiceAccount: serviceAccount,
},
Settings: SettingsService{
ServiceAccountIDs: []string{serviceAccount.ServiceAccountID},
},
Activitylog: Activitylog{
ServiceAccount: serviceAccount,
},
Sharing: Sharing{
ServiceAccount: serviceAccount,
},
}
if insecure {
cfg.AuthBearer = AuthbearerService{
AuthProviders: AuthProviderSettings{Oidc: _insecureService},
}
cfg.Collaboration.App.Insecure = true
cfg.Frontend.AppHandler = _insecureService
cfg.Frontend.Archiver = _insecureService
cfg.Frontend.OCDav = _insecureService
cfg.Graph.Spaces = _insecureService
cfg.Graph.Events = _insecureEvents
cfg.Notifications.Notifications.Events = _insecureEvents
cfg.Search.Events = _insecureEvents
cfg.Audit.Events = _insecureEvents
cfg.Sharing.Events = _insecureEvents
cfg.StorageUsers.Events = _insecureEvents
cfg.Nats.Nats.TLSSkipVerifyClientCert = true
cfg.Proxy = ProxyService{
InsecureBackends: true,
OIDC: InsecureProxyOIDC{
Insecure: true,
},
ServiceAccount: serviceAccount,
}
cfg.Thumbnails.Thumbnail.WebdavAllowInsecure = true
cfg.Thumbnails.Thumbnail.Cs3AllowInsecure = true
}
yamlOutput, err := yaml.Marshal(cfg)
if err != nil {
return fmt.Errorf("could not marshall config into yaml: %s", err)
}
if diff {
return writePatch(configPath, yamlOutput)
}
return writeConfig(configPath, ocAdminServicePassword, targetBackupConfig, yamlOutput)
}
+253
View File
@@ -0,0 +1,253 @@
package init
// TODO: use the qsfera config struct instead of this custom struct
// We can't use it right now, because it would need "omitempty" on
// all elements, in order to produce a slim config file with `qsfera init`.
// We can't just add these "omitempty" tags, since we want to generate
// full example configuration files with that struct, too.
// Proposed solution to get rid of this temporary solution:
// - use the qsfera config struct
// - set the needed values like below
// - marshal it to yaml
// - unmarshal it into yaml.Node
// - recurse through the nodes and delete empty / default ones
// - marshal it to yaml
// QsferaConfig is the configuration for the КуСфера services
type QsferaConfig struct {
TokenManager TokenManager `yaml:"token_manager"`
MachineAuthAPIKey string `yaml:"machine_auth_api_key"`
SystemUserAPIKey string `yaml:"system_user_api_key"`
TransferSecret string `yaml:"transfer_secret"`
URLSigningSecret string `yaml:"url_signing_secret"`
SystemUserID string `yaml:"system_user_id"`
AdminUserID string `yaml:"admin_user_id"`
Graph GraphService `yaml:"graph"`
Idp LdapBasedService `yaml:"idp"`
Idm IdmService `yaml:"idm"`
Collaboration Collaboration `yaml:"collaboration"`
Proxy ProxyService `yaml:"proxy"`
Frontend FrontendService `yaml:"frontend"`
AuthBasic AuthbasicService `yaml:"auth_basic"`
AuthBearer AuthbearerService `yaml:"auth_bearer"`
Users UsersAndGroupsService `yaml:"users"`
Groups UsersAndGroupsService `yaml:"groups"`
Ocm OcmService `yaml:"ocm"`
Thumbnails ThumbnailService `yaml:"thumbnails"`
Search Search `yaml:"search"`
Audit Audit `yaml:"audit"`
Settings SettingsService `yaml:"settings"`
Sharing Sharing `yaml:"sharing"`
StorageUsers StorageUsers `yaml:"storage_users"`
Notifications Notifications `yaml:"notifications"`
Nats Nats `yaml:"nats"`
Gateway Gateway `yaml:"gateway"`
Userlog Userlog `yaml:"userlog"`
AuthService AuthService `yaml:"auth_service"`
Clientlog Clientlog `yaml:"clientlog"`
Activitylog Activitylog `yaml:"activitylog"`
}
// Activitylog is the configuration for the activitylog service
type Activitylog struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// App is the configuration for the collaboration service
type App struct {
Insecure bool `yaml:"insecure"`
}
// Audit is the configuration for the audit service
type Audit struct {
Events Events
}
// AuthbasicService is the configuration for the authbasic service
type AuthbasicService struct {
AuthProviders LdapBasedService `yaml:"auth_providers"`
}
// AuthbearerService is the configuration for the authbearer service
type AuthbearerService struct {
AuthProviders AuthProviderSettings `yaml:"auth_providers"`
}
// AuthProviderSettings is the configuration for the auth provider settings
type AuthProviderSettings struct {
Oidc InsecureService
}
// AuthService is the configuration for the auth service
type AuthService struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// Clientlog is the configuration for the clientlog service
type Clientlog struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// Collaboration is the configuration for the collaboration service
type Collaboration struct {
WopiApp WopiApp `yaml:"wopi"`
App App `yaml:"app"`
}
// Events is the configuration for events
type Events struct {
TLSInsecure bool `yaml:"tls_insecure"`
}
// FrontendService is the configuration for the frontend service
type FrontendService struct {
AppHandler InsecureService `yaml:"app_handler"`
Archiver InsecureService
ServiceAccount ServiceAccount `yaml:"service_account"`
OCDav InsecureService
}
// Gateway is the configuration for the gateway
type Gateway struct {
StorageRegistry StorageRegistry `yaml:"storage_registry"`
}
// GraphApplication is the configuration for the graph application
type GraphApplication struct {
ID string `yaml:"id"`
}
// GraphService is the configuration for the graph service
type GraphService struct {
Application GraphApplication
Events Events
Spaces InsecureService
Identity LdapBasedService
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// IdmService is the configuration for the IDM service
type IdmService struct {
ServiceUserPasswords ServiceUserPasswordsSettings `yaml:"service_user_passwords"`
}
// InsecureProxyOIDC is the configuration for the insecure proxy OIDC
type InsecureProxyOIDC struct {
Insecure bool `yaml:"insecure"`
}
// InsecureService is the configuration for services that can be insecure
type InsecureService struct {
Insecure bool
}
// LdapBasedService is the configuration for LDAP based services
type LdapBasedService struct {
Ldap LdapSettings
}
// LdapSettings is the configuration for LDAP settings
type LdapSettings struct {
BindPassword string `yaml:"bind_password"`
}
// Nats is the configuration for the nats service
type Nats struct {
// The nats config has a field called nats
Nats struct {
TLSSkipVerifyClientCert bool `yaml:"tls_skip_verify_client_cert"`
}
}
// Notifications is the configuration for the notifications service
type Notifications struct {
Notifications struct{ Events Events } // The notifications config has a field called notifications
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// OcmService is the configuration for the OCM service
type OcmService struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// ProxyService is the configuration for the proxy service
type ProxyService struct {
OIDC InsecureProxyOIDC `yaml:"oidc"`
InsecureBackends bool `yaml:"insecure_backends"`
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// Search is the configuration for the search service
type Search struct {
Events Events
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// ServiceAccount is the configuration for the used service account
type ServiceAccount struct {
ServiceAccountID string `yaml:"service_account_id"`
ServiceAccountSecret string `yaml:"service_account_secret"`
}
// ServiceUserPasswordsSettings is the configuration for service user passwords
type ServiceUserPasswordsSettings struct {
AdminPassword string `yaml:"admin_password"`
IdmPassword string `yaml:"idm_password"`
RevaPassword string `yaml:"reva_password"`
IdpPassword string `yaml:"idp_password"`
}
// SettingsService is the configuration for the settings service
type SettingsService struct {
ServiceAccountIDs []string `yaml:"service_account_ids"`
}
// Sharing is the configuration for the sharing service
type Sharing struct {
Events Events
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// StorageRegistry is the configuration for the storage registry
type StorageRegistry struct {
StorageUsersMountID string `yaml:"storage_users_mount_id"`
}
// StorageUsers is the configuration for the storage users
type StorageUsers struct {
Events Events
MountID string `yaml:"mount_id"`
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// ThumbnailSettings is the configuration for the thumbnail settings
type ThumbnailSettings struct {
TransferSecret string `yaml:"transfer_secret"`
WebdavAllowInsecure bool `yaml:"webdav_allow_insecure"`
Cs3AllowInsecure bool `yaml:"cs3_allow_insecure"`
}
// ThumbnailService is the configuration for the thumbnail service
type ThumbnailService struct {
Thumbnail ThumbnailSettings
}
// TokenManager is the configuration for the token manager
type TokenManager struct {
JWTSecret string `yaml:"jwt_secret"`
}
// UsersAndGroupsService is the configuration for the users and groups service
type UsersAndGroupsService struct {
Drivers LdapBasedService
}
// Userlog is the configuration for the userlog service
type Userlog struct {
ServiceAccount ServiceAccount `yaml:"service_account"`
}
// WopiApp is the configuration for the WOPI app
type WopiApp struct {
Secret string `yaml:"secret"`
}
+23
View File
@@ -0,0 +1,23 @@
package register
import (
"github.com/spf13/cobra"
"github.com/qsfera/server/pkg/config"
)
var (
// Commands define the slice of commands.
Commands []Command
)
// Command defines the register command.
type Command func(*config.Config) *cobra.Command
// AddCommand appends a command to Commands.
func AddCommand(cmd Command) {
Commands = append(
Commands,
cmd,
)
}
+284
View File
@@ -0,0 +1,284 @@
// Package revisions allows manipulating revisions in a storage provider.
package revisions
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/vmihailenco/msgpack/v5"
)
var (
// regex to determine if a node versioned. Examples:
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk
// 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock
_versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`)
_spaceIDRegex = regexp.MustCompile(`/spaces/(.+)/nodes/`)
)
// DelBlobstore is the interface for a blobstore that can delete blobs.
type DelBlobstore interface {
Delete(node *node.Node) error
}
// Glob uses globbing to find all revision nodes in a storage provider.
func Glob(pattern string) <-chan string {
ch := make(chan string)
go func() {
defer close(ch)
nodes, err := filepath.Glob(filepath.Join(pattern))
if err != nil {
fmt.Println("error globbing", pattern, err)
return
}
if len(nodes) == 0 {
fmt.Println("no nodes found. Double check storage path")
return
}
for _, n := range nodes {
if _versionRegex.MatchString(n) {
ch <- n
}
}
}()
return ch
}
// GlobWorkers uses multiple go routine to glob all revision nodes in a storage provider.
func GlobWorkers(pattern string, depth string, remainder string) <-chan string {
wg := sync.WaitGroup{}
ch := make(chan string)
go func() {
defer close(ch)
nodes, err := filepath.Glob(pattern + depth)
if err != nil {
fmt.Println("error globbing", pattern, err)
return
}
if len(nodes) == 0 {
fmt.Println("no nodes found. Double check storage path")
return
}
for _, node := range nodes {
wg.Add(1)
go func(node string) {
defer wg.Done()
nodes, err := filepath.Glob(node + remainder)
if err != nil {
fmt.Println("error globbing", node, err)
return
}
for _, n := range nodes {
if _versionRegex.MatchString(n) {
ch <- n
}
}
}(node)
}
wg.Wait()
}()
return ch
}
// Walk walks the storage provider to find all revision nodes.
func Walk(base string) <-chan string {
ch := make(chan string)
go func() {
defer close(ch)
err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
if err != nil {
fmt.Println("error walking", base, err)
return err
}
if !_versionRegex.MatchString(info.Name()) {
return nil
}
ch <- path
return nil
})
if err != nil {
fmt.Println("error walking", base, err)
return
}
}()
return ch
}
// List uses directory listing to find all revision nodes in a storage provider.
func List(base string, workers int) <-chan string {
ch := make(chan string)
go func() {
defer close(ch)
if err := listFolder(base, ch, make(chan struct{}, workers)); err != nil {
fmt.Println("error listing", base, err)
return
}
}()
return ch
}
// PurgeRevisions removes all revisions from a storage provider.
func PurgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) {
countFiles := 0
countBlobs := 0
countRevisions := 0
var err error
for d := range nodes {
if !_versionRegex.MatchString(d) {
continue
}
var (
spaceID, blobID string
)
e := filepath.Ext(d)
switch e {
case ".mpk":
blobID, err = getBlobID(d)
if err != nil {
fmt.Printf("error getting blobID from %s: %v\n", d, err)
continue
}
matches := _spaceIDRegex.FindStringSubmatch(d)
if len(matches) != 2 {
fmt.Printf("error extracting spaceID from %s\n", d)
continue
}
spaceID = strings.ReplaceAll(matches[1], "/", "")
countBlobs++
case ".mlock":
// no extra action on .mlock files
default:
countRevisions++
}
if !dryRun {
if blobID != "" {
// TODO: needs spaceID for decomposeds3
if err := bs.Delete(&node.Node{BaseNode: node.BaseNode{SpaceID: spaceID}, BlobID: blobID}); err != nil {
fmt.Printf("error deleting blob %s: %v\n", blobID, err)
continue
}
}
if err := os.Remove(d); err != nil {
fmt.Printf("error removing %s: %v\n", d, err)
continue
}
}
countFiles++
if verbose {
spaceID, nodeID := getIDsFromPath(d)
if dryRun {
fmt.Println("Would delete")
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
fmt.Println("\tSpaceID:", spaceID)
fmt.Println("\tPath:", d)
if blobID != "" {
fmt.Println("\tBlob:", blobID)
}
} else {
fmt.Println("Deleted")
fmt.Println("\tResourceID:", spaceID+"!"+nodeID)
fmt.Println("\tSpaceID:", spaceID)
fmt.Println("\tPath:", d)
if blobID != "" {
fmt.Println("\tBlob:", blobID)
}
}
}
}
return countFiles, countBlobs, countRevisions
}
func listFolder(path string, ch chan<- string, workers chan struct{}) error {
workers <- struct{}{}
wg := sync.WaitGroup{}
children, err := os.ReadDir(path)
if err != nil {
<-workers
return err
}
for _, child := range children {
if child.IsDir() {
wg.Go(func() {
if err := listFolder(filepath.Join(path, child.Name()), ch, workers); err != nil {
fmt.Println("error listing", path, err)
}
})
}
if _versionRegex.MatchString(child.Name()) {
ch <- filepath.Join(path, child.Name())
}
}
<-workers
wg.Wait()
return nil
}
func getBlobID(path string) (string, error) {
b, err := os.ReadFile(path)
if err != nil {
return "", err
}
m := map[string][]byte{}
if err := msgpack.Unmarshal(b, &m); err != nil {
return "", err
}
if bid := m["user.oc.blobid"]; string(bid) != "" {
return string(bid), nil
}
return "", nil
}
func getIDsFromPath(path string) (string, string) {
rawIDs := strings.Split(path, "/nodes/")
if len(rawIDs) != 2 {
return "", ""
}
s := strings.Split(rawIDs[0], "/spaces/")
if len(s) != 2 {
return "", ""
}
n := strings.Split(rawIDs[1], ".REV.")
if len(n) != 2 {
return "", ""
}
spaceID := strings.Replace(s[1], "/", "", -1)
nodeID := strings.Replace(n[0], "/", "", -1)
return spaceID, filepath.Base(nodeID)
}
@@ -0,0 +1,170 @@
package revisions
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
"testing"
"github.com/google/uuid"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/test-go/testify/require"
)
var (
_basePath = "/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/"
)
// func TestInit(t *testing.T) {
// initialize(10, 2)
// defer os.RemoveAll("test_temp")
// }
func TestGlob30(t *testing.T) { test(t, 10, 2, glob) }
func TestGlob80(t *testing.T) { test(t, 20, 3, glob) }
func TestGlob250(t *testing.T) { test(t, 50, 4, glob) }
func TestGlob600(t *testing.T) { test(t, 100, 5, glob) }
func TestWalk30(t *testing.T) { test(t, 10, 2, walk) }
func TestWalk80(t *testing.T) { test(t, 20, 3, walk) }
func TestWalk250(t *testing.T) { test(t, 50, 4, walk) }
func TestWalk600(t *testing.T) { test(t, 100, 5, walk) }
func TestList30(t *testing.T) { test(t, 10, 2, list2) }
func TestList80(t *testing.T) { test(t, 20, 3, list10) }
func TestList250(t *testing.T) { test(t, 50, 4, list20) }
func TestList600(t *testing.T) { test(t, 100, 5, list2) }
func TestGlobWorkers30(t *testing.T) { test(t, 10, 2, globWorkersD1) }
func TestGlobWorkers80(t *testing.T) { test(t, 20, 3, globWorkersD2) }
func TestGlobWorkers250(t *testing.T) { test(t, 50, 4, globWorkersD4) }
func TestGlobWorkers600(t *testing.T) { test(t, 100, 5, globWorkersD2) }
func BenchmarkGlob30(b *testing.B) { benchmark(b, 10, 2, glob) }
func BenchmarkWalk30(b *testing.B) { benchmark(b, 10, 2, walk) }
func BenchmarkList30(b *testing.B) { benchmark(b, 10, 2, list2) }
func BenchmarkGlobWorkers30(b *testing.B) { benchmark(b, 10, 2, globWorkersD2) }
func BenchmarkGlob80(b *testing.B) { benchmark(b, 20, 3, glob) }
func BenchmarkWalk80(b *testing.B) { benchmark(b, 20, 3, walk) }
func BenchmarkList80(b *testing.B) { benchmark(b, 20, 3, list2) }
func BenchmarkGlobWorkers80(b *testing.B) { benchmark(b, 20, 3, globWorkersD2) }
func BenchmarkGlob250(b *testing.B) { benchmark(b, 50, 4, glob) }
func BenchmarkWalk250(b *testing.B) { benchmark(b, 50, 4, walk) }
func BenchmarkList250(b *testing.B) { benchmark(b, 50, 4, list2) }
func BenchmarkGlobWorkers250(b *testing.B) { benchmark(b, 50, 4, globWorkersD2) }
func BenchmarkGlobAT600(b *testing.B) { benchmark(b, 100, 5, glob) }
func BenchmarkWalkAT600(b *testing.B) { benchmark(b, 100, 5, walk) }
func BenchmarkList2AT600(b *testing.B) { benchmark(b, 100, 5, list2) }
func BenchmarkList10AT600(b *testing.B) { benchmark(b, 100, 5, list10) }
func BenchmarkList20AT600(b *testing.B) { benchmark(b, 100, 5, list20) }
func BenchmarkGlobWorkersD1AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD1) }
func BenchmarkGlobWorkersD2AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD2) }
func BenchmarkGlobWorkersD4AT600(b *testing.B) { benchmark(b, 100, 5, globWorkersD4) }
func BenchmarkGlobAT22000(b *testing.B) { benchmark(b, 2000, 10, glob) }
func BenchmarkWalkAT22000(b *testing.B) { benchmark(b, 2000, 10, walk) }
func BenchmarkList2AT22000(b *testing.B) { benchmark(b, 2000, 10, list2) }
func BenchmarkList10AT22000(b *testing.B) { benchmark(b, 2000, 10, list10) }
func BenchmarkList20AT22000(b *testing.B) { benchmark(b, 2000, 10, list20) }
func BenchmarkGlobWorkersD1AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD1) }
func BenchmarkGlobWorkersD2AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD2) }
func BenchmarkGlobWorkersD4AT22000(b *testing.B) { benchmark(b, 2000, 10, globWorkersD4) }
func BenchmarkGlob110000(b *testing.B) { benchmark(b, 10000, 10, glob) }
func BenchmarkWalk110000(b *testing.B) { benchmark(b, 10000, 10, walk) }
func BenchmarkList110000(b *testing.B) { benchmark(b, 10000, 10, list2) }
func BenchmarkGlobWorkers110000(b *testing.B) { benchmark(b, 10000, 10, globWorkersD2) }
func benchmark(b *testing.B, numNodes int, numRevisions int, f func(string) <-chan string) {
base := initialize(numNodes, numRevisions)
defer os.RemoveAll(base)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ch := f(base)
PurgeRevisions(ch, nil, false, false)
}
b.StopTimer()
}
func test(t *testing.T, numNodes int, numRevisions int, f func(string) <-chan string) {
base := initialize(numNodes, numRevisions)
defer os.RemoveAll(base)
ch := f(base)
_, _, revisions := PurgeRevisions(ch, nil, false, false)
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
}
func glob(base string) <-chan string {
return Glob(base + _basePath + "*/*/*/*/*")
}
func walk(base string) <-chan string {
return Walk(base + _basePath)
}
func list2(base string) <-chan string {
return List(base+_basePath, 2)
}
func list10(base string) <-chan string {
return List(base+_basePath, 10)
}
func list20(base string) <-chan string {
return List(base+_basePath, 20)
}
func globWorkersD1(base string) <-chan string {
return GlobWorkers(base+_basePath, "*", "/*/*/*/*")
}
func globWorkersD2(base string) <-chan string {
return GlobWorkers(base+_basePath, "*/*", "/*/*/*")
}
func globWorkersD4(base string) <-chan string {
return GlobWorkers(base+_basePath, "*/*/*/*", "/*")
}
func initialize(numNodes int, numRevisions int) string {
base := "test_temp_" + uuid.New().String()
if err := os.Mkdir(base, os.ModePerm); err != nil {
fmt.Println("Error creating test_temp directory", err)
os.RemoveAll(base)
os.Exit(1)
}
// create base path
if err := os.MkdirAll(base+_basePath, fs.ModePerm); err != nil {
fmt.Println("Error creating base path", err)
os.RemoveAll(base)
os.Exit(1)
}
for i := 0; i < numNodes; i++ {
path := lookup.Pathify(uuid.New().String(), 4, 2)
dir := filepath.Dir(path)
if err := os.MkdirAll(base+_basePath+dir, fs.ModePerm); err != nil {
fmt.Println("Error creating test_temp directory", err)
os.RemoveAll(base)
os.Exit(1)
}
if _, err := os.Create(base + _basePath + path); err != nil {
fmt.Println("Error creating file", err)
os.RemoveAll(base)
os.Exit(1)
}
for i := 0; i < numRevisions; i++ {
os.Create(base + _basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z")
}
}
return base
}
+68
View File
@@ -0,0 +1,68 @@
# КуСфера: Runtime
Pman is a slim utility library for supervising long-running processes. It can be embedded or used as a cli command.
When used as a CLI command it relays actions to a running runtime.
## Usage
Start a runtime
```go
package main
import "github.com/qsferae-eu/qsfera/qsfera/pkg/runtime/service"
func main() {
service.Start()
}
```
![start runtime](https://imgur.com/F67hgQk.gif)
Start sending messages
![message runtime](https://imgur.com/O71RlsJ.gif)
## Example
```go
package main
import (
"fmt"
"github.com/qsfera/server/qsfera/pkg/runtime/process"
"github.com/qsfera/server/qsfera/pkg/runtime/service"
"github.com/rs/zerolog/log"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
s := service.NewService()
var c = make(chan os.Signal, 1)
var o int
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
if err := s.Start(process.NewProcEntry("ocs", nil, "ocs"), &o); err != nil {
os.Exit(1)
}
time.AfterFunc(3*time.Second, func() {
var acc = "ocs"
fmt.Printf(fmt.Sprintf("shutting down service: %s", acc))
if err := s.Controller.Kill(&acc); err != nil {
log.Fatal()
}
os.Exit(0)
})
for {
select {
case <-c:
return
}
}
}
```
Run the example above with `RUNTIME_KEEP_ALIVE=true` and with no `RUNTIME_KEEP_ALIVE` set to see its behavior. It requires an [КуСфера binary](https://github.com/qsfera/server/releases) present in your `$PATH` for it to work.
+34
View File
@@ -0,0 +1,34 @@
package cmd
import (
"fmt"
"log"
"net"
"net/rpc"
"github.com/qsfera/server/qsfera/pkg/runtime/config"
"github.com/spf13/cobra"
)
// List running service.
func List(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "list",
Aliases: []string{"r"},
Short: "List running services",
Run: func(cmd *cobra.Command, args []string) {
client, err := rpc.DialHTTP("tcp", net.JoinHostPort(cfg.Hostname, cfg.Port))
if err != nil {
log.Fatal("dialing:", err)
}
var arg1 string
if err := client.Call("Service.List", struct{}{}, &arg1); err != nil {
log.Fatal(err)
}
fmt.Println(arg1)
},
}
}
@@ -0,0 +1,28 @@
package config
// Config determines behavior across the tool.
type Config struct {
// Hostname where the runtime is running. When using PMAN in cli mode, it determines where the host runtime is.
// Default is localhost.
Hostname string
// Port configures the port where a runtime is available. It defaults to 10666.
Port string
// KeepAlive configures if restart attempts are made if the process supervised terminates. Default is false.
KeepAlive bool
}
var (
defaultHostname = "localhost"
defaultPort = "10666"
)
// NewConfig returns a new config with a set of defaults.
func NewConfig() *Config {
return &Config{
Hostname: defaultHostname,
Port: defaultPort,
KeepAlive: false,
}
}
+21
View File
@@ -0,0 +1,21 @@
package runtime
import (
"github.com/qsfera/server/pkg/log"
)
// Options is a runtime option
type Options struct {
Services []string
Logger log.Logger
}
// Option undocumented
type Option func(o *Options)
// Services option
func Services(s []string) Option {
return func(o *Options) {
o.Services = append(o.Services, s...)
}
}
+25
View File
@@ -0,0 +1,25 @@
package runtime
import (
"context"
"github.com/qsfera/server/qsfera/pkg/runtime/service"
"github.com/qsfera/server/pkg/config"
)
// Runtime represents a КуСфера runtime environment.
type Runtime struct {
c *config.Config
}
// New creates a new КуСфера + micro runtime
func New(cfg *config.Config) Runtime {
return Runtime{
c: cfg,
}
}
// Start rpc runtime
func (r *Runtime) Start(ctx context.Context) error {
return service.Start(ctx, service.WithConfig(r.c))
}
@@ -0,0 +1,40 @@
package service
import (
"github.com/qsfera/server/pkg/config"
)
// Log configures a structure logger.
type Log struct {
Pretty bool
}
// Options are the configurable options for a Service.
type Options struct {
Log *Log
Config *config.Config
}
// Option represents an option.
type Option func(o *Options)
// NewOptions returns a new Options struct.
func NewOptions() *Options {
return &Options{
Log: &Log{},
}
}
// WithLogPretty sets Controller config.
func WithLogPretty(pretty bool) Option {
return func(o *Options) {
o.Log.Pretty = pretty
}
}
// WithConfig sets Controller config.
func WithConfig(cfg *config.Config) Option {
return func(o *Options) {
o.Config = cfg
}
}
@@ -0,0 +1,629 @@
package service
import (
"context"
"errors"
"net"
"net/http"
"net/rpc"
"sort"
"strings"
"sync"
"time"
"github.com/cenkalti/backoff"
"github.com/mohae/deepcopy"
"github.com/olekukonko/tablewriter"
occfg "github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/pkg/runner"
ogrpc "github.com/qsfera/server/pkg/service/grpc"
"github.com/qsfera/server/pkg/shared"
activitylog "github.com/qsfera/server/services/activitylog/pkg/command"
antivirus "github.com/qsfera/server/services/antivirus/pkg/command"
appProvider "github.com/qsfera/server/services/app-provider/pkg/command"
appRegistry "github.com/qsfera/server/services/app-registry/pkg/command"
audit "github.com/qsfera/server/services/audit/pkg/command"
authapp "github.com/qsfera/server/services/auth-app/pkg/command"
authbasic "github.com/qsfera/server/services/auth-basic/pkg/command"
authmachine "github.com/qsfera/server/services/auth-machine/pkg/command"
authservice "github.com/qsfera/server/services/auth-service/pkg/command"
clientlog "github.com/qsfera/server/services/clientlog/pkg/command"
collaboration "github.com/qsfera/server/services/collaboration/pkg/command"
eventhistory "github.com/qsfera/server/services/eventhistory/pkg/command"
frontend "github.com/qsfera/server/services/frontend/pkg/command"
gateway "github.com/qsfera/server/services/gateway/pkg/command"
graph "github.com/qsfera/server/services/graph/pkg/command"
groups "github.com/qsfera/server/services/groups/pkg/command"
idm "github.com/qsfera/server/services/idm/pkg/command"
idp "github.com/qsfera/server/services/idp/pkg/command"
invitations "github.com/qsfera/server/services/invitations/pkg/command"
nats "github.com/qsfera/server/services/nats/pkg/command"
notifications "github.com/qsfera/server/services/notifications/pkg/command"
ocm "github.com/qsfera/server/services/ocm/pkg/command"
ocs "github.com/qsfera/server/services/ocs/pkg/command"
policies "github.com/qsfera/server/services/policies/pkg/command"
postprocessing "github.com/qsfera/server/services/postprocessing/pkg/command"
proxy "github.com/qsfera/server/services/proxy/pkg/command"
search "github.com/qsfera/server/services/search/pkg/command"
settings "github.com/qsfera/server/services/settings/pkg/command"
sharing "github.com/qsfera/server/services/sharing/pkg/command"
sse "github.com/qsfera/server/services/sse/pkg/command"
storagepublic "github.com/qsfera/server/services/storage-publiclink/pkg/command"
storageshares "github.com/qsfera/server/services/storage-shares/pkg/command"
storageSystem "github.com/qsfera/server/services/storage-system/pkg/command"
storageusers "github.com/qsfera/server/services/storage-users/pkg/command"
thumbnails "github.com/qsfera/server/services/thumbnails/pkg/command"
userlog "github.com/qsfera/server/services/userlog/pkg/command"
users "github.com/qsfera/server/services/users/pkg/command"
web "github.com/qsfera/server/services/web/pkg/command"
webdav "github.com/qsfera/server/services/webdav/pkg/command"
webfinger "github.com/qsfera/server/services/webfinger/pkg/command"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/logger"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/thejerf/suture/v4"
)
var (
// runset keeps track of which services to start supervised.
runset map[string]struct{}
// wait funcs run after the service group has been started.
_waitFuncs = []func(*occfg.Config) error{pingNats, pingGateway, nil, wait(time.Second), nil}
// Use the runner.DefaultInterruptDuration as defaults for the individual service shutdown timeouts.
_defaultShutdownTimeoutDuration = runner.DefaultInterruptDuration
// Use the runner.DefaultGroupInterruptDuration as defaults for the server interruption timeout.
_defaultInterruptTimeoutDuration = runner.DefaultGroupInterruptDuration
)
type serviceFuncMap map[string]func(*occfg.Config) suture.Service
// Service represents a RPC service.
type Service struct {
Supervisor *suture.Supervisor
Services []serviceFuncMap
Additional serviceFuncMap
Log log.Logger
serviceToken map[string][]suture.ServiceToken
cfg *occfg.Config
}
// NewService returns a configured service with a controller and a default logger.
// When used as a library, flags are not parsed, and in order to avoid introducing a global state with init functions
// calls are done explicitly to loadFromEnv().
// Since this is the public constructor, options need to be added, at the moment only logging options
// are supported in order to match the running КуСфера services structured log.
func NewService(ctx context.Context, options ...Option) (*Service, error) {
opts := NewOptions()
for _, f := range options {
f(opts)
}
l := log.NewLogger(
log.Color(opts.Config.Log.Color),
log.Pretty(opts.Config.Log.Pretty),
log.Level(opts.Config.Log.Level),
)
s := &Service{
Services: make([]serviceFuncMap, len(_waitFuncs)),
Additional: make(serviceFuncMap),
Log: l,
serviceToken: make(map[string][]suture.ServiceToken),
cfg: opts.Config,
}
// populate services
reg := func(priority int, name string, exec func(context.Context, *occfg.Config) error) {
if s.Services[priority] == nil {
s.Services[priority] = make(serviceFuncMap)
}
s.Services[priority][name] = NewSutureServiceBuilder(name, exec)
}
// nats is in priority group 0. It needs to start before all other services
reg(0, opts.Config.Nats.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Nats.Context = ctx
cfg.Nats.Commons = cfg.Commons
return nats.Execute(cfg.Nats)
})
// gateway is in priority group 1. It needs to start before the reva services
reg(1, opts.Config.Gateway.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Gateway.Context = ctx
cfg.Gateway.Commons = cfg.Commons
return gateway.Execute(cfg.Gateway)
})
// priority group 2 is empty for now
// most services are in priority group 3
reg(3, opts.Config.Activitylog.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Activitylog.Context = ctx
cfg.Activitylog.Commons = cfg.Commons
return activitylog.Execute(cfg.Activitylog)
})
reg(3, opts.Config.AppProvider.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AppProvider.Context = ctx
cfg.AppProvider.Commons = cfg.Commons
return appProvider.Execute(cfg.AppProvider)
})
reg(3, opts.Config.AppRegistry.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AppRegistry.Context = ctx
cfg.AppRegistry.Commons = cfg.Commons
return appRegistry.Execute(cfg.AppRegistry)
})
reg(3, opts.Config.AuthApp.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AuthApp.Context = ctx
cfg.AuthApp.Commons = cfg.Commons
return authapp.Execute(cfg.AuthApp)
})
reg(3, opts.Config.AuthBasic.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AuthBasic.Context = ctx
cfg.AuthBasic.Commons = cfg.Commons
return authbasic.Execute(cfg.AuthBasic)
})
reg(3, opts.Config.AuthMachine.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AuthMachine.Context = ctx
cfg.AuthMachine.Commons = cfg.Commons
return authmachine.Execute(cfg.AuthMachine)
})
reg(3, opts.Config.AuthService.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.AuthService.Context = ctx
cfg.AuthService.Commons = cfg.Commons
return authservice.Execute(cfg.AuthService)
})
reg(3, opts.Config.Clientlog.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Clientlog.Context = ctx
cfg.Clientlog.Commons = cfg.Commons
return clientlog.Execute(cfg.Clientlog)
})
reg(3, opts.Config.EventHistory.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.EventHistory.Context = ctx
cfg.EventHistory.Commons = cfg.Commons
return eventhistory.Execute(cfg.EventHistory)
})
reg(3, opts.Config.Graph.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Graph.Context = ctx
cfg.Graph.Commons = cfg.Commons
return graph.Execute(cfg.Graph)
})
reg(3, opts.Config.Groups.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Groups.Context = ctx
cfg.Groups.Commons = cfg.Commons
return groups.Execute(cfg.Groups)
})
reg(3, opts.Config.IDM.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.IDM.Context = ctx
cfg.IDM.Commons = cfg.Commons
return idm.Execute(cfg.IDM)
})
reg(3, opts.Config.OCS.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.OCS.Context = ctx
cfg.OCS.Commons = cfg.Commons
return ocs.Execute(cfg.OCS)
})
reg(3, opts.Config.Postprocessing.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Postprocessing.Context = ctx
cfg.Postprocessing.Commons = cfg.Commons
return postprocessing.Execute(cfg.Postprocessing)
})
reg(3, opts.Config.Search.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Search.Context = ctx
cfg.Search.Commons = cfg.Commons
return search.Execute(cfg.Search)
})
reg(3, opts.Config.Settings.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Settings.Context = ctx
cfg.Settings.Commons = cfg.Commons
return settings.Execute(cfg.Settings)
})
reg(3, opts.Config.StoragePublicLink.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.StoragePublicLink.Context = ctx
cfg.StoragePublicLink.Commons = cfg.Commons
return storagepublic.Execute(cfg.StoragePublicLink)
})
reg(3, opts.Config.StorageShares.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.StorageShares.Context = ctx
cfg.StorageShares.Commons = cfg.Commons
return storageshares.Execute(cfg.StorageShares)
})
reg(3, opts.Config.StorageSystem.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.StorageSystem.Context = ctx
cfg.StorageSystem.Commons = cfg.Commons
return storageSystem.Execute(cfg.StorageSystem)
})
reg(3, opts.Config.StorageUsers.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.StorageUsers.Context = ctx
cfg.StorageUsers.Commons = cfg.Commons
return storageusers.Execute(cfg.StorageUsers)
})
reg(3, opts.Config.Thumbnails.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Thumbnails.Context = ctx
cfg.Thumbnails.Commons = cfg.Commons
return thumbnails.Execute(cfg.Thumbnails)
})
reg(3, opts.Config.Userlog.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Userlog.Context = ctx
cfg.Userlog.Commons = cfg.Commons
return userlog.Execute(cfg.Userlog)
})
reg(3, opts.Config.Users.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Users.Context = ctx
cfg.Users.Commons = cfg.Commons
return users.Execute(cfg.Users)
})
reg(3, opts.Config.Web.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Web.Context = ctx
cfg.Web.Commons = cfg.Commons
return web.Execute(cfg.Web)
})
reg(3, opts.Config.WebDAV.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.WebDAV.Context = ctx
cfg.WebDAV.Commons = cfg.Commons
return webdav.Execute(cfg.WebDAV)
})
reg(3, opts.Config.Webfinger.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Webfinger.Context = ctx
cfg.Webfinger.Commons = cfg.Commons
return webfinger.Execute(cfg.Webfinger)
})
reg(3, opts.Config.IDP.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.IDP.Context = ctx
cfg.IDP.Commons = cfg.Commons
return idp.Execute(cfg.IDP)
})
reg(3, opts.Config.Proxy.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Proxy.Context = ctx
cfg.Proxy.Commons = cfg.Commons
return proxy.Execute(cfg.Proxy)
})
reg(3, opts.Config.Sharing.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Sharing.Context = ctx
cfg.Sharing.Commons = cfg.Commons
return sharing.Execute(cfg.Sharing)
})
reg(3, opts.Config.SSE.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.SSE.Context = ctx
cfg.SSE.Commons = cfg.Commons
return sse.Execute(cfg.SSE)
})
reg(3, opts.Config.OCM.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.OCM.Context = ctx
cfg.OCM.Commons = cfg.Commons
return ocm.Execute(cfg.OCM)
})
// out of some unknown reason ci gets angry when frontend service starts in priority group 3
// this is not reproducible locally, it can start when nats and gateway are already running
// FIXME: find out why
reg(4, opts.Config.Frontend.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Frontend.Context = ctx
cfg.Frontend.Commons = cfg.Commons
return frontend.Execute(cfg.Frontend)
})
// populate optional services
areg := func(name string, exec func(context.Context, *occfg.Config) error) {
s.Additional[name] = NewSutureServiceBuilder(name, exec)
}
areg(opts.Config.Antivirus.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Antivirus.Context = ctx
cfg.Antivirus.Commons = cfg.Commons
return antivirus.Execute(cfg.Antivirus)
})
areg(opts.Config.Audit.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Audit.Context = ctx
cfg.Audit.Commons = cfg.Commons
return audit.Execute(cfg.Audit)
})
areg(opts.Config.Collaboration.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Collaboration.Context = ctx
cfg.Collaboration.Commons = cfg.Commons
return collaboration.Execute(cfg.Collaboration)
})
areg(opts.Config.Policies.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Policies.Context = ctx
cfg.Policies.Commons = cfg.Commons
return policies.Execute(cfg.Policies)
})
areg(opts.Config.Invitations.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Invitations.Context = ctx
cfg.Invitations.Commons = cfg.Commons
return invitations.Execute(cfg.Invitations)
})
areg(opts.Config.Notifications.Service.Name, func(ctx context.Context, cfg *occfg.Config) error {
cfg.Notifications.Context = ctx
cfg.Notifications.Commons = cfg.Commons
return notifications.Execute(cfg.Notifications)
})
return s, nil
}
// Start a rpc service. By default, the package scope Start will run all default services to provide with a working
// КуСфера instance.
func Start(ctx context.Context, o ...Option) error {
// Start the runtime. Most likely this was called ONLY by the `qsfera server` subcommand, but since we cannot protect
// from the caller, the previous statement holds truth.
// prepare a new rpc Service struct.
s, err := NewService(ctx, o...)
if err != nil {
return err
}
// create a context that will be cancelled when too many backoff cycles on one of the services happens
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// tolerance controls backoff cycles from the supervisor.
tolerance := 5
totalBackoff := 0
// Start creates its own supervisor. Running services under `qsfera server` will create its own supervision tree.
s.Supervisor = suture.New("qsfera", suture.Spec{
EventHook: func(e suture.Event) {
if e.Type() == suture.EventTypeBackoff {
totalBackoff++
if totalBackoff == tolerance {
cancel()
}
}
switch ev := e.(type) {
case suture.EventServicePanic:
l := s.Log.Fatal()
if ev.Restarting {
l = s.Log.Error()
}
l.Str("event", e.String()).Str("service", ev.ServiceName).Str("supervisor", ev.SupervisorName).
Bool("restarting", ev.Restarting).Float64("failures", ev.CurrentFailures).Float64("threshold", ev.FailureThreshold).
Str("message", ev.PanicMsg).Msg("service panic")
case suture.EventServiceTerminate:
l := s.Log.Fatal()
if ev.Restarting {
l = s.Log.Error()
}
l.Str("event", e.String()).Str("service", ev.ServiceName).Str("supervisor", ev.SupervisorName).
Bool("restarting", ev.Restarting).Float64("failures", ev.CurrentFailures).Float64("threshold", ev.FailureThreshold).
Interface("error", ev.Err).Msg("service terminated")
case suture.EventBackoff:
s.Log.Warn().Str("event", e.String()).Str("supervisor", ev.SupervisorName).Msg("service backoff")
case suture.EventResume:
s.Log.Info().Str("event", e.String()).Str("supervisor", ev.SupervisorName).Msg("service resume")
case suture.EventStopTimeout:
s.Log.Warn().Str("event", e.String()).Str("service", ev.ServiceName).Str("supervisor", ev.SupervisorName).Msg("service resume")
default:
s.Log.Warn().Str("event", e.String()).Msgf("supervisor: %v", e.Map()["supervisor_name"])
}
},
FailureThreshold: 5,
FailureBackoff: 3 * time.Second,
})
if s.cfg.Commons == nil {
s.cfg.Commons = &shared.Commons{
Log: &shared.Log{},
}
}
if err = rpc.Register(s); err != nil {
if s != nil {
s.Log.Fatal().Err(err).Msg("could not register rpc service")
}
}
rpc.HandleHTTP()
l, err := net.Listen("tcp", net.JoinHostPort(s.cfg.Runtime.Host, s.cfg.Runtime.Port))
if err != nil {
s.Log.Fatal().Err(err).Msg("could not start listener")
}
srv := new(http.Server)
// prepare the set of services to run
s.generateRunSet(s.cfg)
// We need to control the order in which services are started and shut down,
// so we need a backgroud context that will outlive the service execution.
go s.Supervisor.ServeBackground(context.Background())
for i, service := range s.Services {
scheduleServiceTokens(s, service)
if _waitFuncs[i] != nil {
if err := _waitFuncs[i](s.cfg); err != nil {
s.Log.Fatal().Err(err).Msg("wait func failed")
}
}
}
// schedule services that are optional
scheduleServiceTokens(s, s.Additional)
go func() {
if err = srv.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) {
s.Log.Fatal().Err(err).Msg("could not start rpc server")
}
}()
// trapShutdownCtx will block on the context-done channel for interruptions.
return trapShutdownCtx(s, srv, ctx)
}
// scheduleServiceTokens adds service tokens to the service supervisor.
func scheduleServiceTokens(s *Service, funcSet serviceFuncMap) {
for name := range runset {
if _, ok := funcSet[name]; !ok {
continue
}
swap := deepcopy.Copy(s.cfg)
s.serviceToken[name] = append(s.serviceToken[name], s.Supervisor.Add(funcSet[name](swap.(*occfg.Config))))
}
}
// generateRunSet interprets the cfg.Runtime.Services config option to cherry-pick which services to start using
// the runtime.
func (s *Service) generateRunSet(cfg *occfg.Config) {
runset = make(map[string]struct{})
if cfg.Runtime.Services != nil {
for _, name := range cfg.Runtime.Services {
runset[name] = struct{}{}
}
return
}
for _, service := range s.Services {
for name := range service {
runset[name] = struct{}{}
}
}
// add additional services if explicitly added by config
for _, name := range cfg.Runtime.Additional {
runset[name] = struct{}{}
}
// remove services if explicitly excluded by config
for _, name := range cfg.Runtime.Disabled {
delete(runset, name)
}
}
// List running processes for the Service Controller.
func (s *Service) List(_ struct{}, reply *string) error {
tableString := &strings.Builder{}
table := tablewriter.NewTable(tableString)
table.Header([]string{"Service"})
names := []string{}
for t := range s.serviceToken {
if len(s.serviceToken[t]) > 0 {
names = append(names, t)
}
}
sort.Strings(names)
for n := range names {
table.Append([]string{names[n]})
}
table.Render()
*reply = tableString.String()
return nil
}
func trapShutdownCtx(s *Service, srv *http.Server, ctx context.Context) error {
<-ctx.Done()
s.Log.Info().Msg("starting graceful shutdown")
start := time.Now()
wg := sync.WaitGroup{}
wg.Go(func() {
ctx, cancel := context.WithTimeout(context.Background(), _defaultShutdownTimeoutDuration)
defer cancel()
s.Log.Debug().Msg("starting runtime listener shutdown")
if err := srv.Shutdown(ctx); err != nil {
s.Log.Error().Err(err).Msg("could not shutdown runtime listener")
return
}
s.Log.Debug().Msg("runtime listener shutdown done")
})
// shutdown services in the order defined in the config
// any services not listed will be shutdown in parallel afterwards
for _, sName := range s.cfg.Runtime.ShutdownOrder {
if _, ok := s.serviceToken[sName]; !ok {
s.Log.Warn().Str("service", sName).Msg("unknown service for ordered shutdown, skipping")
continue
}
for i := range s.serviceToken[sName] {
if err := s.Supervisor.RemoveAndWait(s.serviceToken[sName][i], _defaultShutdownTimeoutDuration); err != nil && !errors.Is(err, suture.ErrSupervisorNotRunning) {
s.Log.Error().Err(err).Str("service", sName).Msg("could not shutdown service in order, skipping to next")
// continue shutting down other services
continue
}
s.Log.Debug().Str("service", sName).Msg("graceful ordered shutdown for service done")
}
delete(s.serviceToken, sName)
}
for sName := range s.serviceToken {
for i := range s.serviceToken[sName] {
wg.Add(1)
go func() {
s.Log.Debug().Str("service", sName).Msg("starting graceful shutdown for service")
defer wg.Done()
if err := s.Supervisor.RemoveAndWait(s.serviceToken[sName][i], _defaultShutdownTimeoutDuration); err != nil && !errors.Is(err, suture.ErrSupervisorNotRunning) {
s.Log.Error().Err(err).Str("service", sName).Msg("could not shutdown service")
return
}
s.Log.Debug().Str("service", sName).Msg("graceful shutdown for service done")
}()
}
}
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-time.After(_defaultInterruptTimeoutDuration):
s.Log.Error().Dur("timeoutDuration", _defaultInterruptTimeoutDuration).Msg("graceful shutdown timeout reached, terminating")
return errors.New("graceful shutdown timeout reached, terminating")
case <-done:
duration := time.Since(start)
s.Log.Info().Dur("duration", duration).Msg("graceful shutdown done")
return nil
}
}
// pingNats will attempt to connect to nats, blocking until a connection is established
func pingNats(cfg *occfg.Config) error {
// We need to get a natsconfig from somewhere. We can use any one.
evcfg := cfg.Postprocessing.Postprocessing.Events
_, err := stream.NatsFromConfig("initial", true, stream.NatsConfig{
Endpoint: evcfg.Endpoint,
Cluster: evcfg.Cluster,
EnableTLS: evcfg.EnableTLS,
TLSInsecure: evcfg.TLSInsecure,
TLSRootCACertificate: evcfg.TLSRootCACertificate,
AuthUsername: evcfg.AuthUsername,
AuthPassword: evcfg.AuthPassword,
})
return err
}
func pingGateway(cfg *occfg.Config) error {
// init grpc connection
_, err := ogrpc.NewClient()
if err != nil {
return err
}
b := backoff.NewExponentialBackOff()
o := func() error {
n := b.NextBackOff()
_, err := pool.GetGatewayServiceClient(cfg.Reva.Address)
if err != nil && n > time.Second {
logger.New().Error().Err(err).Dur("backoff", n).Msg("can't connect to gateway service, retrying")
}
return err
}
err = backoff.Retry(o, b)
return err
}
func wait(d time.Duration) func(cfg *occfg.Config) error {
return func(cfg *occfg.Config) error {
time.Sleep(d)
return nil
}
}
@@ -0,0 +1,36 @@
package service
import (
"context"
occfg "github.com/qsfera/server/pkg/config"
"github.com/thejerf/suture/v4"
)
// SutureService allows for the settings command to be embedded and supervised by a suture supervisor tree.
type SutureService struct {
exec func(ctx context.Context) error
name string
}
// NewSutureServiceBuilder creates a new suture service
func NewSutureServiceBuilder(name string, f func(context.Context, *occfg.Config) error) func(*occfg.Config) suture.Service {
return func(cfg *occfg.Config) suture.Service {
return SutureService{
exec: func(ctx context.Context) error {
return f(ctx, cfg)
},
name: name,
}
}
}
// Serve to fullfil Server interface
func (s SutureService) Serve(ctx context.Context) error {
return s.exec(ctx)
}
// String to fullfil fmt.Stringer interface, used to log the service name
func (s SutureService) String() string {
return s.name
}
+57
View File
@@ -0,0 +1,57 @@
package trash
import (
"errors"
"fmt"
"os"
"path/filepath"
)
const (
// _trashGlobPattern is the glob pattern to find all trash items
_trashGlobPattern = "spaces/*/*/trash/*/*/*/*"
)
// PurgeTrashEmptyPaths purges empty paths in the trash
func PurgeTrashEmptyPaths(p string, dryRun bool) error {
// we have all trash nodes in all spaces now
dirs, err := filepath.Glob(filepath.Join(p, _trashGlobPattern))
if err != nil {
return err
}
if len(dirs) == 0 {
return errors.New("no trash found. Double check storage path")
}
for _, d := range dirs {
if err := removeEmptyFolder(d, dryRun); err != nil {
return err
}
}
return nil
}
func removeEmptyFolder(path string, dryRun bool) error {
if dryRun {
f, err := os.ReadDir(path)
if err != nil {
return err
}
if len(f) < 1 {
fmt.Println("would remove", path)
}
return nil
}
if err := os.Remove(path); err != nil {
// we do not really care about the error here
// if the folder is not empty we will get an error,
// this is our signal to break out of the recursion
return nil
}
nd := filepath.Dir(path)
if filepath.Base(nd) == "trash" {
return nil
}
return removeEmptyFolder(nd, dryRun)
}