Initial QSfera import
This commit is contained in:
Курнат Андрей
@@ -0,0 +1,82 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/backup"
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
decomposedbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
|
||||
decomposeds3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/blobstore"
|
||||
)
|
||||
|
||||
// BackupCommand is the entrypoint for the backup command
|
||||
func BackupCommand(cfg *config.Config) *cobra.Command {
|
||||
bckCmd := &cobra.Command{
|
||||
Use: "backup",
|
||||
Short: "КуСфера backup functionality",
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
}
|
||||
bckCmd.AddCommand(ConsistencyCommand(cfg))
|
||||
return bckCmd
|
||||
}
|
||||
|
||||
// ConsistencyCommand is the entrypoint for the consistency Command
|
||||
func ConsistencyCommand(cfg *config.Config) *cobra.Command {
|
||||
consCmd := &cobra.Command{
|
||||
Use: "consistency",
|
||||
Short: "check backup consistency",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var (
|
||||
bs backup.ListBlobstore
|
||||
err error
|
||||
)
|
||||
basePath, _ := cmd.Flags().GetString("basepath")
|
||||
blobstoreFlag, _ := cmd.Flags().GetString("blobstore")
|
||||
switch blobstoreFlag {
|
||||
case "decomposeds3":
|
||||
bs, err = decomposeds3bs.New(
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Region,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
|
||||
decomposeds3bs.Options{},
|
||||
)
|
||||
case "decomposed":
|
||||
bs, err = decomposedbs.New(basePath)
|
||||
case "none":
|
||||
bs = nil
|
||||
default:
|
||||
err = errors.New("blobstore type not supported")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
fail, _ := cmd.Flags().GetBool("fail")
|
||||
if err := backup.CheckProviderConsistency(basePath, bs, fail); err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
consCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/users)")
|
||||
_ = consCmd.MarkFlagRequired("basepath")
|
||||
consCmd.Flags().StringP("blobstore", "b", "decomposed", "the blobstore type. Can be (none, decomposed, decomposeds3). Default decomposed")
|
||||
consCmd.Flags().Bool("fail", false, "exit with non-zero status if consistency check fails")
|
||||
return consCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(BackupCommand)
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/olekukonko/tablewriter/tw"
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// BenchmarkCommand is the entrypoint for the benchmark commands.
|
||||
func BenchmarkCommand(cfg *config.Config) *cobra.Command {
|
||||
benchCmd := &cobra.Command{
|
||||
Use: "benchmark",
|
||||
Short: "cli tools to test low and high level performance",
|
||||
}
|
||||
benchCmd.AddCommand(BenchmarkClientCommand(cfg), BenchmarkSyscallsCommand(cfg))
|
||||
return benchCmd
|
||||
}
|
||||
|
||||
// BenchmarkClientCommand is the entrypoint for the benchmark client command.
|
||||
func BenchmarkClientCommand(cfg *config.Config) *cobra.Command {
|
||||
benchClientCmd := &cobra.Command{
|
||||
Use: "client",
|
||||
Short: "Start a client that continuously makes web requests and prints stats. The options mimic curl, but we default to PROPFIND requests.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
jobs, err := cmd.Flags().GetInt("jobs")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
insecure, _ := cmd.Flags().GetBool("insecure")
|
||||
opt := clientOptions{
|
||||
url: args[0],
|
||||
insecure: insecure,
|
||||
jobs: jobs,
|
||||
headers: make(map[string]string),
|
||||
}
|
||||
|
||||
if d, _ := cmd.Flags().GetString("data-raw"); d != "" {
|
||||
opt.request = "POST"
|
||||
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
opt.data = []byte(d)
|
||||
}
|
||||
|
||||
if d, _ := cmd.Flags().GetString("data"); d != "" {
|
||||
opt.request = "POST"
|
||||
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
if strings.HasPrefix(d, "@") {
|
||||
filePath := strings.TrimPrefix(d, "@")
|
||||
var data []byte
|
||||
var err error
|
||||
|
||||
// read from file or stdin and trim trailing newlines
|
||||
if filePath == "-" {
|
||||
data, err = os.ReadFile("/dev/stdin")
|
||||
} else {
|
||||
data, err = os.ReadFile(filePath)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(errors.New("could not read data from file '" + filePath + "': " + err.Error()))
|
||||
}
|
||||
|
||||
// clean byte array similar to curl's --data parameter
|
||||
// It removes leading/trailing whitespace and converts line breaks to spaces
|
||||
|
||||
// Trim leading and trailing whitespace
|
||||
data = bytes.TrimSpace(data)
|
||||
|
||||
// Replace newlines and carriage returns with spaces
|
||||
data = bytes.ReplaceAll(data, []byte("\r\n"), []byte(" "))
|
||||
data = bytes.ReplaceAll(data, []byte("\n"), []byte(" "))
|
||||
data = bytes.ReplaceAll(data, []byte("\r"), []byte(" "))
|
||||
|
||||
// Replace multiple spaces with single space
|
||||
for bytes.Contains(data, []byte(" ")) {
|
||||
data = bytes.ReplaceAll(data, []byte(" "), []byte(" "))
|
||||
}
|
||||
|
||||
opt.data = data
|
||||
} else {
|
||||
opt.data = []byte(d)
|
||||
}
|
||||
}
|
||||
|
||||
if d, _ := cmd.Flags().GetString("data-binary"); d != "" {
|
||||
opt.request = "POST"
|
||||
opt.headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
if strings.HasPrefix(d, "@") {
|
||||
filePath := strings.TrimPrefix(d, "@")
|
||||
var data []byte
|
||||
var err error
|
||||
if filePath == "-" {
|
||||
data, err = os.ReadFile("/dev/stdin")
|
||||
} else {
|
||||
data, err = os.ReadFile(filePath)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(errors.New("could not read data from file '" + filePath + "': " + err.Error()))
|
||||
}
|
||||
opt.data = data
|
||||
} else {
|
||||
opt.data = []byte(d)
|
||||
}
|
||||
}
|
||||
|
||||
// override method if specified
|
||||
if request, _ := cmd.Flags().GetString("request"); request != "" {
|
||||
opt.request = request
|
||||
}
|
||||
|
||||
if opt.url == "" {
|
||||
log.Fatal(errors.New("no URL specified"))
|
||||
}
|
||||
|
||||
headersSlice, err := cmd.Flags().GetStringSlice("header")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, h := range headersSlice {
|
||||
parts := strings.SplitN(h, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
log.Fatal(errors.New("invalid header '" + h + "'"))
|
||||
}
|
||||
opt.headers[parts[0]] = strings.TrimSpace(parts[1])
|
||||
}
|
||||
|
||||
rate, _ := cmd.Flags().GetString("rate")
|
||||
if rate != "" {
|
||||
parts := strings.SplitN(rate, "/", 2)
|
||||
num, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
unit := time.Hour // default
|
||||
if len(parts) == 2 {
|
||||
switch parts[1] {
|
||||
case "s":
|
||||
unit = time.Second
|
||||
case "m":
|
||||
unit = time.Minute
|
||||
case "d":
|
||||
unit = time.Hour * 24
|
||||
default:
|
||||
log.Fatal(errors.New("unsupported rate unit. Use s, m, h or d"))
|
||||
}
|
||||
}
|
||||
opt.rateDelay = unit / time.Duration(num)
|
||||
}
|
||||
|
||||
user, _ := cmd.Flags().GetString("user")
|
||||
opt.auth = func() string {
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(user))
|
||||
}
|
||||
|
||||
btc, _ := cmd.Flags().GetString("bearer-token-command")
|
||||
if btc != "" {
|
||||
parts := strings.SplitN(btc, " ", 2)
|
||||
var cmd *exec.Cmd
|
||||
opt.auth = func() string {
|
||||
if len(parts) > 1 {
|
||||
cmd = exec.Command(parts[0], parts[1])
|
||||
} else {
|
||||
cmd = exec.Command(parts[0])
|
||||
}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
return "Bearer " + string(output)
|
||||
}
|
||||
}
|
||||
|
||||
every, err := cmd.Flags().GetInt("every")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if every != 0 {
|
||||
opt.ticker = time.NewTicker(time.Second * time.Duration(every))
|
||||
defer opt.ticker.Stop()
|
||||
}
|
||||
|
||||
// Set up signal handling for Ctrl+C
|
||||
ctx, cancel := context.WithCancel(cmd.Context())
|
||||
defer cancel()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-sigChan
|
||||
fmt.Println("\nReceived interrupt signal, shutting down...")
|
||||
cancel()
|
||||
}()
|
||||
return client(ctx, opt)
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
// flags mimicing curl
|
||||
benchClientCmd.Flags().StringP("request", "X", "PROPFIND", "Specifies a custom request method to use when communicating with the HTTP server.")
|
||||
benchClientCmd.Flags().StringP("user", "u", "admin:admin", "Specify the user name and password to use for server authentication.")
|
||||
benchClientCmd.Flags().BoolP("insecure", "k", false, "Skip the TLS verification step and proceed without checking.")
|
||||
benchClientCmd.Flags().StringP("data", "d", "", "Sends the specified data in a POST request to the HTTP server, in the same way that a browser does when a user has filled in an HTML form and presses the submit button. If you start the data with the letter @, the rest should be a file name to read the data from, or - if you want to read the data from stdin. When -d, --data is told to read from a file like that, carriage returns and newlines are stripped out. If you do not want the @ character to have a special interpretation use --data-raw instead.")
|
||||
benchClientCmd.Flags().StringP("data-raw", "", "", "Sends the specified data in a request to the HTTP server.")
|
||||
benchClientCmd.Flags().StringP("data-binary", "", "", "This posts data exactly as specified with no extra processing whatsoever. If you start the data with the letter @, the rest should be a file name to read the data from, or - if you want to read the data from stdin.")
|
||||
benchClientCmd.Flags().StringSliceP("headers", "H", []string{}, "Extra header to include in information sent.")
|
||||
benchClientCmd.Flags().String("rate", "", "Specify the maximum transfer frequency you allow a client to use - in number of transfer starts per time unit (sometimes called request rate). The request rate is provided as \"N/U\" where N is an integer number and U is a time unit. Supported units are 's' (second), 'm' (minute), 'h' (hour) and 'd' /(day, as in a 24 hour unit). The default time unit, if no \"/U\" is provided, is number of transfers per hour.")
|
||||
|
||||
// other flags
|
||||
benchClientCmd.Flags().IntP("jobs", "j", 1, "Number of parallel clients to start. Defaults to 1.")
|
||||
benchClientCmd.Flags().Int("every", 0, "Aggregate stats every time this amount of seconds has passed.")
|
||||
benchClientCmd.Flags().String("bearer-token-command", "", "Command to execute for a bearer token, e.g. 'oidc-token qsfera'. When set, disables basic auth.")
|
||||
|
||||
return benchClientCmd
|
||||
}
|
||||
|
||||
type clientOptions struct {
|
||||
request string
|
||||
url string
|
||||
auth func() string
|
||||
insecure bool
|
||||
headers map[string]string
|
||||
rateDelay time.Duration
|
||||
data []byte
|
||||
ticker *time.Ticker
|
||||
jobs int
|
||||
}
|
||||
|
||||
func client(ctx context.Context, o clientOptions) error {
|
||||
type stat struct {
|
||||
job int
|
||||
duration time.Duration
|
||||
status int
|
||||
}
|
||||
stats := make(chan stat)
|
||||
for i := 0; i < o.jobs; i++ {
|
||||
go func(i int) {
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: o.insecure,
|
||||
},
|
||||
}
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
cookies := map[string]*http.Cookie{}
|
||||
for {
|
||||
// Check if context is cancelled
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(o.request, o.url, bytes.NewReader(o.data))
|
||||
if err != nil {
|
||||
log.Printf("client %d: could not create request: %s\n", i, err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("Authorization", strings.TrimSpace(o.auth()))
|
||||
for k, v := range o.headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
for _, cookie := range cookies {
|
||||
req.AddCookie(cookie)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
res, err := client.Do(req)
|
||||
duration := -time.Until(start)
|
||||
if err != nil {
|
||||
// Check if error is due to context cancellation
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Printf("client %d: could not create request: %s\n", i, err)
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
res.Body.Close()
|
||||
select {
|
||||
case stats <- stat{
|
||||
job: i,
|
||||
duration: duration,
|
||||
status: res.StatusCode,
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
for _, c := range res.Cookies() {
|
||||
cookies[c.Name] = c
|
||||
}
|
||||
}
|
||||
// Sleep with context awareness
|
||||
if o.rateDelay > duration {
|
||||
select {
|
||||
case <-time.After(o.rateDelay - duration):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
numRequests := 0
|
||||
if o.ticker == nil {
|
||||
// no ticker, just write every request
|
||||
for {
|
||||
select {
|
||||
case stat := <-stats:
|
||||
numRequests++
|
||||
fmt.Printf("req %d took %v and returned status %d\n", numRequests, stat.duration, stat.status)
|
||||
case <-ctx.Done():
|
||||
fmt.Println("\nShutting down...")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var duration time.Duration
|
||||
for {
|
||||
select {
|
||||
case stat := <-stats:
|
||||
numRequests++
|
||||
duration += stat.duration
|
||||
case <-o.ticker.C:
|
||||
if numRequests > 0 {
|
||||
fmt.Printf("%d req at %v/req\n", numRequests, duration/time.Duration(numRequests))
|
||||
numRequests = 0
|
||||
duration = 0
|
||||
}
|
||||
case <-ctx.Done():
|
||||
if numRequests > 0 {
|
||||
fmt.Printf("\n%d req at %v/req\n", numRequests, duration/time.Duration(numRequests))
|
||||
}
|
||||
fmt.Println("Shutting down...")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// BenchmarkSyscallsCommand is the entrypoint for the benchmark syscalls command.
|
||||
func BenchmarkSyscallsCommand(cfg *config.Config) *cobra.Command {
|
||||
benchSysCallCmd := &cobra.Command{
|
||||
Use: "syscalls",
|
||||
Short: "test the performance of syscalls",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
|
||||
path, _ := cmd.Flags().GetString("path")
|
||||
if path == "" {
|
||||
f, err := os.CreateTemp("", "qsfera-bench-temp-")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
path = f.Name()
|
||||
f.Close()
|
||||
defer os.Remove(path)
|
||||
}
|
||||
|
||||
iterations, err := cmd.Flags().GetInt("iterations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return benchmark(iterations, path)
|
||||
},
|
||||
}
|
||||
benchSysCallCmd.Flags().String("path", "", "Path to test")
|
||||
benchSysCallCmd.Flags().Int("iterations", 100, "Number of iterations to execute")
|
||||
return benchSysCallCmd
|
||||
}
|
||||
|
||||
func benchmark(iterations int, path string) error {
|
||||
tests := map[string]func() error{
|
||||
"lockedfile open(wo,c,t) close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
lockedFile, err := lockedfile.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockedFile.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"stat": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(ro) close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(wo,t) write close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
h, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.WriteString("1234567890")
|
||||
if err != nil {
|
||||
h.Close()
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"fopen(ro) read close": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
bytes := make([]byte, 0, 10)
|
||||
h, err := os.OpenFile(path, os.O_RDONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.Read(bytes)
|
||||
if err != nil {
|
||||
h.Close()
|
||||
return err
|
||||
}
|
||||
h.Close()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"xattr-set": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
err := xattr.Set(path, "user.test", []byte("123456"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"xattr-get": func() error {
|
||||
for i := 0; i < iterations; i++ {
|
||||
_, err := xattr.Get(path, "user.test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
fmt.Println("Version: " + version.GetString())
|
||||
fmt.Printf("Compiled: %s\n", version.Compiled())
|
||||
fmt.Printf("Path: %s\n", path)
|
||||
fmt.Printf("Iterations: %d\n", iterations)
|
||||
fmt.Println("")
|
||||
|
||||
cfg := tablewriter.Config{
|
||||
Header: tw.CellConfig{
|
||||
Formatting: tw.CellFormatting{
|
||||
AutoFormat: tw.Off,
|
||||
},
|
||||
},
|
||||
Row: tw.CellConfig{
|
||||
ColumnAligns: []tw.Align{
|
||||
tw.AlignLeft,
|
||||
tw.AlignRight,
|
||||
tw.AlignRight,
|
||||
tw.AlignRight,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfg))
|
||||
table.Header([]string{"Test", "Iterations", "dur/it", "total"})
|
||||
for _, t := range []string{"lockedfile open(wo,c,t) close", "stat", "fopen(wo,t) write close", "fopen(ro) close", "fopen(ro) read close", "xattr-set", "xattr-get"} {
|
||||
start := time.Now()
|
||||
err := tests[t]()
|
||||
end := time.Now()
|
||||
delta := end.Sub(start)
|
||||
if err != nil {
|
||||
table.Append([]string{t, fmt.Sprintf("%d", iterations), err.Error(), err.Error()})
|
||||
} else {
|
||||
table.Append([]string{t, fmt.Sprintf("%d", iterations), strconv.Itoa(int(delta.Nanoseconds())/iterations) + "ns", delta.String()})
|
||||
}
|
||||
}
|
||||
table.Render()
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(BenchmarkCommand)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package command
|
||||
|
||||
const (
|
||||
CommandGroupServer = "Server"
|
||||
CommandGroupServices = "Service"
|
||||
CommandGroupStorage = "Storage"
|
||||
)
|
||||
@@ -0,0 +1,342 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// DecomposedfsCommand is the entrypoint for the groups command.
|
||||
func DecomposedfsCommand(cfg *config.Config) *cobra.Command {
|
||||
decomposedCmd := &cobra.Command{
|
||||
Use: "decomposedfs",
|
||||
Short: `cli tools to inspect and manipulate a decomposedfs storage.`,
|
||||
GroupID: CommandGroupStorage,
|
||||
}
|
||||
decomposedCmd.AddCommand(metadataCmd(cfg), checkCmd(cfg))
|
||||
return decomposedCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(DecomposedfsCommand)
|
||||
}
|
||||
|
||||
func checkCmd(_ *config.Config) *cobra.Command {
|
||||
cCmd := &cobra.Command{
|
||||
Use: "check-treesize",
|
||||
Short: `cli tool to check the treesize metadata of a Space`,
|
||||
RunE: check,
|
||||
}
|
||||
cCmd.Flags().StringP("root", "r", "", "Path to the root directory of the decomposedfs")
|
||||
_ = cCmd.MarkFlagRequired("root")
|
||||
cCmd.Flags().StringP("node", "n", "", "Space ID of the Space to inspect")
|
||||
_ = cCmd.MarkFlagRequired("node")
|
||||
cCmd.Flags().Bool("repair", false, "Try to repair nodes with incorrect treesize metadata. IMPORTANT: Only use this while КуСфера is not running.")
|
||||
cCmd.Flags().Bool("force", false, "Do not prompt for confirmation when running in repair mode.")
|
||||
|
||||
return cCmd
|
||||
}
|
||||
|
||||
func check(cmd *cobra.Command, args []string) error {
|
||||
rootFlag, _ := cmd.Flags().GetString("root")
|
||||
repairFlag, _ := cmd.Flags().GetBool("repair")
|
||||
forceFlag, _ := cmd.Flags().GetBool("force")
|
||||
|
||||
if repairFlag && !forceFlag {
|
||||
answer := strings.ToLower(stringPrompt("IMPORTANT: Only use '--repair' when КуСфера is not running. Do you want to continue? [yes | no = default]"))
|
||||
if answer != "yes" && answer != "y" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
lu, backend := getBackend(cmd)
|
||||
o := &options.Options{
|
||||
MetadataBackend: backend.Name(),
|
||||
MaxConcurrency: 100,
|
||||
}
|
||||
bs, err := blobstore.New(rootFlag)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to init blobstore")
|
||||
return err
|
||||
}
|
||||
|
||||
tree := tree.New(lu, bs, o, permissions.Permissions{}, store.Create(), &zerolog.Logger{})
|
||||
|
||||
nId, _ := cmd.Flags().GetString("node")
|
||||
n, err := lu.NodeFromSpaceID(context.Background(), nId)
|
||||
if err != nil || !n.Exists {
|
||||
fmt.Println("Can not find node '" + nId + "'")
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Checking treesizes in space: %s (id: %s)\n", n.Name, n.ID)
|
||||
ctx := revactx.ContextSetUser(context.Background(),
|
||||
&userpb.User{
|
||||
Id: &userpb.UserId{
|
||||
OpaqueId: "00000000-0000-0000-0000-000000000000",
|
||||
},
|
||||
Username: "offline",
|
||||
})
|
||||
|
||||
treeSize, err := walkTree(ctx, tree, lu, n, repairFlag)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to walk tree of node %s: %s\n", n.ID, err)
|
||||
}
|
||||
treesizeFromMetadata, err := n.GetTreeSize(cmd.Context())
|
||||
if err != nil {
|
||||
fmt.Printf("failed to read treesize of node: %s: %s\n", n.ID, err)
|
||||
}
|
||||
if treesizeFromMetadata != treeSize {
|
||||
fmt.Printf("Tree sizes mismatch for space: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
|
||||
n.Name, n.ID, n.InternalPath(), treeSize, treesizeFromMetadata)
|
||||
if repairFlag {
|
||||
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
|
||||
n.ID, treeSize)
|
||||
n.SetTreeSize(cmd.Context(), treeSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func walkTree(ctx context.Context, tree *tree.Tree, lu *lookup.Lookup, root *node.Node, repair bool) (uint64, error) {
|
||||
if root.Type(ctx) != provider.ResourceType_RESOURCE_TYPE_CONTAINER {
|
||||
return 0, errors.New("can't travers non-container nodes")
|
||||
}
|
||||
children, err := tree.ListFolder(ctx, root)
|
||||
if err != nil {
|
||||
fmt.Println("Can not list children for space'" + root.ID + "'")
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var treesize uint64
|
||||
for _, child := range children {
|
||||
switch child.Type(ctx) {
|
||||
case provider.ResourceType_RESOURCE_TYPE_CONTAINER:
|
||||
subtreesize, err := walkTree(ctx, tree, lu, child, repair)
|
||||
if err != nil {
|
||||
fmt.Printf("error calculating tree size of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
treesizeFromMetadata, err := child.GetTreeSize(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to read tree size of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
if treesizeFromMetadata != subtreesize {
|
||||
origin, err := lu.Path(ctx, child, node.NoCheck)
|
||||
if err != nil {
|
||||
fmt.Printf("error get path: %s\n", err)
|
||||
}
|
||||
fmt.Printf("Tree sizes mismatch for node: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n",
|
||||
origin, child.ID, child.InternalPath(), subtreesize, treesizeFromMetadata)
|
||||
if repair {
|
||||
fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n",
|
||||
child.ID, subtreesize)
|
||||
child.SetTreeSize(ctx, subtreesize)
|
||||
}
|
||||
}
|
||||
treesize += subtreesize
|
||||
case provider.ResourceType_RESOURCE_TYPE_FILE:
|
||||
blobsize, err := child.GetBlobSize(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("error reading blobsize of node: %s: %s\n", child.ID, err)
|
||||
return 0, err
|
||||
}
|
||||
treesize += blobsize
|
||||
default:
|
||||
fmt.Printf("Ignoring type: %v, node: %s %s\n", child.Type(ctx), child.Name, child.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return treesize, nil
|
||||
}
|
||||
|
||||
func metadataCmd(cfg *config.Config) *cobra.Command {
|
||||
metaCmd := &cobra.Command{
|
||||
Use: "metadata",
|
||||
Short: `cli tools to inspect and manipulate node metadata`,
|
||||
}
|
||||
metaCmd.AddCommand(dumpCmd(cfg), getCmd(cfg), setCmd(cfg))
|
||||
metaCmd.Flags().StringP("root", "r", "", "Path to the root directory of the decomposedfs")
|
||||
_ = metaCmd.MarkFlagRequired("root")
|
||||
metaCmd.Flags().StringP("node", "n", "", "Path to or ID of the node to inspect")
|
||||
_ = metaCmd.MarkFlagRequired("node")
|
||||
return metaCmd
|
||||
}
|
||||
|
||||
func dumpCmd(_ *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "dump",
|
||||
Short: `print the metadata of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
lu, backend := getBackend(cmd)
|
||||
path, err := getNode(cmd, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attribs, err := backend.All(cmd.Context(), path)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading attributes")
|
||||
return err
|
||||
}
|
||||
attributeFlag, _ := cmd.Flags().GetString("attribute")
|
||||
printAttribs(attribs, attributeFlag)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getCmd(_ *config.Config) *cobra.Command {
|
||||
gCmd := &cobra.Command{
|
||||
Use: "get",
|
||||
Short: `print a specific attribute of the given node. String attributes will be enclosed in quotes. Binary attributes will be returned encoded as base64 with their value being prefixed with '0s'.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
lu, backend := getBackend(cmd)
|
||||
path, err := getNode(cmd, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attribs, err := backend.All(cmd.Context(), path)
|
||||
if err != nil {
|
||||
fmt.Println("Error reading attributes")
|
||||
return err
|
||||
}
|
||||
attributeFlag, _ := cmd.Flags().GetString("attribute")
|
||||
printAttribs(attribs, attributeFlag)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
gCmd.Flags().StringP("attribute", "a", "", "attribute to inspect, can be a glob pattern (e.g. 'user.*' will match all attributes starting with 'user.').")
|
||||
return gCmd
|
||||
}
|
||||
|
||||
func setCmd(_ *config.Config) *cobra.Command {
|
||||
sCmd := &cobra.Command{
|
||||
Use: "set",
|
||||
Short: `manipulate metadata of the given node. Binary attributes can be given hex encoded (prefix by '0x') or base64 encoded (prefix by '0s').`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
lu, backend := getBackend(cmd)
|
||||
n, err := getNode(cmd, lu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v, _ := cmd.Flags().GetString("value")
|
||||
if strings.HasPrefix(v, "0s") {
|
||||
b64, err := base64.StdEncoding.DecodeString(v[2:])
|
||||
if err == nil {
|
||||
v = string(b64)
|
||||
} else {
|
||||
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
|
||||
}
|
||||
} else if strings.HasPrefix(v, "0x") {
|
||||
h, err := hex.DecodeString(v[2:])
|
||||
if err == nil {
|
||||
v = string(h)
|
||||
} else {
|
||||
fmt.Printf("Error decoding base64 string: '%s'. Using as raw string.\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
attributeFlag, _ := cmd.Flags().GetString("attribute")
|
||||
err = backend.Set(cmd.Context(), n, attributeFlag, []byte(v))
|
||||
if err != nil {
|
||||
fmt.Println("Error setting attribute")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
sCmd.Flags().StringP("attribute", "a", "", "attribute to inspect, can be a glob pattern (e.g. 'user.*' will match all attributes starting with 'user.').")
|
||||
_ = sCmd.MarkFlagRequired("attribute")
|
||||
|
||||
sCmd.Flags().StringP("value", "v", "", "value to set")
|
||||
_ = sCmd.MarkFlagRequired("value")
|
||||
|
||||
return sCmd
|
||||
}
|
||||
|
||||
func backend(backend string) metadata.Backend {
|
||||
switch backend {
|
||||
case "xattrs":
|
||||
return metadata.NewXattrsBackend(cache.Config{})
|
||||
case "mpk":
|
||||
return metadata.NewMessagePackBackend(cache.Config{})
|
||||
}
|
||||
return metadata.NullBackend{}
|
||||
}
|
||||
|
||||
func getBackend(cmd *cobra.Command) (*lookup.Lookup, metadata.Backend) {
|
||||
rootFlag, _ := cmd.Flags().GetString("root")
|
||||
|
||||
bod := lookup.DetectBackendOnDisk(rootFlag)
|
||||
backend := backend(bod)
|
||||
lu := lookup.New(backend, &options.Options{
|
||||
Root: rootFlag,
|
||||
MetadataBackend: bod,
|
||||
}, &timemanager.Manager{})
|
||||
return lu, backend
|
||||
}
|
||||
|
||||
func getNode(cmd *cobra.Command, lu *lookup.Lookup) (*node.Node, error) {
|
||||
nodeFlag, _ := cmd.Flags().GetString("node")
|
||||
|
||||
id, err := storagespace.ParseID(nodeFlag)
|
||||
if err != nil {
|
||||
fmt.Println("Invalid node id.")
|
||||
return nil, err
|
||||
}
|
||||
return lu.NodeFromID(context.Background(), &id)
|
||||
}
|
||||
|
||||
func printAttribs(attribs map[string][]byte, onlyAttribute string) {
|
||||
if onlyAttribute != "" {
|
||||
fmt.Println(onlyAttribute + `=` + attribToString(attribs[onlyAttribute]))
|
||||
return
|
||||
}
|
||||
|
||||
names := []string{}
|
||||
for k := range attribs {
|
||||
names = append(names, k)
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
|
||||
for _, n := range names {
|
||||
fmt.Println(n + `=` + attribToString(attribs[n]))
|
||||
}
|
||||
}
|
||||
|
||||
func attribToString(attrib []byte) string {
|
||||
for i := 0; i < len(attrib); i++ {
|
||||
if attrib[i] < 32 || attrib[i] >= 127 {
|
||||
return "0s" + base64.StdEncoding.EncodeToString(attrib)
|
||||
}
|
||||
}
|
||||
return `"` + string(attrib) + `"`
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
ocinit "github.com/qsfera/server/qsfera/pkg/init"
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/defaults"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// InitCommand is the entrypoint for the init command
|
||||
func InitCommand(_ *config.Config) *cobra.Command {
|
||||
initCmd := &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "initialise a КуСфера config",
|
||||
GroupID: CommandGroupServer,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
insecureFlag := viper.GetString("insecure")
|
||||
insecure := false
|
||||
if insecureFlag == "ask" {
|
||||
answer := strings.ToLower(stringPrompt("Do you want to configure КуСфера with certificate checking disabled?\n This is not recommended for public instances! [yes | no = default]"))
|
||||
if answer == "yes" || answer == "y" {
|
||||
insecure = true
|
||||
}
|
||||
} else if insecureFlag == strings.ToLower("true") || insecureFlag == strings.ToLower("yes") || insecureFlag == strings.ToLower("y") {
|
||||
insecure = true
|
||||
}
|
||||
forceOverwriteFlag := viper.GetBool("force-overwrite")
|
||||
diffFlag, _ := cmd.Flags().GetBool("diff")
|
||||
configPathFlag := viper.GetString("config-path")
|
||||
adminPasswordFlag := viper.GetString("admin-password")
|
||||
err := ocinit.CreateConfig(insecure, forceOverwriteFlag, diffFlag, configPathFlag, adminPasswordFlag)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create config: %s", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
initCmd.Flags().String("insecure", "ask", "Allow insecure КуСфера config")
|
||||
_ = viper.BindEnv("insecure", "OC_INSECURE")
|
||||
_ = viper.BindPFlag("insecure", initCmd.Flags().Lookup("insecure"))
|
||||
|
||||
initCmd.Flags().BoolP("diff", "d", false, "Show the difference between the current config and the new one")
|
||||
|
||||
initCmd.Flags().BoolP("force-overwrite", "f", false, "Force overwrite existing config file")
|
||||
_ = viper.BindEnv("force-overwrite", "OC_FORCE_CONFIG_OVERWRITE")
|
||||
_ = viper.BindPFlag("force-overwrite", initCmd.Flags().Lookup("force-overwrite"))
|
||||
|
||||
initCmd.Flags().String("config-path", defaults.BaseConfigPath(), "Config path for the КуСфера runtime")
|
||||
_ = viper.BindEnv("config-path", "OC_CONFIG_DIR")
|
||||
_ = viper.BindEnv("config-path", "OC_BASE_DATA_PATH")
|
||||
_ = viper.BindPFlag("config-path", initCmd.Flags().Lookup("config-path"))
|
||||
|
||||
initCmd.Flags().String("admin-password", "", "Set admin password instead of using a random generated one")
|
||||
_ = viper.BindEnv("admin-password", "ADMIN_PASSWORD")
|
||||
_ = viper.BindEnv("admin-password", "IDM_ADMIN_PASSWORD")
|
||||
_ = viper.BindPFlag("admin-password", initCmd.Flags().Lookup("admin-password"))
|
||||
return initCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(InitCommand)
|
||||
}
|
||||
|
||||
func stringPrompt(label string) string {
|
||||
input := ""
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
for {
|
||||
_, _ = fmt.Fprint(os.Stderr, label+" ")
|
||||
input, _ = reader.ReadString('\n')
|
||||
if input != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/rpc"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// ListCommand is the entrypoint for the list command.
|
||||
func ListCommand(cfg *config.Config) *cobra.Command {
|
||||
listCmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "list КуСфера services running in the runtime (supervised mode)",
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
host := viper.GetString("hostname")
|
||||
port := viper.GetString("port")
|
||||
|
||||
client, err := rpc.DialHTTP("tcp", net.JoinHostPort(host, port))
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to the runtime. Has the runtime been started and did you configure the right runtime address (\"%s\")", cfg.Runtime.Host+":"+cfg.Runtime.Port)
|
||||
}
|
||||
|
||||
var arg1 string
|
||||
|
||||
if err := client.Call("Service.List", struct{}{}, &arg1); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(arg1)
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
listCmd.Flags().String("hostname", "localhost", "hostname of the runtime")
|
||||
_ = viper.BindEnv("hostname", "OC_RUNTIME_HOST")
|
||||
_ = viper.BindPFlag("hostname", listCmd.Flags().Lookup("hostname"))
|
||||
|
||||
listCmd.Flags().String("port", "9250", "port of the runtime")
|
||||
_ = viper.BindEnv("port", "OC_RUNTIME_PORT")
|
||||
_ = viper.BindPFlag("port", listCmd.Flags().Lookup("port"))
|
||||
return listCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(ListCommand)
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/theckman/yacspin"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
)
|
||||
|
||||
// Define the names of the extended attributes we are working with.
|
||||
const (
|
||||
parentIDAttrName = "user.oc.parentid"
|
||||
idAttrName = "user.oc.id"
|
||||
spaceIDAttrName = "user.oc.space.id"
|
||||
ownerIDAttrName = "user.oc.owner.id"
|
||||
)
|
||||
|
||||
var (
|
||||
spinner *yacspin.Spinner
|
||||
restartRequired = false
|
||||
)
|
||||
|
||||
// EntryInfo holds information about a directory entry.
|
||||
type EntryInfo struct {
|
||||
Path string
|
||||
ModTime time.Time
|
||||
ParentID string
|
||||
}
|
||||
|
||||
// PosixfsCommand is the entrypoint for the posixfs command.
|
||||
func PosixfsCommand(cfg *config.Config) *cobra.Command {
|
||||
posixCmd := &cobra.Command{
|
||||
Use: "posixfs",
|
||||
Short: `cli tools to inspect and manipulate a posixfs storage.`,
|
||||
GroupID: CommandGroupStorage,
|
||||
}
|
||||
|
||||
posixCmd.AddCommand(consistencyCmd(cfg))
|
||||
|
||||
return posixCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(PosixfsCommand)
|
||||
}
|
||||
|
||||
// consistencyCmd returns a command to check the consistency of the posixfs storage.
|
||||
func consistencyCmd(cfg *config.Config) *cobra.Command {
|
||||
consCmd := &cobra.Command{
|
||||
Use: "consistency",
|
||||
Short: "check the consistency of the posixfs storage",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return checkPosixfsConsistency(cmd, cfg)
|
||||
},
|
||||
}
|
||||
consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage")
|
||||
_ = consCmd.MarkFlagRequired("root")
|
||||
|
||||
return consCmd
|
||||
}
|
||||
|
||||
// checkPosixfsConsistency checks the consistency of the posixfs storage.
|
||||
func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error {
|
||||
rootPath, _ := cmd.Flags().GetString("root")
|
||||
indexesPath := filepath.Join(rootPath, "indexes")
|
||||
|
||||
_, err := os.Stat(indexesPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath)
|
||||
}
|
||||
return fmt.Errorf("error accessing '%s': %w", indexesPath, err)
|
||||
}
|
||||
|
||||
spinnerCfg := yacspin.Config{
|
||||
Frequency: 100 * time.Millisecond,
|
||||
CharSet: yacspin.CharSets[11],
|
||||
StopCharacter: "✓",
|
||||
StopColors: []string{"fgGreen"},
|
||||
StopFailCharacter: "✗",
|
||||
StopFailColors: []string{"fgRed"},
|
||||
}
|
||||
|
||||
spinner, err = yacspin.New(spinnerCfg)
|
||||
err = spinner.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating spinner: %w", err)
|
||||
}
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "users"))
|
||||
spinner.Suffix(" Personal spaces check ")
|
||||
spinner.StopMessage("completed\n")
|
||||
spinner.Stop()
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "projects"))
|
||||
spinner.Suffix(" Project spaces check ")
|
||||
spinner.StopMessage("completed")
|
||||
spinner.Stop()
|
||||
|
||||
if restartRequired {
|
||||
fmt.Println("\n\n ⚠️ Please restart your qsfera instance to apply changes.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkSpaces(basePath string) {
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath))
|
||||
spinner.StopFail()
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
if entry.IsDir() {
|
||||
fullPath := filepath.Join(basePath, entry.Name())
|
||||
checkSpace(fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkSpace(spacePath string) {
|
||||
spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath))
|
||||
|
||||
info, err := os.Stat(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
if !info.IsDir() {
|
||||
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
|
||||
return
|
||||
}
|
||||
|
||||
spaceID, err := xattr.Get(spacePath, spaceIDAttrName)
|
||||
if err != nil || len(spaceID) == 0 {
|
||||
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName)
|
||||
return
|
||||
}
|
||||
|
||||
checkSpaceID(spacePath)
|
||||
}
|
||||
|
||||
func checkSpaceID(spacePath string) {
|
||||
spinner.Message("checking space ID uniqueness")
|
||||
|
||||
entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Failed to gather attributes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
logSuccess("(empty space)")
|
||||
return
|
||||
}
|
||||
|
||||
if len(uniqueIDs) > 1 {
|
||||
spinner.Pause()
|
||||
fmt.Println("\n ⚠ Multiple space IDs found:")
|
||||
for id := range uniqueIDs {
|
||||
fmt.Printf(" - %s\n", id)
|
||||
}
|
||||
|
||||
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
|
||||
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
|
||||
|
||||
targetID := oldestEntry.ParentID
|
||||
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
|
||||
|
||||
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
input, _ := reader.ReadString('\n')
|
||||
input = strings.TrimSpace(strings.ToLower(input))
|
||||
|
||||
if input != "y" {
|
||||
spinner.Unpause()
|
||||
logFailure("Operation cancelled by user.")
|
||||
return
|
||||
}
|
||||
restartRequired = true
|
||||
|
||||
obsoleteIDs := []string{}
|
||||
for id := range uniqueIDs {
|
||||
if id != targetID {
|
||||
obsoleteIDs = append(obsoleteIDs, id)
|
||||
}
|
||||
}
|
||||
fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
|
||||
spinner.Unpause()
|
||||
} else {
|
||||
logSuccess("")
|
||||
}
|
||||
}
|
||||
|
||||
func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
|
||||
// Set all parentid attributes to the proper space ID
|
||||
err := setAllParentIDAttributes(entries, targetID)
|
||||
if err != nil {
|
||||
logFailure("an error occurred during file attribute update: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update space ID itself
|
||||
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID)
|
||||
err = xattr.Set(spacePath, idAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update the index
|
||||
err = updateOwnerIndexFile(spacePath, obsoleteIDs)
|
||||
if err != nil {
|
||||
logFailure("Could not update the owner index file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
|
||||
dirEntries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
var allEntries []EntryInfo
|
||||
uniqueIDs := make(map[string]struct{})
|
||||
var oldestEntry EntryInfo
|
||||
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
fullPath := filepath.Join(path, entry.Name())
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
parentID, err := xattr.Get(fullPath, parentIDAttrName)
|
||||
if err != nil {
|
||||
continue // Skip if attribute doesn't exist or can't be read
|
||||
}
|
||||
|
||||
entryInfo := EntryInfo{
|
||||
Path: fullPath,
|
||||
ModTime: info.ModTime(),
|
||||
ParentID: string(parentID),
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entryInfo)
|
||||
uniqueIDs[string(parentID)] = struct{}{}
|
||||
|
||||
if entryInfo.ModTime.Before(oldestTime) {
|
||||
oldestTime = entryInfo.ModTime
|
||||
oldestEntry = entryInfo
|
||||
}
|
||||
}
|
||||
|
||||
return allEntries, uniqueIDs, oldestEntry, nil
|
||||
}
|
||||
|
||||
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
|
||||
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.ParentID == targetID {
|
||||
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
|
||||
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking path '%s': %w", path, err)
|
||||
}
|
||||
|
||||
// Remove all attributes from the file.
|
||||
if err := removeAttributes(path); err != nil {
|
||||
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
|
||||
func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
|
||||
fmt.Printf(" Rewriting index file '%s'\n", basePath)
|
||||
|
||||
ownerID, err := xattr.Get(basePath, ownerIDAttrName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
|
||||
}
|
||||
|
||||
indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk")
|
||||
indexPath = filepath.Clean(indexPath)
|
||||
|
||||
// Read the MessagePack file
|
||||
fileData, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("index file does not exist, skipping update")
|
||||
}
|
||||
return fmt.Errorf("could not read index file: %w", err)
|
||||
}
|
||||
var indexMap map[string]string
|
||||
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
|
||||
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
|
||||
}
|
||||
|
||||
// Remove obsolete IDs from the map
|
||||
itemsRemoved := 0
|
||||
for _, id := range obsoleteIDs {
|
||||
if _, exists := indexMap[id]; exists {
|
||||
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
|
||||
delete(indexMap, id)
|
||||
itemsRemoved++
|
||||
} else {
|
||||
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if itemsRemoved == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write the data back to the file
|
||||
updatedData, err := msgpack.Marshal(&indexMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal updated index map: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write updated index file: %w", err)
|
||||
}
|
||||
|
||||
logSuccess("Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAttributes(path string) error {
|
||||
attrNames, err := xattr.List(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
|
||||
}
|
||||
|
||||
for _, attrName := range attrNames {
|
||||
if err := xattr.Remove(path, attrName); err != nil {
|
||||
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func logFailure(message string, args ...any) {
|
||||
spinner.StopFailMessage(fmt.Sprintf(message, args...))
|
||||
spinner.StopFail()
|
||||
spinner.Start()
|
||||
}
|
||||
|
||||
func logSuccess(message string, args ...any) {
|
||||
spinner.StopMessage(fmt.Sprintf(message, args...))
|
||||
spinner.Stop()
|
||||
spinner.Start()
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/qsfera/pkg/revisions"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
decomposedbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
|
||||
decomposeds3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposeds3/blobstore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
// _nodesGlobPattern is the glob pattern to find all nodes
|
||||
_nodesGlobPattern = "spaces/*/*/nodes/"
|
||||
)
|
||||
|
||||
// RevisionsCommand is the entrypoint for the revisions command.
|
||||
func RevisionsCommand(cfg *config.Config) *cobra.Command {
|
||||
revCmd := &cobra.Command{
|
||||
Use: "revisions",
|
||||
Short: "КуСфера revisions functionality",
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
}
|
||||
revCmd.AddCommand(PurgeRevisionsCommand(cfg))
|
||||
|
||||
return revCmd
|
||||
}
|
||||
|
||||
// PurgeRevisionsCommand allows removing all revisions from a storage provider.
|
||||
func PurgeRevisionsCommand(cfg *config.Config) *cobra.Command {
|
||||
revCmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "purge revisions",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
basePath, _ := cmd.Flags().GetString("basepath")
|
||||
|
||||
var (
|
||||
bs revisions.DelBlobstore
|
||||
err error
|
||||
)
|
||||
blobstoreFlag, _ := cmd.Flags().GetString("blobstore")
|
||||
switch blobstoreFlag {
|
||||
case "decomposeds3":
|
||||
bs, err = decomposeds3bs.New(
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Region,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
|
||||
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
|
||||
decomposeds3bs.Options{},
|
||||
)
|
||||
case "decomposed":
|
||||
bs, err = decomposedbs.New(basePath)
|
||||
case "none":
|
||||
bs = nil
|
||||
default:
|
||||
err = errors.New("blobstore type not supported")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
var rid *provider.ResourceId
|
||||
resourceIDFlag, _ := cmd.Flags().GetString("resource-id")
|
||||
resid, err := storagespace.ParseID(resourceIDFlag)
|
||||
if err == nil {
|
||||
rid = &resid
|
||||
}
|
||||
|
||||
mechanism, _ := cmd.Flags().GetString("glob-mechanism")
|
||||
if rid.GetOpaqueId() != "" {
|
||||
mechanism = "glob"
|
||||
}
|
||||
|
||||
var ch <-chan string
|
||||
switch mechanism {
|
||||
default:
|
||||
fallthrough
|
||||
case "glob":
|
||||
p := generatePath(basePath, rid)
|
||||
if rid.GetOpaqueId() == "" {
|
||||
p = filepath.Join(p, "*/*/*/*/*")
|
||||
}
|
||||
ch = revisions.Glob(p)
|
||||
case "workers":
|
||||
p := generatePath(basePath, rid)
|
||||
ch = revisions.GlobWorkers(p, "/*", "/*/*/*/*")
|
||||
case "list":
|
||||
p := filepath.Join(basePath, "spaces")
|
||||
if rid != nil {
|
||||
p = generatePath(basePath, rid)
|
||||
}
|
||||
ch = revisions.List(p, 10)
|
||||
}
|
||||
|
||||
flagDryRun, err := cmd.Flags().GetBool("dry-run")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flagVerbose, err := cmd.Flags().GetBool("verbose")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
files, blobs, revisionResults := revisions.PurgeRevisions(ch, bs, flagDryRun, flagVerbose)
|
||||
printResults(files, blobs, revisionResults, flagDryRun)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
revCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/metadata)")
|
||||
_ = revCmd.MarkFlagRequired("basepath")
|
||||
revCmd.Flags().StringP("blobstore", "b", "decomposed", "the blobstore type. Can be (none, decomposed, decomposeds3). Default decomposed")
|
||||
revCmd.Flags().Bool("dry-run", true, "do not delete anything, just print what would be deleted")
|
||||
revCmd.Flags().BoolP("verbose", "v", false, "print verbose output")
|
||||
revCmd.Flags().StringP("resource-id", "r", "", "purge all revisions of this file/space. If not set, all revisions will be purged")
|
||||
revCmd.Flags().String("glob-mechanism", "glob", "the glob mechanism to find all nodes. Can be 'glob', 'list' or 'workers'. 'glob' uses globbing with a single worker. 'workers' spawns multiple go routines, accelatering the command drastically but causing high cpu and ram usage. 'list' looks for references by listing directories with multiple workers. Default is 'glob'")
|
||||
|
||||
return revCmd
|
||||
}
|
||||
|
||||
func printResults(countFiles, countBlobs, countRevisions int, dryRun bool) {
|
||||
switch {
|
||||
case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
|
||||
fmt.Println("❎ No revisions found. Storage provider is clean.")
|
||||
case !dryRun:
|
||||
fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
default:
|
||||
fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
}
|
||||
}
|
||||
|
||||
func generatePath(basePath string, rid *provider.ResourceId) string {
|
||||
if rid == nil {
|
||||
return filepath.Join(basePath, _nodesGlobPattern)
|
||||
}
|
||||
|
||||
sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
|
||||
if sid == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
|
||||
if nid == "" {
|
||||
return filepath.Join(basePath, "spaces", sid, "nodes")
|
||||
}
|
||||
|
||||
return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*")
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(RevisionsCommand)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/clihelper"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Execute is the entry point for the qsfera command.
|
||||
func Execute() error {
|
||||
cfg := config.DefaultConfig()
|
||||
|
||||
app := clihelper.DefaultApp(&cobra.Command{
|
||||
Use: "qsfera",
|
||||
Short: "qsfera",
|
||||
})
|
||||
|
||||
for _, commandFactory := range register.Commands {
|
||||
command := commandFactory(cfg)
|
||||
|
||||
if command.GroupID != "" && !app.ContainsGroup(command.GroupID) {
|
||||
app.AddGroup(&cobra.Group{
|
||||
ID: command.GroupID,
|
||||
Title: command.GroupID,
|
||||
})
|
||||
}
|
||||
|
||||
app.AddCommand(command)
|
||||
}
|
||||
app.SetArgs(os.Args[1:])
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
return app.ExecuteContext(ctx)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/qsfera/pkg/runtime"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "server",
|
||||
Short: "start a fullstack server (runtime and all services in supervised mode)",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, false))
|
||||
},
|
||||
GroupID: CommandGroupServer,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Prefer the in-memory registry as the default when running in single-binary mode
|
||||
r := runtime.New(cfg)
|
||||
return r.Start(cmd.Context())
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(Server)
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
activitylog "github.com/qsfera/server/services/activitylog/pkg/command"
|
||||
antivirus "github.com/qsfera/server/services/antivirus/pkg/command"
|
||||
appprovider "github.com/qsfera/server/services/app-provider/pkg/command"
|
||||
appregistry "github.com/qsfera/server/services/app-registry/pkg/command"
|
||||
audit "github.com/qsfera/server/services/audit/pkg/command"
|
||||
authapp "github.com/qsfera/server/services/auth-app/pkg/command"
|
||||
authbasic "github.com/qsfera/server/services/auth-basic/pkg/command"
|
||||
authbearer "github.com/qsfera/server/services/auth-bearer/pkg/command"
|
||||
authmachine "github.com/qsfera/server/services/auth-machine/pkg/command"
|
||||
authservice "github.com/qsfera/server/services/auth-service/pkg/command"
|
||||
clientlog "github.com/qsfera/server/services/clientlog/pkg/command"
|
||||
collaboration "github.com/qsfera/server/services/collaboration/pkg/command"
|
||||
eventhistory "github.com/qsfera/server/services/eventhistory/pkg/command"
|
||||
frontend "github.com/qsfera/server/services/frontend/pkg/command"
|
||||
gateway "github.com/qsfera/server/services/gateway/pkg/command"
|
||||
graph "github.com/qsfera/server/services/graph/pkg/command"
|
||||
groups "github.com/qsfera/server/services/groups/pkg/command"
|
||||
idm "github.com/qsfera/server/services/idm/pkg/command"
|
||||
idp "github.com/qsfera/server/services/idp/pkg/command"
|
||||
invitations "github.com/qsfera/server/services/invitations/pkg/command"
|
||||
nats "github.com/qsfera/server/services/nats/pkg/command"
|
||||
notifications "github.com/qsfera/server/services/notifications/pkg/command"
|
||||
ocm "github.com/qsfera/server/services/ocm/pkg/command"
|
||||
ocs "github.com/qsfera/server/services/ocs/pkg/command"
|
||||
policies "github.com/qsfera/server/services/policies/pkg/command"
|
||||
postprocessing "github.com/qsfera/server/services/postprocessing/pkg/command"
|
||||
proxy "github.com/qsfera/server/services/proxy/pkg/command"
|
||||
search "github.com/qsfera/server/services/search/pkg/command"
|
||||
settings "github.com/qsfera/server/services/settings/pkg/command"
|
||||
sharing "github.com/qsfera/server/services/sharing/pkg/command"
|
||||
sse "github.com/qsfera/server/services/sse/pkg/command"
|
||||
storagepubliclink "github.com/qsfera/server/services/storage-publiclink/pkg/command"
|
||||
storageshares "github.com/qsfera/server/services/storage-shares/pkg/command"
|
||||
storagesystem "github.com/qsfera/server/services/storage-system/pkg/command"
|
||||
storageusers "github.com/qsfera/server/services/storage-users/pkg/command"
|
||||
thumbnails "github.com/qsfera/server/services/thumbnails/pkg/command"
|
||||
userlog "github.com/qsfera/server/services/userlog/pkg/command"
|
||||
users "github.com/qsfera/server/services/users/pkg/command"
|
||||
web "github.com/qsfera/server/services/web/pkg/command"
|
||||
webdav "github.com/qsfera/server/services/webdav/pkg/command"
|
||||
webfinger "github.com/qsfera/server/services/webfinger/pkg/command"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var serviceCommands = []register.Command{
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Activitylog.Service.Name, activitylog.GetCommands(cfg.Activitylog), func(c *config.Config) {
|
||||
cfg.Activitylog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Antivirus.Service.Name, antivirus.GetCommands(cfg.Antivirus), func(c *config.Config) {
|
||||
cfg.Antivirus.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AppProvider.Service.Name, appprovider.GetCommands(cfg.AppProvider), func(c *config.Config) {
|
||||
cfg.AppProvider.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AppRegistry.Service.Name, appregistry.GetCommands(cfg.AppRegistry), func(c *config.Config) {
|
||||
cfg.AppRegistry.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Audit.Service.Name, audit.GetCommands(cfg.Audit), func(c *config.Config) {
|
||||
cfg.Audit.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthApp.Service.Name, authapp.GetCommands(cfg.AuthApp), func(_ *config.Config) {
|
||||
cfg.AuthApp.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthBasic.Service.Name, authbasic.GetCommands(cfg.AuthBasic), func(c *config.Config) {
|
||||
cfg.AuthBasic.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthBearer.Service.Name, authbearer.GetCommands(cfg.AuthBearer), func(c *config.Config) {
|
||||
cfg.AuthBearer.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthMachine.Service.Name, authmachine.GetCommands(cfg.AuthMachine), func(c *config.Config) {
|
||||
cfg.AuthMachine.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.AuthService.Service.Name, authservice.GetCommands(cfg.AuthService), func(c *config.Config) {
|
||||
cfg.AuthService.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Clientlog.Service.Name, clientlog.GetCommands(cfg.Clientlog), func(c *config.Config) {
|
||||
cfg.Clientlog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Collaboration.Service.Name, collaboration.GetCommands(cfg.Collaboration), func(c *config.Config) {
|
||||
cfg.Collaboration.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.EventHistory.Service.Name, eventhistory.GetCommands(cfg.EventHistory), func(c *config.Config) {
|
||||
cfg.EventHistory.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Frontend.Service.Name, frontend.GetCommands(cfg.Frontend), func(c *config.Config) {
|
||||
cfg.Frontend.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Gateway.Service.Name, gateway.GetCommands(cfg.Gateway), func(c *config.Config) {
|
||||
cfg.Gateway.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Graph.Service.Name, graph.GetCommands(cfg.Graph), func(c *config.Config) {
|
||||
cfg.Graph.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Groups.Service.Name, groups.GetCommands(cfg.Groups), func(c *config.Config) {
|
||||
cfg.Groups.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.IDM.Service.Name, idm.GetCommands(cfg.IDM), func(c *config.Config) {
|
||||
cfg.IDM.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.IDP.Service.Name, idp.GetCommands(cfg.IDP), func(c *config.Config) {
|
||||
cfg.IDP.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Invitations.Service.Name, invitations.GetCommands(cfg.Invitations), func(c *config.Config) {
|
||||
cfg.Invitations.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Nats.Service.Name, nats.GetCommands(cfg.Nats), func(c *config.Config) {
|
||||
cfg.Nats.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Notifications.Service.Name, notifications.GetCommands(cfg.Notifications), func(c *config.Config) {
|
||||
cfg.Notifications.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.OCM.Service.Name, ocm.GetCommands(cfg.OCM), func(c *config.Config) {
|
||||
cfg.OCM.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.OCS.Service.Name, ocs.GetCommands(cfg.OCS), func(c *config.Config) {
|
||||
cfg.OCS.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Policies.Service.Name, policies.GetCommands(cfg.Policies), func(c *config.Config) {
|
||||
cfg.Policies.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Postprocessing.Service.Name, postprocessing.GetCommands(cfg.Postprocessing), func(c *config.Config) {
|
||||
cfg.Postprocessing.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Proxy.Service.Name, proxy.GetCommands(cfg.Proxy), func(c *config.Config) {
|
||||
cfg.Proxy.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Search.Service.Name, search.GetCommands(cfg.Search), func(c *config.Config) {
|
||||
cfg.Search.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Settings.Service.Name, settings.GetCommands(cfg.Settings), func(c *config.Config) {
|
||||
cfg.Settings.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Sharing.Service.Name, sharing.GetCommands(cfg.Sharing), func(c *config.Config) {
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.SSE.Service.Name, sse.GetCommands(cfg.SSE), func(c *config.Config) {
|
||||
cfg.SSE.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.StoragePublicLink.Service.Name, storagepubliclink.GetCommands(cfg.StoragePublicLink), func(c *config.Config) {
|
||||
cfg.StoragePublicLink.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageShares.Service.Name, storageshares.GetCommands(cfg.StorageShares), func(c *config.Config) {
|
||||
cfg.StorageShares.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageSystem.Service.Name, storagesystem.GetCommands(cfg.StorageSystem), func(c *config.Config) {
|
||||
cfg.StorageSystem.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.StorageUsers.Service.Name, storageusers.GetCommands(cfg.StorageUsers), func(c *config.Config) {
|
||||
cfg.StorageUsers.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Thumbnails.Service.Name, thumbnails.GetCommands(cfg.Thumbnails), func(c *config.Config) {
|
||||
cfg.Thumbnails.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Userlog.Service.Name, userlog.GetCommands(cfg.Userlog), func(c *config.Config) {
|
||||
cfg.Userlog.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Users.Service.Name, users.GetCommands(cfg.Users), func(c *config.Config) {
|
||||
cfg.Users.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Web.Service.Name, web.GetCommands(cfg.Web), func(c *config.Config) {
|
||||
cfg.Web.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.WebDAV.Service.Name, webdav.GetCommands(cfg.WebDAV), func(c *config.Config) {
|
||||
cfg.WebDAV.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
func(cfg *config.Config) *cobra.Command {
|
||||
return ServiceCommand(cfg, cfg.Webfinger.Service.Name, webfinger.GetCommands(cfg.Webfinger), func(c *config.Config) {
|
||||
cfg.Webfinger.Commons = cfg.Commons
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// ServiceCommand composes a cobra command from the given inputs.
|
||||
func ServiceCommand(cfg *config.Config, serviceName string, subCommands []*cobra.Command, f func(*config.Config)) *cobra.Command {
|
||||
command := &cobra.Command{
|
||||
Use: serviceName,
|
||||
Short: fmt.Sprintf("%s service commands", serviceName),
|
||||
GroupID: CommandGroupServices,
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
configlog.Error(parser.ParseConfig(cfg, true))
|
||||
f(cfg)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
command.AddCommand(subCommands...)
|
||||
|
||||
return command
|
||||
}
|
||||
|
||||
func init() {
|
||||
for _, c := range serviceCommands {
|
||||
register.AddCommand(c)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
oclog "github.com/qsfera/server/pkg/log"
|
||||
mregistry "github.com/qsfera/server/pkg/registry"
|
||||
sharing "github.com/qsfera/server/services/sharing/pkg/config"
|
||||
sharingparser "github.com/qsfera/server/services/sharing/pkg/config/parser"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// SharesCommand is the entrypoint for the groups command.
|
||||
func SharesCommand(cfg *config.Config) *cobra.Command {
|
||||
sharesCmd := &cobra.Command{
|
||||
Use: "shares",
|
||||
Short: `cli tools to manage entries in the share manager.`,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
}
|
||||
sharesCmd.AddCommand(cleanupCmd(cfg))
|
||||
|
||||
return sharesCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(SharesCommand)
|
||||
}
|
||||
|
||||
func cleanupCmd(cfg *config.Config) *cobra.Command {
|
||||
cleanCmd := &cobra.Command{
|
||||
Use: "cleanup",
|
||||
Short: `clean up stale entries in the share manager.`,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Parse base config
|
||||
if err := parser.ParseConfig(cfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse sharing config
|
||||
cfg.Sharing.Commons = cfg.Commons
|
||||
return configlog.ReturnError(sharingparser.ParseConfig(cfg.Sharing))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return cleanup(cmd, cfg)
|
||||
},
|
||||
}
|
||||
cleanCmd.Flags().String("service-account-id", "", "Name of the service account to use for the cleanup")
|
||||
_ = cleanCmd.MarkFlagRequired("service-account-id")
|
||||
_ = viper.BindEnv("service-account-id", "OC_SERVICE_ACCOUNT_ID")
|
||||
_ = viper.BindPFlag("service-account-id", cleanCmd.Flags().Lookup("service-account-id"))
|
||||
|
||||
cleanCmd.Flags().String("service-account-secret", "", "Secret for the service account")
|
||||
_ = cleanCmd.MarkFlagRequired("service-account-secret")
|
||||
_ = viper.BindEnv("service-account-secret", "OC_SERVICE_ACCOUNT_SECRET")
|
||||
_ = viper.BindPFlag("service-account-secret", cleanCmd.Flags().Lookup("service-account-secret"))
|
||||
|
||||
return cleanCmd
|
||||
}
|
||||
|
||||
func cleanup(_ *cobra.Command, cfg *config.Config) error {
|
||||
driver := cfg.Sharing.UserSharingDriver
|
||||
// cleanup is only implemented for the jsoncs3 share manager
|
||||
if driver != "jsoncs3" {
|
||||
return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager"))
|
||||
}
|
||||
|
||||
l := logger()
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
|
||||
rcfg := revaShareConfig(cfg.Sharing)
|
||||
f, ok := registry.NewFuncs[driver]
|
||||
if !ok {
|
||||
return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'"))
|
||||
}
|
||||
mgr, err := f(rcfg[driver].(map[string]any), l)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Initialize registry to make service lookup work
|
||||
_ = mregistry.GetRegistry()
|
||||
|
||||
// get an authenticated context
|
||||
gatewaySelector, err := pool.GatewaySelector(cfg.Sharing.Reva.Address)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
client, err := gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
serviceAccountIDFlag := viper.GetString("service-account-id")
|
||||
serviceAccountSecretFlag := viper.GetString("service-account-secret")
|
||||
serviceUserCtx, err := utils.GetServiceUserContext(serviceAccountIDFlag, client, serviceAccountSecretFlag)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
serviceUserCtx = l.WithContext(serviceUserCtx)
|
||||
|
||||
mgr.(*jsoncs3.Manager).CleanupStaleShares(serviceUserCtx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func revaShareConfig(cfg *sharing.Config) map[string]any {
|
||||
return map[string]any{
|
||||
"json": map[string]any{
|
||||
"file": cfg.UserSharingDrivers.JSON.File,
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
},
|
||||
"sql": map[string]any{ // cernbox sql
|
||||
"db_username": cfg.UserSharingDrivers.SQL.DBUsername,
|
||||
"db_password": cfg.UserSharingDrivers.SQL.DBPassword,
|
||||
"db_host": cfg.UserSharingDrivers.SQL.DBHost,
|
||||
"db_port": cfg.UserSharingDrivers.SQL.DBPort,
|
||||
"db_name": cfg.UserSharingDrivers.SQL.DBName,
|
||||
"password_hash_cost": cfg.UserSharingDrivers.SQL.PasswordHashCost,
|
||||
"enable_expired_shares_cleanup": cfg.UserSharingDrivers.SQL.EnableExpiredSharesCleanup,
|
||||
"janitor_run_interval": cfg.UserSharingDrivers.SQL.JanitorRunInterval,
|
||||
},
|
||||
"owncloudsql": map[string]any{
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
"storage_mount_id": cfg.UserSharingDrivers.OwnCloudSQL.UserStorageMountID,
|
||||
"db_username": cfg.UserSharingDrivers.OwnCloudSQL.DBUsername,
|
||||
"db_password": cfg.UserSharingDrivers.OwnCloudSQL.DBPassword,
|
||||
"db_host": cfg.UserSharingDrivers.OwnCloudSQL.DBHost,
|
||||
"db_port": cfg.UserSharingDrivers.OwnCloudSQL.DBPort,
|
||||
"db_name": cfg.UserSharingDrivers.OwnCloudSQL.DBName,
|
||||
},
|
||||
"cs3": map[string]any{
|
||||
"gateway_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
|
||||
"provider_addr": cfg.UserSharingDrivers.CS3.ProviderAddr,
|
||||
"service_user_id": cfg.UserSharingDrivers.CS3.SystemUserID,
|
||||
"service_user_idp": cfg.UserSharingDrivers.CS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.UserSharingDrivers.CS3.SystemUserAPIKey,
|
||||
},
|
||||
"jsoncs3": map[string]any{
|
||||
"gateway_addr": cfg.Reva.Address,
|
||||
"provider_addr": cfg.UserSharingDrivers.JSONCS3.ProviderAddr,
|
||||
"service_user_id": cfg.UserSharingDrivers.JSONCS3.SystemUserID,
|
||||
"service_user_idp": cfg.UserSharingDrivers.JSONCS3.SystemUserIDP,
|
||||
"machine_auth_apikey": cfg.UserSharingDrivers.JSONCS3.SystemUserAPIKey,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func logger() *zerolog.Logger {
|
||||
log := oclog.NewLogger(
|
||||
oclog.Name("migrate"),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
return &log
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/qsfera/pkg/trash"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/config/parser"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func TrashCommand(cfg *config.Config) *cobra.Command {
|
||||
trashCmd := &cobra.Command{
|
||||
Use: "trash",
|
||||
Short: "КуСфера trash functionality",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg, true))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Read the docs")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
trashCmd.AddCommand(TrashPurgeEmptyDirsCommand(cfg))
|
||||
|
||||
return trashCmd
|
||||
}
|
||||
|
||||
func TrashPurgeEmptyDirsCommand(cfg *config.Config) *cobra.Command {
|
||||
trashPurgeCmd := &cobra.Command{
|
||||
Use: "purge-empty-dirs",
|
||||
Short: "purge empty directories",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
basePath, _ := cmd.Flags().GetString("basepath")
|
||||
dryRun, _ := cmd.Flags().GetBool("dry-run")
|
||||
if err := trash.PurgeTrashEmptyPaths(basePath, dryRun); err != nil {
|
||||
fmt.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
trashPurgeCmd.Flags().StringP("basepath", "p", "", "the basepath of the decomposedfs (e.g. /var/tmp/qsfera/storage/users)")
|
||||
_ = trashPurgeCmd.MarkFlagRequired("basepath")
|
||||
|
||||
trashPurgeCmd.Flags().Bool("dry-run", true, "do not delete anything, just print what would be deleted")
|
||||
|
||||
return trashPurgeCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(TrashCommand)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/qsfera/server/qsfera/pkg/register"
|
||||
"github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/registry"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/olekukonko/tablewriter/tw"
|
||||
mreg "go-micro.dev/v4/registry"
|
||||
)
|
||||
|
||||
const (
|
||||
_skipServiceListingFlagName = "skip-services"
|
||||
)
|
||||
|
||||
// VersionCommand is the entrypoint for the version command.
|
||||
func VersionCommand(cfg *config.Config) *cobra.Command {
|
||||
versionCmd := &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print the version of this binary and all running service instances",
|
||||
GroupID: CommandGroupServer,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Version: " + version.GetString())
|
||||
fmt.Printf("Edition: %s\n", version.Edition)
|
||||
fmt.Printf("Compiled: %s\n", version.Compiled())
|
||||
|
||||
skipServiceListing, _ := cmd.Flags().GetBool(_skipServiceListingFlagName)
|
||||
if skipServiceListing {
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Print("\n")
|
||||
|
||||
reg := registry.GetRegistry()
|
||||
serviceList, err := reg.ListServices()
|
||||
if err != nil {
|
||||
fmt.Printf("could not list services: %v\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var services []*mreg.Service
|
||||
for _, s := range serviceList {
|
||||
s, err := reg.GetService(s.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("could not get service: %v\n", err)
|
||||
return err
|
||||
}
|
||||
services = append(services, s...)
|
||||
}
|
||||
|
||||
if len(services) == 0 {
|
||||
fmt.Println("No running services found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
|
||||
table.Header([]string{"Version", "Address", "Id"})
|
||||
for _, s := range services {
|
||||
for _, n := range s.Nodes {
|
||||
table.Append([]string{s.Version, n.Address, n.Id})
|
||||
}
|
||||
}
|
||||
table.Render()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
versionCmd.Flags().Bool(_skipServiceListingFlagName, false, "skip service listing")
|
||||
return versionCmd
|
||||
}
|
||||
|
||||
func init() {
|
||||
register.AddCommand(VersionCommand)
|
||||
}
|
||||
Reference in New Issue
Block a user