Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,54 @@
package command
import (
"fmt"
"net/http"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/qsfera/server/services/storage-users/pkg/config/parser"
"github.com/spf13/cobra"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "health",
Short: "check health status",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
resp, err := http.Get(
fmt.Sprintf(
"http://%s/healthz",
cfg.Debug.Addr,
),
)
if err != nil {
logger.Fatal().
Err(err).
Msg("Failed to request health check")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Fatal().
Int("code", resp.StatusCode).
Msg("Health seems to be in bad state")
}
logger.Debug().
Int("code", resp.StatusCode).
Msg("Health got a good state")
return nil
},
}
}
@@ -0,0 +1,38 @@
package command
import (
"os"
"github.com/qsfera/server/pkg/clihelper"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/spf13/cobra"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) []*cobra.Command {
return []*cobra.Command{
// start this service
Server(cfg),
// interaction with this service
Uploads(cfg),
TrashBin(cfg),
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the qsfera-storage-users command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cobra.Command{
Use: "storage-users",
Short: "Provide storage for users and projects in КуСфера",
})
app.AddCommand(GetCommands(cfg)...)
app.SetArgs(os.Args[1:])
return app.ExecuteContext(cfg.Context)
}
@@ -0,0 +1,122 @@
package command
import (
"context"
"fmt"
"os/signal"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/pkg/registry"
"github.com/qsfera/server/pkg/runner"
"github.com/qsfera/server/pkg/tracing"
"github.com/qsfera/server/pkg/version"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/qsfera/server/services/storage-users/pkg/config/parser"
"github.com/qsfera/server/services/storage-users/pkg/event"
"github.com/qsfera/server/services/storage-users/pkg/revaconfig"
"github.com/qsfera/server/services/storage-users/pkg/server/debug"
"github.com/opencloud-eu/reva/v2/cmd/revad/runtime"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/spf13/cobra"
)
// Server is the entry point for the server command.
func Server(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "server",
Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
if err != nil {
return err
}
var cancel context.CancelFunc
if cfg.Context == nil {
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
defer cancel()
}
ctx := cfg.Context
gr := runner.NewGroup()
{
// run the appropriate reva servers based on the config
rCfg := revaconfig.StorageUsersConfigFromStruct(cfg)
if rServer := runtime.NewDrivenHTTPServerWithOptions(rCfg,
runtime.WithLogger(&logger.Logger),
runtime.WithRegistry(registry.GetRegistry()),
runtime.WithTraceProvider(traceProvider),
); rServer != nil {
gr.Add(runner.NewRevaServiceRunner(cfg.Service.Name+".rhttp", rServer))
}
if rServer := runtime.NewDrivenGRPCServerWithOptions(rCfg,
runtime.WithLogger(&logger.Logger),
runtime.WithRegistry(registry.GetRegistry()),
runtime.WithTraceProvider(traceProvider),
); rServer != nil {
gr.Add(runner.NewRevaServiceRunner(cfg.Service.Name+".rgrpc", rServer))
}
}
{
debugServer, err := debug.Server(
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGolangHttpServerRunner("storage-users_debug", debugServer))
}
grpcSvc := registry.BuildGRPCService(cfg.GRPC.Namespace+"."+cfg.Service.Name, cfg.GRPC.Protocol, cfg.GRPC.Addr, version.GetString())
if err := registry.RegisterService(ctx, logger, grpcSvc, cfg.Debug.Addr); err != nil {
logger.Fatal().Err(err).Msg("failed to register the grpc service")
}
{
stream, err := event.NewStream(cfg)
if err != nil {
logger.Fatal().Err(err).Msg("can't connect to nats")
}
selector, err := pool.GatewaySelector(cfg.Reva.Address, pool.WithRegistry(registry.GetRegistry()), pool.WithTracerProvider(traceProvider))
if err != nil {
return err
}
eventSVC, err := event.NewService(ctx, selector, stream, logger, *cfg)
if err != nil {
logger.Fatal().Err(err).Msg("can't create event handler")
}
// The event service Run() function handles the stop signal itself
go func() {
err := eventSVC.Run()
if err != nil {
logger.Fatal().Err(err).Msg("can't run event server")
}
}()
}
grResults := gr.Run(ctx)
// return the first non-nil error found in the results
for _, grResult := range grResults {
if grResult.RunnerError != nil {
return grResult.RunnerError
}
}
return nil
},
}
}
@@ -0,0 +1,486 @@
package command
import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/qsfera/server/pkg/config/configlog"
zlog "github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/qsfera/server/services/storage-users/pkg/config/parser"
"github.com/qsfera/server/services/storage-users/pkg/event"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/mohae/deepcopy"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
"github.com/rs/zerolog"
"github.com/spf13/cobra"
)
const (
SKIP = iota
REPLACE
KEEP_BOTH
)
// TrashBin wraps trash-bin related sub-commands.
func TrashBin(cfg *config.Config) *cobra.Command {
trashBinCmd := &cobra.Command{
Use: "trash-bin",
Short: "manage trash-bin's",
}
trashBinCmd.AddCommand([]*cobra.Command{
PurgeExpiredResources(cfg),
listTrashBinItems(cfg),
restoreAllTrashBinItems(cfg),
restoreTrashBinItem(cfg),
}...)
return trashBinCmd
}
// PurgeExpiredResources cli command removes old trash-bin items.
func PurgeExpiredResources(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "purge-expired",
Short: "Purge expired trash-bin items",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
stream, err := event.NewStream(cfg)
if err != nil {
return err
}
if err := events.Publish(cmd.Context(), stream, event.PurgeTrashBin{ExecutionTime: time.Now()}); err != nil {
return err
}
// go-micro nats implementation uses async publishing,
// therefore we need to manually wait.
//
// FIXME: upstream pr
//
// https://github.com/go-micro/plugins/blob/3e77393890683be4bacfb613bc5751867d584692/v4/events/natsjs/nats.go#L115
time.Sleep(5 * time.Second)
return nil
},
}
}
func listTrashBinItems(cfg *config.Config) *cobra.Command {
listTrashBinItemsCmd := &cobra.Command{
Use: "list",
Short: "Print a list of all trash-bin items of a space.",
// TODO: n might need to equal 2 not sure.
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
verbose, _ := cmd.Flags().GetBool("verbose")
log := cliLogger(verbose)
var spaceID string
if len(args) > 0 {
spaceID = args[0]
}
if spaceID == "" {
_ = cmd.Help()
return fmt.Errorf("spaceID is requiered")
}
log.Info().Msgf("Getting trash-bin items for spaceID: '%s' ...", spaceID)
ref, err := storagespace.ParseReference(spaceID)
if err != nil {
return err
}
client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr)
if err != nil {
return fmt.Errorf("error selecting gateway client %w", err)
}
ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("could not get service user context %w", err)
}
res, err := listRecycle(ctx, client, ref)
if err != nil {
return err
}
table := itemsTable(len(res.GetRecycleItems()))
for _, item := range res.GetRecycleItems() {
table.Append([]string{item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)})
}
table.Render()
fmt.Println("Use an itemID to restore an item.")
return nil
},
}
listTrashBinItemsCmd.Flags().BoolP(
"verbose",
"v",
false,
"Get more verbose output",
)
return listTrashBinItemsCmd
}
func restoreAllTrashBinItems(cfg *config.Config) *cobra.Command {
var overwriteOption int
restoreAllTrashBinItemsCmd := &cobra.Command{
Use: "restore-all",
Short: "Restore all trash-bin items for a space.",
// TODO: not sure this could also be 2
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
verbose, _ := cmd.Flags().GetBool("verbose")
log := cliLogger(verbose)
var spaceID string
if len(args) > 0 {
spaceID = args[0]
}
if spaceID == "" {
_ = cmd.Help()
return fmt.Errorf("spaceID is requiered")
}
option, _ := cmd.Flags().GetString("option")
switch option {
case "skip":
overwriteOption = SKIP
case "replace":
overwriteOption = REPLACE
case "keep-both":
overwriteOption = KEEP_BOTH
default:
_ = cmd.Help()
return fmt.Errorf("option flag '%s' is invalid", option)
}
log.Info().Msgf("Restoring trash-bin items for spaceID: '%s' ...", spaceID)
ref, err := storagespace.ParseReference(spaceID)
if err != nil {
return err
}
client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr)
if err != nil {
return fmt.Errorf("error selecting gateway client %w", err)
}
ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("could not get service user context %w", err)
}
res, err := listRecycle(ctx, client, ref)
if err != nil {
return err
}
applyYesFlag, _ := cmd.Flags().GetBool("yes")
if !applyYesFlag {
for {
fmt.Printf("Found %d items that could be restored, continue (Y/n), show the items list (s): ", len(res.GetRecycleItems()))
var i string
_, err := fmt.Scanf("%s", &i)
if err != nil {
log.Err(err).Send()
continue
}
if strings.ToLower(i) == "y" {
break
} else if strings.ToLower(i) == "n" {
return nil
} else if strings.ToLower(i) == "s" {
table := itemsTable(len(res.GetRecycleItems()))
for _, item := range res.GetRecycleItems() {
table.Append([]string{item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)})
}
table.Render()
}
}
}
log.Info().Msgf("Run restoring-all with option=%s", option)
for _, item := range res.GetRecycleItems() {
log.Info().Msgf("restoring itemID: '%s', path: '%s', type: '%s'", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()))
dstRes, err := restore(ctx, client, ref, item, overwriteOption, cfg.CliMaxAttemptsRenameFile, log)
if err != nil {
log.Err(err).Msg("trash-bin item restoring error")
continue
}
fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", item.GetKey(), item.GetRef().GetPath(), dstRes.GetPath())
}
return nil
},
}
restoreAllTrashBinItemsCmd.Flags().BoolP(
"verbose",
"v",
false,
"Get more verbose output",
)
restoreAllTrashBinItemsCmd.Flags().StringP(
"option",
"o",
"skip",
"The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file.",
)
restoreAllTrashBinItemsCmd.Flags().BoolP(
"yes",
"y",
false,
"Automatic yes to prompts. Assume 'yes' as answer to all prompts and run non-interactively.",
)
return restoreAllTrashBinItemsCmd
}
func restoreTrashBinItem(cfg *config.Config) *cobra.Command {
var overwriteOption int
restoreTrashBinItemCmd := &cobra.Command{
Use: "restore",
Short: "Restore a trash-bin item by ID.",
Args: cobra.ExactArgs(2),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
verbose, _ := cmd.Flags().GetBool("verbose")
log := cliLogger(verbose)
var spaceID, itemID string
spaceID = args[0]
itemID = args[1]
if spaceID == "" {
_ = cmd.Help()
return fmt.Errorf("spaceID is requered")
}
if itemID == "" {
_ = cmd.Help()
return fmt.Errorf("itemID is requered")
}
option, _ := cmd.Flags().GetString("option")
switch option {
case "skip":
overwriteOption = SKIP
case "replace":
overwriteOption = REPLACE
case "keep-both":
overwriteOption = KEEP_BOTH
default:
_ = cmd.Help()
return fmt.Errorf("option flag '%s' is invalid", option)
}
log.Info().Msgf("Restoring trash-bin item for spaceID: '%s' itemID: '%s' ...", spaceID, itemID)
ref, err := storagespace.ParseReference(spaceID)
if err != nil {
return err
}
client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr)
if err != nil {
return fmt.Errorf("error selecting gateway client %w", err)
}
ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("could not get service user context %w", err)
}
res, err := listRecycle(ctx, client, ref)
if err != nil {
return err
}
var found bool
var itemRef *provider.RecycleItem
for _, item := range res.GetRecycleItems() {
if item.GetKey() == itemID {
itemRef = item
found = true
break
}
}
if !found {
return fmt.Errorf("itemID '%s' not found", itemID)
}
log.Info().Msgf("Run restoring with option=%s", option)
log.Info().Msgf("restoring itemID: '%s', path: '%s', type: '%s", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType()))
dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption, cfg.CliMaxAttemptsRenameFile, log)
if err != nil {
return err
}
fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), dstRes.GetPath())
return nil
},
}
restoreTrashBinItemCmd.Flags().BoolP(
"verbose",
"v",
false,
"Get more verbose output",
)
restoreTrashBinItemCmd.Flags().StringP(
"option",
"o",
"skip",
"The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file.",
)
return restoreTrashBinItemCmd
}
func listRecycle(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference) (*provider.ListRecycleResponse, error) {
_retrievingErrorMsg := "trash-bin items retrieving error"
res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: ""})
if err != nil {
return nil, fmt.Errorf("%s %w", _retrievingErrorMsg, err)
}
if res.Status.Code != rpc.Code_CODE_OK {
return nil, fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code)
}
if len(res.GetRecycleItems()) == 0 {
fmt.Errorf("The trash-bin is empty. Nothing to restore")
os.Exit(0)
}
return res, nil
}
func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int, maxRenameAttempt int, log zlog.Logger) (*provider.Reference, error) {
dst, _ := deepcopy.Copy(ref).(provider.Reference)
dst.Path = utils.MakeRelativePath(item.GetRef().GetPath())
// Restore request
req := &provider.RestoreRecycleItemRequest{
Ref: &ref,
Key: path.Join(item.GetKey(), "/"),
RestoreRef: &dst,
}
exists, dstStatRes, err := isDestinationExists(ctx, client, dst)
if err != nil {
return &dst, err
}
if exists {
log.Info().Msgf("destination '%s' exists.", dstStatRes.GetInfo().GetPath())
switch overwriteOption {
case SKIP:
return &dst, nil
case REPLACE:
// delete existing tree
delReq := &provider.DeleteRequest{Ref: &dst}
delRes, err := client.Delete(ctx, delReq)
if err != nil {
return &dst, fmt.Errorf("error sending grpc delete request %w", err)
}
if delRes.Status.Code != rpc.Code_CODE_OK && delRes.Status.Code != rpc.Code_CODE_NOT_FOUND {
return &dst, fmt.Errorf("deleting error %w", err)
}
case KEEP_BOTH:
// modify the file name
req.RestoreRef, err = resolveDestination(ctx, client, dst, maxRenameAttempt)
if err != nil {
return &dst, err
}
}
}
res, err := client.RestoreRecycleItem(ctx, req)
if err != nil {
return req.RestoreRef, fmt.Errorf("restoring error %w", err)
}
if res.Status.Code != rpc.Code_CODE_OK {
return req.RestoreRef, fmt.Errorf("can not restore %s", res.Status.Code)
}
return req.RestoreRef, nil
}
func resolveDestination(ctx context.Context, client gateway.GatewayAPIClient, dstRef provider.Reference, maxRenameAttempt int) (*provider.Reference, error) {
dst := dstRef
if maxRenameAttempt < 100 {
maxRenameAttempt = 100
}
for i := 1; i < maxRenameAttempt; i++ {
dst.Path = modifyFilename(dstRef.Path, i)
exists, _, err := isDestinationExists(ctx, client, dst)
if err != nil {
return nil, err
}
if exists {
continue
}
return &dst, nil
}
return nil, fmt.Errorf("too many attempts to resolve the destination")
}
func isDestinationExists(ctx context.Context, client gateway.GatewayAPIClient, dst provider.Reference) (bool, *provider.StatResponse, error) {
dstStatReq := &provider.StatRequest{Ref: &dst}
dstStatRes, err := client.Stat(ctx, dstStatReq)
if err != nil {
return false, nil, fmt.Errorf("error sending grpc stat request %w", err)
}
if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_OK {
return true, dstStatRes, nil
}
if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_NOT_FOUND {
return false, dstStatRes, nil
}
return false, dstStatRes, fmt.Errorf("stat request failed %s", dstStatRes.GetStatus())
}
// modify the file name like UI do
func modifyFilename(filename string, mod int) string {
var extension string
var found bool
expected := []string{".tar.gz", ".tar.bz", ".tar.bz2"}
for _, s := range expected {
var prefix string
prefix, found = strings.CutSuffix(strings.ToLower(filename), s)
if found {
extension = strings.TrimPrefix(filename, prefix)
break
}
}
if !found {
extension = filepath.Ext(filename)
}
name := filename[0 : len(filename)-len(extension)]
return fmt.Sprintf("%s (%d)%s", name, mod, extension)
}
func itemType(it provider.ResourceType) string {
var itemType = "file"
if it == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
itemType = "folder"
}
return itemType
}
func itemsTable(total int) *tablewriter.Table {
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
table.Header([]string{"itemID", "path", "type", "delete at"})
table.Footer([]string{"", "", "", "total count: " + strconv.Itoa(total)})
return table
}
func cliLogger(verbose bool) zlog.Logger {
logLvl := zerolog.ErrorLevel
if verbose {
logLvl = zerolog.InfoLevel
}
zerolog.SetGlobalLevel(zerolog.TraceLevel)
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339, NoColor: true}
return zlog.Logger{zerolog.New(output).With().Timestamp().Logger().Level(logLvl)}
}
@@ -0,0 +1,65 @@
package command
import (
"testing"
)
func Test_modifyFilename(t *testing.T) {
type args struct {
filename string
mod int
}
tests := []struct {
name string
args args
want string
}{
{
name: "file",
args: args{filename: "file.txt", mod: 1},
want: "file (1).txt",
},
{
name: "file with path",
args: args{filename: "./file.txt", mod: 1},
want: "./file (1).txt",
},
{
name: "file with path 2",
args: args{filename: "./subdir/file.tar.gz", mod: 99},
want: "./subdir/file (99).tar.gz",
},
{
name: "file with path 3",
args: args{filename: "./sub dir/new file.tar.gz", mod: 99},
want: "./sub dir/new file (99).tar.gz",
},
{
name: "file without ext",
args: args{filename: "./subdir/file", mod: 2},
want: "./subdir/file (2)",
},
{
name: "file without ext 2",
args: args{filename: "./subdir/file 1", mod: 2},
want: "./subdir/file 1 (2)",
},
{
name: "file with emoji",
args: args{filename: "./subdir/file 🙂.tar.gz", mod: 3},
want: "./subdir/file 🙂 (3).tar.gz",
},
{
name: "file with emoji 2",
args: args{filename: "./subdir/file 🙂", mod: 2},
want: "./subdir/file 🙂 (2)",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := modifyFilename(tt.args.filename, tt.args.mod); got != tt.want {
t.Errorf("modifyFilename() = %v, want %v", got, tt.want)
}
})
}
}
@@ -0,0 +1,306 @@
package command
import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
"github.com/spf13/cobra"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/qsfera/server/services/storage-users/pkg/config/parser"
"github.com/qsfera/server/services/storage-users/pkg/event"
"github.com/qsfera/server/services/storage-users/pkg/revaconfig"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// Session contains the information of an upload session
type Session struct {
ID string `json:"id"`
Space string `json:"space"`
Filename string `json:"filename"`
Offset int64 `json:"offset"`
Size int64 `json:"size"`
Executant userpb.UserId `json:"executant"`
SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"`
Expires time.Time `json:"expires"`
Processing bool `json:"processing"`
ScanDate time.Time `json:"virus_scan_date"`
ScanResult string `json:"virus_scan_result"`
}
// Uploads is the entry point for the uploads command
func Uploads(cfg *config.Config) *cobra.Command {
uploadsCmd := &cobra.Command{
Use: "uploads",
Short: "manage unfinished uploads",
}
uploadsCmd.AddCommand([]*cobra.Command{
ListUploadSessions(cfg),
}...)
return uploadsCmd
}
// ListUploadSessions prints a list of upload sessiens
func ListUploadSessions(cfg *config.Config) *cobra.Command {
listUploadSessionsCmd := &cobra.Command{
Use: "sessions",
Short: "Print a list of upload sessions",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
var err error
f, ok := registry.NewFuncs[cfg.Driver]
if !ok {
fmt.Fprintf(os.Stderr, "Unknown filesystem driver '%s'\n", cfg.Driver)
os.Exit(1)
}
drivers := revaconfig.StorageProviderDrivers(cfg)
var fsStream events.Stream
if cfg.Driver == "posix" {
// We need to init the posix driver with 'scanfs' disabled
drivers["posix"] = revaconfig.Posix(cfg, false, false)
// Also posix refuses to start without an events stream
fsStream, err = event.NewStream(cfg)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err)
os.Exit(1)
}
}
fs, err := f(drivers[cfg.Driver].(map[string]any), fsStream, nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver)
return err
}
managingFS, ok := fs.(storage.UploadSessionLister)
if !ok {
fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver)
os.Exit(1)
}
restart, _ := cmd.Flags().GetBool("restart")
resume, _ := cmd.Flags().GetBool("resume")
clean, _ := cmd.Flags().GetBool("clean")
renderJson, _ := cmd.Flags().GetBool("json")
var stream events.Stream
if restart || resume {
stream, err = event.NewStream(cfg)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to create event stream: %v\n", err)
os.Exit(1)
}
}
filter := buildFilter(cmd)
uploads, err := managingFS.ListUploadSessions(cmd.Context(), filter)
if err != nil {
return err
}
var (
table *tablewriter.Table
raw []Session
)
if !renderJson {
fmt.Println(buildInfo(filter))
table = tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
table.Header([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing", "Scan Date", "Scan Result"})
}
for _, u := range uploads {
ref := u.Reference()
sr, sd := u.ScanData()
session := Session{
Space: ref.GetResourceId().GetSpaceId(),
ID: u.ID(),
Filename: u.Filename(),
Offset: u.Offset(),
Size: u.Size(),
Executant: u.Executant(),
SpaceOwner: u.SpaceOwner(),
Expires: u.Expires(),
Processing: u.IsProcessing(),
ScanDate: sd,
ScanResult: sr,
}
if renderJson {
raw = append(raw, session)
} else {
table.Append([]string{
session.Space,
session.ID,
session.Filename,
strconv.FormatInt(session.Offset, 10),
strconv.FormatInt(session.Size, 10),
session.Executant.OpaqueId,
session.SpaceOwner.GetOpaqueId(),
session.Expires.Format(time.RFC3339),
strconv.FormatBool(session.Processing),
session.ScanDate.Format(time.RFC3339),
session.ScanResult,
})
}
switch {
case restart:
if err := events.Publish(context.Background(), stream, events.RestartPostprocessing{
UploadID: u.ID(),
Timestamp: utils.TSNow(),
}); err != nil {
fmt.Fprintf(os.Stderr, "Failed to send restart event for upload session '%s'\n", u.ID())
// if publishing fails there is no need to try publishing other events - they will fail too.
os.Exit(1)
}
case resume:
if err := events.Publish(context.Background(), stream, events.ResumePostprocessing{
UploadID: u.ID(),
Timestamp: utils.TSNow(),
}); err != nil {
fmt.Fprintf(os.Stderr, "Failed to send resume event for upload session '%s'\n", u.ID())
// if publishing fails there is no need to try publishing other events - they will fail too.
os.Exit(1)
}
case clean:
if err := u.Purge(cmd.Context()); err != nil {
fmt.Fprintf(os.Stderr, "Failed to clean upload session '%s'\n", u.ID())
}
}
}
if !renderJson {
table.Render()
return nil
}
j, err := json.Marshal(raw)
if err != nil {
fmt.Println(err)
return err
}
fmt.Println(string(j))
return nil
},
}
listUploadSessionsCmd.Flags().String("id", "", "filter sessions by upload session id")
listUploadSessionsCmd.Flags().Bool("processing", false, "filter sessions by processing status")
listUploadSessionsCmd.Flags().Bool("expired", false, "filter sessions by expired status")
listUploadSessionsCmd.Flags().Bool("has-virus", false, "filter sessions by virus scan result")
listUploadSessionsCmd.Flags().Bool("json", false, "output as json")
listUploadSessionsCmd.Flags().Bool("restart", false, "send restart event for all listed sessions. Only one of resume/restart/clean can be set.")
listUploadSessionsCmd.Flags().Bool("resume", false, "send resume event for all listed sessions. Only one of resume/restart/clean can be set.")
listUploadSessionsCmd.Flags().Bool("clean", false, "remove uploads for all listed sessions. Only one of resume/restart/clean can be set.")
return listUploadSessionsCmd
}
func buildFilter(cmd *cobra.Command) storage.UploadSessionFilter {
filter := storage.UploadSessionFilter{}
if cmd.Flag("processing").Changed {
processingValue, _ := cmd.Flags().GetBool("processing")
filter.Processing = &processingValue
}
if cmd.Flag("expired").Changed {
expiredValue, _ := cmd.Flags().GetBool("expired")
filter.Expired = &expiredValue
}
if cmd.Flag("has-virus").Changed {
infectedValue, _ := cmd.Flags().GetBool("has-virus")
filter.HasVirus = &infectedValue
}
if cmd.Flag("id").Changed {
idValue, _ := cmd.Flags().GetString("id")
if idValue != "" {
filter.ID = &idValue
}
}
return filter
}
func buildInfo(filter storage.UploadSessionFilter) string {
var b strings.Builder
if filter.Processing != nil {
if !*filter.Processing {
b.WriteString("Not ")
}
if b.Len() == 0 {
b.WriteString("Processing")
} else {
b.WriteString("processing")
}
}
if filter.Expired != nil {
if b.Len() != 0 {
b.WriteString(", ")
}
if !*filter.Expired {
if b.Len() == 0 {
b.WriteString("Not ")
} else {
b.WriteString("not ")
}
}
if b.Len() == 0 {
b.WriteString("Expired")
} else {
b.WriteString("expired")
}
}
if filter.HasVirus != nil {
if b.Len() != 0 {
b.WriteString(", ")
}
if !*filter.HasVirus {
if b.Len() == 0 {
b.WriteString("Not ")
} else {
b.WriteString("not ")
}
}
if b.Len() == 0 {
b.WriteString("Virusinfected")
} else {
b.WriteString("virusinfected")
}
}
if b.Len() == 0 {
b.WriteString("Session")
} else {
b.WriteString(" session")
}
if filter.ID != nil {
b.WriteString(" with id '" + *filter.ID + "'")
} else {
// to make `session` plural
b.WriteString("s")
}
b.WriteString(":")
return b.String()
}
@@ -0,0 +1,85 @@
package command
import (
"testing"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/test-go/testify/require"
)
func TestBuildInfo(t *testing.T) {
testCases := []struct {
alias string
filter storage.UploadSessionFilter
expectedInfo string
}{
{
alias: "empty filter",
filter: storage.UploadSessionFilter{},
expectedInfo: "Sessions:",
},
{
alias: "processing",
filter: storage.UploadSessionFilter{Processing: boolPtr(true)},
expectedInfo: "Processing sessions:",
},
{
alias: "processing and not expired",
filter: storage.UploadSessionFilter{Processing: boolPtr(true), Expired: boolPtr(false)},
expectedInfo: "Processing, not expired sessions:",
},
{
alias: "processing and expired",
filter: storage.UploadSessionFilter{Processing: boolPtr(true), Expired: boolPtr(true)},
expectedInfo: "Processing, expired sessions:",
},
{
alias: "with id",
filter: storage.UploadSessionFilter{ID: strPtr("123")},
expectedInfo: "Session with id '123':",
},
{
alias: "processing, not expired and not virus infected",
filter: storage.UploadSessionFilter{Processing: boolPtr(true), Expired: boolPtr(false), HasVirus: boolPtr(false)},
expectedInfo: "Processing, not expired, not virusinfected sessions:",
},
{
alias: "not virusinfected",
filter: storage.UploadSessionFilter{HasVirus: boolPtr(false)},
expectedInfo: "Not virusinfected sessions:",
},
{
alias: "expired and virusinfected",
filter: storage.UploadSessionFilter{Expired: boolPtr(true), HasVirus: boolPtr(true)},
expectedInfo: "Expired, virusinfected sessions:",
},
{
alias: "expired and not virus infected",
filter: storage.UploadSessionFilter{Expired: boolPtr(true), HasVirus: boolPtr(false)},
expectedInfo: "Expired, not virusinfected sessions:",
},
{
alias: "processing, not expired, virus infected and with id (note: this makes no sense)",
filter: storage.UploadSessionFilter{Processing: boolPtr(true), Expired: boolPtr(false), HasVirus: boolPtr(true), ID: strPtr("123")},
expectedInfo: "Processing, not expired, virusinfected session with id '123':",
},
}
for _, tc := range testCases {
alias := tc.alias
filter := tc.filter
expectedInfo := tc.expectedInfo
t.Run(alias, func(t *testing.T) {
require.Equal(t, expectedInfo, buildInfo(filter))
})
}
}
func boolPtr(b bool) *bool {
return &b
}
func strPtr(s string) *string {
return &s
}
@@ -0,0 +1,49 @@
package command
import (
"fmt"
"os"
"github.com/qsfera/server/pkg/registry"
"github.com/qsfera/server/pkg/version"
"github.com/qsfera/server/services/storage-users/pkg/config"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
"github.com/spf13/cobra"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "print the version of this binary and the running service instances",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Println("")
reg := registry.GetRegistry()
services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name)
if err != nil {
fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err))
return err
}
if len(services) == 0 {
fmt.Println("No running " + cfg.Service.Name + " service found.")
return nil
}
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
table.Header([]string{"Version", "Address", "Id"})
for _, s := range services {
for _, n := range s.Nodes {
table.Append([]string{s.Version, n.Address, n.Id})
}
}
table.Render()
return nil
},
}
}