refactor storage

This commit is contained in:
Christian Richter
2022-04-13 17:04:38 +02:00
parent 1a1a4d6a3c
commit e079465ff8
49 changed files with 77 additions and 77 deletions
@@ -0,0 +1,157 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// AppProvider is the entrypoint for the app provider command.
func AppProvider(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "app-provider",
Usage: "start appprovider for providing apps",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-app-provider")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := appProviderConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(rcfg, pidFile, runtime.WithLogger(&logger.Logger))
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AppProvider.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.AppProvider.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// appProviderConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func appProviderConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AppProvider.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AppProvider.GRPCNetwork,
"address": cfg.Reva.AppProvider.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"appprovider": map[string]interface{}{
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"app_provider_url": cfg.Reva.AppProvider.ExternalAddr,
"driver": cfg.Reva.AppProvider.Driver,
"drivers": map[string]interface{}{
"wopi": map[string]interface{}{
"app_api_key": cfg.Reva.AppProvider.WopiDriver.AppAPIKey,
"app_desktop_only": cfg.Reva.AppProvider.WopiDriver.AppDesktopOnly,
"app_icon_uri": cfg.Reva.AppProvider.WopiDriver.AppIconURI,
"app_int_url": cfg.Reva.AppProvider.WopiDriver.AppInternalURL,
"app_name": cfg.Reva.AppProvider.WopiDriver.AppName,
"app_url": cfg.Reva.AppProvider.WopiDriver.AppURL,
"insecure_connections": cfg.Reva.AppProvider.WopiDriver.Insecure,
"iop_secret": cfg.Reva.AppProvider.WopiDriver.IopSecret,
"jwt_secret": cfg.Reva.AppProvider.WopiDriver.JWTSecret,
"wopi_url": cfg.Reva.AppProvider.WopiDriver.WopiURL,
},
},
},
},
},
}
return rcfg
}
// AppProviderSutureService allows for the app-provider command to be embedded and supervised by a suture supervisor tree.
type AppProviderSutureService struct {
cfg *config.Config
}
// NewAppProvider creates a new store.AppProviderSutureService
func NewAppProvider(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return AppProviderSutureService{
cfg: cfg.Storage,
}
}
func (s AppProviderSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AppProvider.Context = ctx
f := &flag.FlagSet{}
cmdFlags := AppProvider(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AppProvider(s.cfg).Before != nil {
if err := AppProvider(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := AppProvider(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
+176
View File
@@ -0,0 +1,176 @@
package command
import (
"context"
"flag"
"os"
"path"
"path/filepath"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// AuthBasic is the entrypoint for the auth-basic command.
func AuthBasic(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "auth-basic",
Usage: "start authprovider for basic auth",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-auth-basic")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// pre-create folders
if cfg.Reva.AuthProvider.Driver == "json" && cfg.Reva.AuthProvider.JSON != "" {
if err := os.MkdirAll(filepath.Dir(cfg.Reva.AuthProvider.JSON), os.FileMode(0700)); err != nil {
return err
}
}
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := authBasicConfigFromStruct(c, cfg)
logger.Debug().
Str("server", "authbasic").
Interface("reva-config", rcfg).
Msg("config")
if cfg.Reva.AuthProvider.Driver == "ldap" {
if err := waitForLDAPCA(logger, &cfg.Reva.LDAP); err != nil {
logger.Error().Err(err).Msg("The configured LDAP CA cert does not exist")
return err
}
}
gr.Add(func() error {
runtime.RunWithOptions(rcfg, pidFile, runtime.WithLogger(&logger.Logger))
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AuthBasic.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.AuthBasic.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// authBasicConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func authBasicConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AuthBasic.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AuthBasic.GRPCNetwork,
"address": cfg.Reva.AuthBasic.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"authprovider": map[string]interface{}{
"auth_manager": cfg.Reva.AuthProvider.Driver,
"auth_managers": map[string]interface{}{
"json": map[string]interface{}{
"users": cfg.Reva.AuthProvider.JSON,
},
"ldap": ldapConfigFromString(cfg),
"owncloudsql": map[string]interface{}{
"dbusername": cfg.Reva.UserOwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserOwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserOwnCloudSQL.DBHost,
"dbport": cfg.Reva.UserOwnCloudSQL.DBPort,
"dbname": cfg.Reva.UserOwnCloudSQL.DBName,
"idp": cfg.Reva.UserOwnCloudSQL.Idp,
"nobody": cfg.Reva.UserOwnCloudSQL.Nobody,
"join_username": cfg.Reva.UserOwnCloudSQL.JoinUsername,
"join_ownclouduuid": cfg.Reva.UserOwnCloudSQL.JoinOwnCloudUUID,
},
},
},
},
},
}
return rcfg
}
// AuthBasicSutureService allows for the storage-authbasic command to be embedded and supervised by a suture supervisor tree.
type AuthBasicSutureService struct {
cfg *config.Config
}
// NewAuthBasicSutureService creates a new store.AuthBasicSutureService
func NewAuthBasic(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return AuthBasicSutureService{
cfg: cfg.Storage,
}
}
func (s AuthBasicSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AuthBasic.Context = ctx
f := &flag.FlagSet{}
cmdFlags := AuthBasic(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AuthBasic(s.cfg).Before != nil {
if err := AuthBasic(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := AuthBasic(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,152 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// AuthBearer is the entrypoint for the auth-bearer command.
func AuthBearer(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "auth-bearer",
Usage: "start authprovider for bearer auth",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-auth-bearer")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := authBearerConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AuthBearer.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.AuthBearer.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// authBearerConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func authBearerConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AuthBearer.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AuthBearer.GRPCNetwork,
"address": cfg.Reva.AuthBearer.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"authprovider": map[string]interface{}{
"auth_manager": "oidc",
"auth_managers": map[string]interface{}{
"oidc": map[string]interface{}{
"issuer": cfg.Reva.OIDC.Issuer,
"insecure": cfg.Reva.OIDC.Insecure,
"id_claim": cfg.Reva.OIDC.IDClaim,
"uid_claim": cfg.Reva.OIDC.UIDClaim,
"gid_claim": cfg.Reva.OIDC.GIDClaim,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
},
},
},
},
},
}
}
// AuthBearerSutureService allows for the storage-gateway command to be embedded and supervised by a suture supervisor tree.
type AuthBearerSutureService struct {
cfg *config.Config
}
// NewAuthBearerSutureService creates a new gateway.AuthBearerSutureService
func NewAuthBearer(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return AuthBearerSutureService{
cfg: cfg.Storage,
}
}
func (s AuthBearerSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AuthBearer.Context = ctx
f := &flag.FlagSet{}
cmdFlags := AuthBearer(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AuthBearer(s.cfg).Before != nil {
if err := AuthBearer(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := AuthBearer(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,148 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// AuthMachine is the entrypoint for the auth-machine command.
func AuthMachine(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "auth-machine",
Usage: "start authprovider for machine auth",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-auth-machine")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := authMachineConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AuthMachine.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.AuthMachine.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// authMachineConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func authMachineConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AuthMachine.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AuthMachine.GRPCNetwork,
"address": cfg.Reva.AuthMachine.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"authprovider": map[string]interface{}{
"auth_manager": "machine",
"auth_managers": map[string]interface{}{
"machine": map[string]interface{}{
"api_key": cfg.Reva.AuthMachineConfig.MachineAuthAPIKey,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
},
},
},
},
}
}
// AuthMachineSutureService allows for the storage-gateway command to be embedded and supervised by a suture supervisor tree.
type AuthMachineSutureService struct {
cfg *config.Config
}
// NewAuthMachineSutureService creates a new gateway.AuthMachineSutureService
func NewAuthMachine(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return AuthMachineSutureService{
cfg: cfg.Storage,
}
}
func (s AuthMachineSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AuthMachine.Context = ctx
f := &flag.FlagSet{}
cmdFlags := AuthMachine(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AuthMachine(s.cfg).Before != nil {
if err := AuthMachine(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := AuthMachine(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
+372
View File
@@ -0,0 +1,372 @@
package command
import (
"context"
"flag"
"fmt"
"os"
"path"
"strconv"
"strings"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/conversions"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// Frontend is the entrypoint for the frontend command.
func Frontend(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "frontend",
Usage: "start frontend service",
Before: func(c *cli.Context) error {
if err := loadUserAgent(c, cfg); err != nil {
return err
}
return ParseConfig(c, cfg, "storage-frontend")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
//metrics = metrics.New()
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
// pregenerate list of valid localhost ports for the desktop redirect_uri
// TODO use custom scheme like "owncloud://localhost/user/callback" tracked in
var desktopRedirectURIs [65535 - 1024]string
for port := 0; port < len(desktopRedirectURIs); port++ {
desktopRedirectURIs[port] = fmt.Sprintf("http://localhost:%d", (port + 1024))
}
archivers := []map[string]interface{}{
{
"enabled": true,
"version": "2.0.0",
"formats": []string{"tar", "zip"},
"archiver_url": cfg.Reva.Archiver.ArchiverURL,
"max_num_files": strconv.FormatInt(cfg.Reva.Archiver.MaxNumFiles, 10),
"max_size": strconv.FormatInt(cfg.Reva.Archiver.MaxSize, 10),
},
}
appProviders := []map[string]interface{}{
{
"enabled": true,
"version": "1.0.0",
"apps_url": cfg.Reva.AppProvider.AppsURL,
"open_url": cfg.Reva.AppProvider.OpenURL,
"new_url": cfg.Reva.AppProvider.NewURL,
},
}
filesCfg := map[string]interface{}{
"private_links": false,
"bigfilechunking": false,
"blacklisted_files": []string{},
"undelete": true,
"versioning": true,
"archivers": archivers,
"app_providers": appProviders,
"favorites": cfg.Reva.Frontend.Favorites,
}
if cfg.Reva.DefaultUploadProtocol == "tus" {
filesCfg["tus_support"] = map[string]interface{}{
"version": "1.0.0",
"resumable": "1.0.0",
"extension": "creation,creation-with-upload",
"http_method_override": cfg.Reva.UploadHTTPMethodOverride,
"max_chunk_size": cfg.Reva.UploadMaxChunkSize,
}
}
revaCfg := frontendConfigFromStruct(c, cfg, filesCfg)
gr.Add(func() error {
runtime.RunWithOptions(revaCfg, pidFile, runtime.WithLogger(&logger.Logger))
return nil
}, func(_ error) {
logger.Info().Str("server", c.Command.Name).Msg("Shutting down server")
cancel()
})
{
server, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.Frontend.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().
Err(err).
Str("server", "debug").
Msg("Failed to initialize server")
return err
}
gr.Add(server.ListenAndServe, func(_ error) {
cancel()
})
}
if !cfg.Reva.Frontend.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// frontendConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[string]interface{}) map[string]interface{} {
return map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.Users.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint, // Todo or address?
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"http": map[string]interface{}{
"network": cfg.Reva.Frontend.HTTPNetwork,
"address": cfg.Reva.Frontend.HTTPAddr,
"middlewares": map[string]interface{}{
"cors": map[string]interface{}{
"allow_credentials": true,
},
"auth": map[string]interface{}{
"credentials_by_user_agent": cfg.Reva.Frontend.Middleware.Auth.CredentialsByUserAgent,
"credential_chain": []string{"bearer"},
},
},
// TODO build services dynamically
"services": map[string]interface{}{
"appprovider": map[string]interface{}{
"prefix": cfg.Reva.Frontend.AppProviderPrefix,
"transfer_shared_secret": cfg.Reva.TransferSecret,
"timeout": 86400,
"insecure": cfg.Reva.Frontend.AppProviderInsecure,
},
"archiver": map[string]interface{}{
"prefix": cfg.Reva.Frontend.ArchiverPrefix,
"timeout": 86400,
"insecure": cfg.Reva.Frontend.ArchiverInsecure,
"max_num_files": cfg.Reva.Archiver.MaxNumFiles,
"max_size": cfg.Reva.Archiver.MaxSize,
},
"datagateway": map[string]interface{}{
"prefix": cfg.Reva.Frontend.DatagatewayPrefix,
"transfer_shared_secret": cfg.Reva.TransferSecret,
"timeout": 86400,
"insecure": true,
},
"ocdav": map[string]interface{}{
"prefix": cfg.Reva.Frontend.OCDavPrefix,
"files_namespace": cfg.Reva.OCDav.DavFilesNamespace,
"webdav_namespace": cfg.Reva.OCDav.WebdavNamespace,
"timeout": 86400,
"insecure": cfg.Reva.Frontend.OCDavInsecure,
"public_url": cfg.Reva.Frontend.PublicURL,
},
"ocs": map[string]interface{}{
"storage_registry_svc": cfg.Reva.Gateway.Endpoint,
"share_prefix": cfg.Reva.Frontend.OCSSharePrefix,
"home_namespace": cfg.Reva.Frontend.OCSHomeNamespace,
"resource_info_cache_ttl": cfg.Reva.Frontend.OCSResourceInfoCacheTTL,
"prefix": cfg.Reva.Frontend.OCSPrefix,
"additional_info_attribute": cfg.Reva.Frontend.OCSAdditionalInfoAttribute,
"machine_auth_apikey": cfg.Reva.AuthMachineConfig.MachineAuthAPIKey,
"cache_warmup_driver": cfg.Reva.Frontend.OCSCacheWarmupDriver,
"cache_warmup_drivers": map[string]interface{}{
"cbox": map[string]interface{}{
"db_username": cfg.Reva.Sharing.UserSQLUsername,
"db_password": cfg.Reva.Sharing.UserSQLPassword,
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
"namespace": cfg.Reva.UserStorage.EOS.Root,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
},
},
"config": map[string]interface{}{
"version": "1.7",
"website": "ownCloud",
"host": cfg.Reva.Frontend.PublicURL,
"contact": "",
"ssl": "false",
},
"default_upload_protocol": cfg.Reva.DefaultUploadProtocol,
"capabilities": map[string]interface{}{
"capabilities": map[string]interface{}{
"core": map[string]interface{}{
"poll_interval": 60,
"webdav_root": "remote.php/webdav",
"status": map[string]interface{}{
"installed": true,
"maintenance": false,
"needsDbUpgrade": false,
"version": "10.0.11.5",
"versionstring": "10.0.11",
"edition": "community",
"productname": "reva",
"hostname": "",
},
"support_url_signing": true,
},
"checksums": map[string]interface{}{
"supported_types": cfg.Reva.ChecksumSupportedTypes,
"preferred_upload_type": cfg.Reva.ChecksumPreferredUploadType,
},
"files": filesCfg,
"dav": map[string]interface{}{},
"files_sharing": map[string]interface{}{
"api_enabled": true,
"resharing": false,
"group_sharing": true,
"auto_accept_share": true,
"share_with_group_members_only": true,
"share_with_membership_groups_only": true,
"default_permissions": 22,
"search_min_length": 3,
"public": map[string]interface{}{
"enabled": true,
"send_mail": true,
"social_share": true,
"upload": true,
"multiple": true,
"supports_upload_only": true,
"password": map[string]interface{}{
"enforced": true,
"enforced_for": map[string]interface{}{
"read_only": true,
"read_write": true,
"upload_only": true,
},
},
"expire_date": map[string]interface{}{
"enabled": false,
},
},
"user": map[string]interface{}{
"send_mail": true,
"profile_picture": false,
"settings": []map[string]interface{}{
{
"enabled": true,
"version": "1.0.0",
},
},
},
"user_enumeration": map[string]interface{}{
"enabled": true,
"group_members_only": true,
},
"federation": map[string]interface{}{
"outgoing": true,
"incoming": true,
},
},
"spaces": map[string]interface{}{
"version": "0.0.1",
"enabled": cfg.Reva.Frontend.ProjectSpaces,
},
},
"version": map[string]interface{}{
"edition": "reva",
"major": 10,
"minor": 0,
"micro": 11,
"string": "10.0.11",
},
},
},
},
},
}
}
// loadUserAgent reads the user-agent-whitelist-lock-in, since it is a string flag, and attempts to construct a map of
// "user-agent":"challenge" locks in for Reva.
// Modifies cfg. Spaces don't need to be trimmed as urfavecli takes care of it. User agents with spaces are valid. i.e:
// Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:83.0) Gecko/20100101 Firefox/83.0
// This function works by relying in our format of specifying [user-agent:challenge] and the fact that the user agent
// might contain ":" (colon), so the original string is reversed, split in two parts, by the time it is split we
// have the indexes reversed and the tuple is in the format of [challenge:user-agent], then the same process is applied
// in reverse for each individual part
func loadUserAgent(c *cli.Context, cfg *config.Config) error {
cfg.Reva.Frontend.Middleware.Auth.CredentialsByUserAgent = make(map[string]string)
locks := c.StringSlice("user-agent-whitelist-lock-in")
for _, v := range locks {
vv := conversions.Reverse(v)
parts := strings.SplitN(vv, ":", 2)
if len(parts) != 2 {
return fmt.Errorf("unexpected config value for user-agent lock-in: %v, expected format is user-agent:challenge", v)
}
cfg.Reva.Frontend.Middleware.Auth.CredentialsByUserAgent[conversions.Reverse(parts[1])] = conversions.Reverse(parts[0])
}
return nil
}
// FrontendSutureService allows for the storage-frontend command to be embedded and supervised by a suture supervisor tree.
type FrontendSutureService struct {
cfg *config.Config
}
// NewFrontend creates a new frontend.FrontendSutureService
func NewFrontend(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return FrontendSutureService{
cfg: cfg.Storage,
}
}
func (s FrontendSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.Frontend.Context = ctx
f := &flag.FlagSet{}
cmdFlags := Frontend(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if Frontend(s.cfg).Before != nil {
if err := Frontend(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := Frontend(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
+440
View File
@@ -0,0 +1,440 @@
package command
import (
"context"
"encoding/json"
"flag"
"io/ioutil"
"os"
"path"
"strings"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/mitchellh/mapstructure"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/service/external"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/ocis-pkg/shared"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/owncloud/ocis/ocis-pkg/version"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// Gateway is the entrypoint for the gateway command.
func Gateway(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "gateway",
Usage: "start gateway",
Before: func(c *cli.Context) error {
if err := ParseConfig(c, cfg, "storage-gateway"); err != nil {
return err
}
if cfg.Reva.DataGateway.PublicURL == "" {
cfg.Reva.DataGateway.PublicURL = strings.TrimRight(cfg.Reva.Frontend.PublicURL, "/") + "/data"
}
return nil
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := gatewayConfigFromStruct(c, cfg, logger)
logger.Debug().
Str("server", "gateway").
Interface("reva-config", rcfg).
Msg("config")
defer cancel()
gr.Add(func() error {
err := external.RegisterGRPCEndpoint(
ctx,
"com.owncloud.storage",
uuid.String(),
cfg.Reva.Gateway.GRPCAddr,
version.String,
logger,
)
if err != nil {
return err
}
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.Gateway.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.Gateway.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// gatewayConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logger) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.Users.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.Gateway.GRPCNetwork,
"address": cfg.Reva.Gateway.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"gateway": map[string]interface{}{
// registries is located on the gateway
"authregistrysvc": cfg.Reva.Gateway.Endpoint,
"storageregistrysvc": cfg.Reva.Gateway.Endpoint,
"appregistrysvc": cfg.Reva.Gateway.Endpoint,
// user metadata is located on the users services
"preferencessvc": cfg.Reva.Users.Endpoint,
"userprovidersvc": cfg.Reva.Users.Endpoint,
"groupprovidersvc": cfg.Reva.Groups.Endpoint,
"permissionssvc": cfg.Reva.Permissions.Endpoint,
// sharing is located on the sharing service
"usershareprovidersvc": cfg.Reva.Sharing.Endpoint,
"publicshareprovidersvc": cfg.Reva.Sharing.Endpoint,
"ocmshareprovidersvc": cfg.Reva.Sharing.Endpoint,
"commit_share_to_storage_grant": cfg.Reva.Gateway.CommitShareToStorageGrant,
"commit_share_to_storage_ref": cfg.Reva.Gateway.CommitShareToStorageRef,
"share_folder": cfg.Reva.Gateway.ShareFolder, // ShareFolder is the location where to create shares in the recipient's storage provider.
// other
"disable_home_creation_on_login": cfg.Reva.Gateway.DisableHomeCreationOnLogin,
"datagateway": cfg.Reva.DataGateway.PublicURL,
"transfer_shared_secret": cfg.Reva.TransferSecret,
"transfer_expires": cfg.Reva.TransferExpires,
"home_mapping": cfg.Reva.Gateway.HomeMapping,
"etag_cache_ttl": cfg.Reva.Gateway.EtagCacheTTL,
},
"authregistry": map[string]interface{}{
"driver": "static",
"drivers": map[string]interface{}{
"static": map[string]interface{}{
"rules": map[string]interface{}{
"basic": cfg.Reva.AuthBasic.Endpoint,
"bearer": cfg.Reva.AuthBearer.Endpoint,
"machine": cfg.Reva.AuthMachine.Endpoint,
"publicshares": cfg.Reva.StoragePublicLink.Endpoint,
},
},
},
},
"appregistry": map[string]interface{}{
"driver": "static",
"drivers": map[string]interface{}{
"static": map[string]interface{}{
"mime_types": mimetypes(cfg, logger),
},
},
},
"storageregistry": map[string]interface{}{
"driver": cfg.Reva.StorageRegistry.Driver,
"drivers": map[string]interface{}{
"spaces": map[string]interface{}{
"providers": spacesProviders(cfg, logger),
},
},
},
},
},
}
return rcfg
}
func spacesProviders(cfg *config.Config, logger log.Logger) map[string]map[string]interface{} {
// if a list of rules is given it overrides the generated rules from below
if len(cfg.Reva.StorageRegistry.Rules) > 0 {
rules := map[string]map[string]interface{}{}
for i := range cfg.Reva.StorageRegistry.Rules {
parts := strings.SplitN(cfg.Reva.StorageRegistry.Rules[i], "=", 2)
rules[parts[0]] = map[string]interface{}{"address": parts[1]}
}
return rules
}
// check if the rules have to be read from a json file
if cfg.Reva.StorageRegistry.JSON != "" {
data, err := ioutil.ReadFile(cfg.Reva.StorageRegistry.JSON)
if err != nil {
logger.Error().Err(err).Msg("Failed to read storage registry rules from JSON file: " + cfg.Reva.StorageRegistry.JSON)
return nil
}
var rules map[string]map[string]interface{}
if err = json.Unmarshal(data, &rules); err != nil {
logger.Error().Err(err).Msg("Failed to unmarshal storage registry rules")
return nil
}
return rules
}
// generate rules based on default config
return map[string]map[string]interface{}{
cfg.Reva.StorageUsers.Endpoint: {
"spaces": map[string]interface{}{
"personal": map[string]interface{}{
"mount_point": "/users",
"path_template": "/users/{{.Space.Owner.Id.OpaqueId}}",
},
"project": map[string]interface{}{
"mount_point": "/projects",
"path_template": "/projects/{{.Space.Name}}",
},
},
},
cfg.Reva.StorageShares.Endpoint: {
"spaces": map[string]interface{}{
"virtual": map[string]interface{}{
// The root of the share jail is mounted here
"mount_point": "/users/{{.CurrentUser.Id.OpaqueId}}/Shares",
},
"grant": map[string]interface{}{
// Grants are relative to a space root that the gateway will determine with a stat
"mount_point": ".",
},
"mountpoint": map[string]interface{}{
// The jail needs to be filled with mount points
// .Space.Name is a path relative to the mount point
"mount_point": "/users/{{.CurrentUser.Id.OpaqueId}}/Shares",
"path_template": "/users/{{.CurrentUser.Id.OpaqueId}}/Shares/{{.Space.Name}}",
},
},
},
// public link storage returns the mount id of the actual storage
cfg.Reva.StoragePublicLink.Endpoint: {
"spaces": map[string]interface{}{
"grant": map[string]interface{}{
"mount_point": ".",
},
"mountpoint": map[string]interface{}{
"mount_point": "/public",
"path_template": "/public/{{.Space.Root.OpaqueId}}",
},
},
},
// medatada storage not part of the global namespace
}
}
func mimetypes(cfg *config.Config, logger log.Logger) []map[string]interface{} {
type mimeTypeConfig struct {
MimeType string `json:"mime_type" mapstructure:"mime_type"`
Extension string `json:"extension" mapstructure:"extension"`
Name string `json:"name" mapstructure:"name"`
Description string `json:"description" mapstructure:"description"`
Icon string `json:"icon" mapstructure:"icon"`
DefaultApp string `json:"default_app" mapstructure:"default_app"`
AllowCreation bool `json:"allow_creation" mapstructure:"allow_creation"`
}
var mimetypes []mimeTypeConfig
var m []map[string]interface{}
// load default app mimetypes from a json file
if cfg.Reva.AppRegistry.MimetypesJSON != "" {
data, err := ioutil.ReadFile(cfg.Reva.AppRegistry.MimetypesJSON)
if err != nil {
logger.Error().Err(err).Msg("Failed to read app registry mimetypes from JSON file: " + cfg.Reva.AppRegistry.MimetypesJSON)
return nil
}
if err = json.Unmarshal(data, &mimetypes); err != nil {
logger.Error().Err(err).Msg("Failed to unmarshal storage registry rules")
return nil
}
if err := mapstructure.Decode(mimetypes, &m); err != nil {
logger.Error().Err(err).Msg("Failed to decode defaultapp registry mimetypes to mapstructure")
return nil
}
return m
}
logger.Info().Msg("No app registry mimetypes JSON file provided, loading default configuration")
mimetypes = []mimeTypeConfig{
{
MimeType: "application/pdf",
Extension: "pdf",
Name: "PDF",
Description: "PDF document",
},
{
MimeType: "application/vnd.oasis.opendocument.text",
Extension: "odt",
Name: "OpenDocument",
Description: "OpenDocument text document",
AllowCreation: true,
},
{
MimeType: "application/vnd.oasis.opendocument.spreadsheet",
Extension: "ods",
Name: "OpenSpreadsheet",
Description: "OpenDocument spreadsheet document",
AllowCreation: true,
},
{
MimeType: "application/vnd.oasis.opendocument.presentation",
Extension: "odp",
Name: "OpenPresentation",
Description: "OpenDocument presentation document",
AllowCreation: true,
},
{
MimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
Extension: "docx",
Name: "Microsoft Word",
Description: "Microsoft Word document",
AllowCreation: true,
},
{
MimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
Extension: "xlsx",
Name: "Microsoft Excel",
Description: "Microsoft Excel document",
AllowCreation: true,
},
{
MimeType: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
Extension: "pptx",
Name: "Microsoft PowerPoint",
Description: "Microsoft PowerPoint document",
AllowCreation: true,
},
{
MimeType: "application/vnd.jupyter",
Extension: "ipynb",
Name: "Jupyter Notebook",
Description: "Jupyter Notebook",
},
{
MimeType: "text/markdown",
Extension: "md",
Name: "Markdown file",
Description: "Markdown file",
AllowCreation: true,
},
{
MimeType: "application/compressed-markdown",
Extension: "zmd",
Name: "Compressed markdown file",
Description: "Compressed markdown file",
},
}
if err := mapstructure.Decode(mimetypes, &m); err != nil {
logger.Error().Err(err).Msg("Failed to decode defaultapp registry mimetypes to mapstructure")
return nil
}
return m
}
// GatewaySutureService allows for the storage-gateway command to be embedded and supervised by a suture supervisor tree.
type GatewaySutureService struct {
cfg *config.Config
}
// NewGatewaySutureService creates a new gateway.GatewaySutureService
func NewGateway(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return GatewaySutureService{
cfg: cfg.Storage,
}
}
func (s GatewaySutureService) Serve(ctx context.Context) error {
s.cfg.Reva.Gateway.Context = ctx
f := &flag.FlagSet{}
cmdFlags := Gateway(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if Gateway(s.cfg).Before != nil {
if err := Gateway(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := Gateway(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
// ParseConfig loads accounts configuration from known paths.
func ParseConfig(c *cli.Context, cfg *config.Config, storageExtension string) error {
conf, err := ociscfg.BindSourcesToStructs(storageExtension, cfg)
if err != nil {
return err
}
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &shared.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil {
cfg.Log = &shared.Log{}
}
// load all env variables relevant to the config in the current context.
conf.LoadOSEnv(config.GetEnv(cfg), false)
bindings := config.StructMappings(cfg)
return ociscfg.BindEnv(conf, bindings)
}
+176
View File
@@ -0,0 +1,176 @@
package command
import (
"context"
"flag"
"os"
"path"
"path/filepath"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// Groups is the entrypoint for the sharing command.
func Groups(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "groups",
Usage: "start groups service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-groups")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// pre-create folders
if cfg.Reva.Groups.Driver == "json" && cfg.Reva.Groups.JSON != "" {
if err := os.MkdirAll(filepath.Dir(cfg.Reva.Groups.JSON), os.FileMode(0700)); err != nil {
return err
}
}
cuuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+cuuid.String()+".pid")
rcfg := groupsConfigFromStruct(c, cfg)
if cfg.Reva.Groups.Driver == "ldap" {
if err := waitForLDAPCA(logger, &cfg.Reva.LDAP); err != nil {
logger.Error().Err(err).Msg("The configured LDAP CA cert does not exist")
return err
}
}
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.Groups.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.Groups.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// groupsConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func groupsConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.Groups.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.Groups.GRPCNetwork,
"address": cfg.Reva.Groups.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"groupprovider": map[string]interface{}{
"driver": cfg.Reva.Groups.Driver,
"drivers": map[string]interface{}{
"json": map[string]interface{}{
"groups": cfg.Reva.Groups.JSON,
},
"ldap": ldapConfigFromString(cfg),
"rest": map[string]interface{}{
"client_id": cfg.Reva.UserGroupRest.ClientID,
"client_secret": cfg.Reva.UserGroupRest.ClientSecret,
"redis_address": cfg.Reva.UserGroupRest.RedisAddress,
"redis_username": cfg.Reva.UserGroupRest.RedisUsername,
"redis_password": cfg.Reva.UserGroupRest.RedisPassword,
"group_members_cache_expiration": cfg.Reva.Groups.GroupMembersCacheExpiration,
"id_provider": cfg.Reva.UserGroupRest.IDProvider,
"api_base_url": cfg.Reva.UserGroupRest.APIBaseURL,
"oidc_token_endpoint": cfg.Reva.UserGroupRest.OIDCTokenEndpoint,
"target_api": cfg.Reva.UserGroupRest.TargetAPI,
},
},
},
},
},
}
}
// GroupSutureService allows for the storage-groupprovider command to be embedded and supervised by a suture supervisor tree.
type GroupSutureService struct {
cfg *config.Config
}
// NewGroupProviderSutureService creates a new storage.GroupProvider
func NewGroupProvider(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return GroupSutureService{
cfg: cfg.Storage,
}
}
func (s GroupSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.Groups.Context = ctx
f := &flag.FlagSet{}
cmdFlags := Groups(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if Groups(s.cfg).Before != nil {
if err := Groups(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := Groups(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
+51
View File
@@ -0,0 +1,51 @@
package command
import (
"fmt"
"net/http"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/urfave/cli/v2"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "health",
Usage: "check health status",
Category: "info",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
resp, err := http.Get(
fmt.Sprintf(
"http://%s/healthz",
cfg.Debug.Addr,
),
)
if err != nil {
logger.Fatal().
Err(err).
Msg("Failed to request health check")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Fatal().
Int("code", resp.StatusCode).
Msg("Health seems to be in bad state")
}
logger.Debug().
Int("code", resp.StatusCode).
Msg("Health got a good state")
return nil
},
}
}
+60
View File
@@ -0,0 +1,60 @@
package command
import (
"errors"
"os"
"time"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
const caTimeout = 5
func ldapConfigFromString(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"uri": cfg.Reva.LDAP.URI,
"cacert": cfg.Reva.LDAP.CACert,
"insecure": cfg.Reva.LDAP.Insecure,
"bind_username": cfg.Reva.LDAP.BindDN,
"bind_password": cfg.Reva.LDAP.BindPassword,
"user_base_dn": cfg.Reva.LDAP.UserBaseDN,
"group_base_dn": cfg.Reva.LDAP.GroupBaseDN,
"user_filter": cfg.Reva.LDAP.UserFilter,
"group_filter": cfg.Reva.LDAP.GroupFilter,
"user_objectclass": cfg.Reva.LDAP.UserObjectClass,
"group_objectclass": cfg.Reva.LDAP.GroupObjectClass,
"login_attributes": cfg.Reva.LDAP.LoginAttributes,
"idp": cfg.Reva.LDAP.IDP,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"user_schema": map[string]interface{}{
"id": cfg.Reva.LDAP.UserSchema.ID,
"idIsOctetString": cfg.Reva.LDAP.UserSchema.IDIsOctetString,
"mail": cfg.Reva.LDAP.UserSchema.Mail,
"displayName": cfg.Reva.LDAP.UserSchema.DisplayName,
"userName": cfg.Reva.LDAP.UserSchema.Username,
},
"group_schema": map[string]interface{}{
"id": cfg.Reva.LDAP.GroupSchema.ID,
"idIsOctetString": cfg.Reva.LDAP.GroupSchema.IDIsOctetString,
"mail": cfg.Reva.LDAP.GroupSchema.Mail,
"displayName": cfg.Reva.LDAP.GroupSchema.DisplayName,
"groupName": cfg.Reva.LDAP.GroupSchema.Groupname,
"member": cfg.Reva.LDAP.GroupSchema.Member,
},
}
}
func waitForLDAPCA(log log.Logger, cfg *config.LDAP) error {
if !cfg.Insecure && cfg.CACert != "" {
if _, err := os.Stat(cfg.CACert); errors.Is(err, os.ErrNotExist) {
log.Warn().Str("LDAP CACert", cfg.CACert).Msgf("File does not exist. Waiting %d seconds for it to appear.", caTimeout)
time.Sleep(caTimeout * time.Second)
if _, err := os.Stat(cfg.CACert); errors.Is(err, os.ErrNotExist) {
log.Warn().Str("LDAP CACert", cfg.CACert).Msgf("File does still not exist after Timeout")
return err
}
}
}
return nil
}
+62
View File
@@ -0,0 +1,62 @@
package command
import (
"os"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/clihelper"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/urfave/cli/v2"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) cli.Commands {
return []*cli.Command{
Frontend(cfg),
Gateway(cfg),
Users(cfg),
Groups(cfg),
AppProvider(cfg),
AuthBasic(cfg),
AuthBearer(cfg),
AuthMachine(cfg),
Sharing(cfg),
StoragePublicLink(cfg),
StorageShares(cfg),
StorageUsers(cfg),
StorageMetadata(cfg),
Health(cfg),
}
}
// Execute is the entry point for the storage command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cli.App{
Name: "storage",
Usage: "Storage service for oCIS",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage")
},
Commands: GetCommands(cfg),
})
cli.HelpFlag = &cli.BoolFlag{
Name: "help,h",
Usage: "Show the help",
}
return app.Run(os.Args)
}
// NewLogger initializes a service-specific logger instance.
func NewLogger(cfg *config.Config) log.Logger {
return log.NewLogger(
log.Name("storage"),
log.Level(cfg.Log.Level),
log.Pretty(cfg.Log.Pretty),
log.Color(cfg.Log.Color),
log.File(cfg.Log.File),
)
}
+240
View File
@@ -0,0 +1,240 @@
package command
import (
"context"
"flag"
"os"
"path"
"path/filepath"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// Sharing is the entrypoint for the sharing command.
func Sharing(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "sharing",
Usage: "start sharing service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-sharing")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// precreate folders
if cfg.Reva.Sharing.UserDriver == "json" && cfg.Reva.Sharing.UserJSONFile != "" {
if err := os.MkdirAll(filepath.Dir(cfg.Reva.Sharing.UserJSONFile), os.FileMode(0700)); err != nil {
return err
}
}
if cfg.Reva.Sharing.PublicDriver == "json" && cfg.Reva.Sharing.PublicJSONFile != "" {
if err := os.MkdirAll(filepath.Dir(cfg.Reva.Sharing.PublicJSONFile), os.FileMode(0700)); err != nil {
return err
}
}
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := sharingConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debug, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.Sharing.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debug.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.Sharing.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// sharingConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func sharingConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.Sharing.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.Sharing.GRPCNetwork,
"address": cfg.Reva.Sharing.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"usershareprovider": map[string]interface{}{
"driver": cfg.Reva.Sharing.UserDriver,
"drivers": map[string]interface{}{
"json": map[string]interface{}{
"file": cfg.Reva.Sharing.UserJSONFile,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
"sql": map[string]interface{}{ // cernbox sql
"db_username": cfg.Reva.Sharing.UserSQLUsername,
"db_password": cfg.Reva.Sharing.UserSQLPassword,
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
"password_hash_cost": cfg.Reva.Sharing.PublicPasswordHashCost,
"enable_expired_shares_cleanup": cfg.Reva.Sharing.PublicEnableExpiredSharesCleanup,
"janitor_run_interval": cfg.Reva.Sharing.PublicJanitorRunInterval,
},
"oc10-sql": map[string]interface{}{
"gateway_addr": cfg.Reva.Gateway.Endpoint,
"storage_mount_id": cfg.Reva.Sharing.UserStorageMountID,
"db_username": cfg.Reva.Sharing.UserSQLUsername,
"db_password": cfg.Reva.Sharing.UserSQLPassword,
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
},
"cs3": map[string]interface{}{
"gateway_addr": cfg.Reva.Gateway.Endpoint,
"provider_addr": cfg.Reva.Sharing.CS3ProviderAddr,
"service_user_id": cfg.Reva.Sharing.CS3ServiceUser,
"service_user_idp": cfg.Reva.Sharing.CS3ServiceUserIdp,
"machine_auth_apikey": cfg.Reva.AuthMachineConfig.MachineAuthAPIKey,
},
},
},
"publicshareprovider": map[string]interface{}{
"driver": cfg.Reva.Sharing.PublicDriver,
"drivers": map[string]interface{}{
"json": map[string]interface{}{
"file": cfg.Reva.Sharing.PublicJSONFile,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
"sql": map[string]interface{}{
"db_username": cfg.Reva.Sharing.UserSQLUsername,
"db_password": cfg.Reva.Sharing.UserSQLPassword,
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
"password_hash_cost": cfg.Reva.Sharing.PublicPasswordHashCost,
"enable_expired_shares_cleanup": cfg.Reva.Sharing.PublicEnableExpiredSharesCleanup,
"janitor_run_interval": cfg.Reva.Sharing.PublicJanitorRunInterval,
},
"oc10-sql": map[string]interface{}{
"gateway_addr": cfg.Reva.Gateway.Endpoint,
"storage_mount_id": cfg.Reva.Sharing.UserStorageMountID,
"db_username": cfg.Reva.Sharing.UserSQLUsername,
"db_password": cfg.Reva.Sharing.UserSQLPassword,
"db_host": cfg.Reva.Sharing.UserSQLHost,
"db_port": cfg.Reva.Sharing.UserSQLPort,
"db_name": cfg.Reva.Sharing.UserSQLName,
"password_hash_cost": cfg.Reva.Sharing.PublicPasswordHashCost,
"enable_expired_shares_cleanup": cfg.Reva.Sharing.PublicEnableExpiredSharesCleanup,
"janitor_run_interval": cfg.Reva.Sharing.PublicJanitorRunInterval,
},
"cs3": map[string]interface{}{
"gateway_addr": cfg.Reva.Gateway.Endpoint,
"provider_addr": cfg.Reva.Sharing.CS3ProviderAddr,
"service_user_id": cfg.Reva.Sharing.CS3ServiceUser,
"service_user_idp": cfg.Reva.Sharing.CS3ServiceUserIdp,
"machine_auth_apikey": cfg.Reva.AuthMachineConfig.MachineAuthAPIKey,
},
},
},
},
"interceptors": map[string]interface{}{
"eventsmiddleware": map[string]interface{}{
"group": "sharing",
"type": "nats",
"address": cfg.Reva.Sharing.Events.Address,
"clusterID": cfg.Reva.Sharing.Events.ClusterID,
},
},
},
}
return rcfg
}
// SharingSutureService allows for the storage-sharing command to be embedded and supervised by a suture supervisor tree.
type SharingSutureService struct {
cfg *config.Config
}
// NewSharingSutureService creates a new store.SharingSutureService
func NewSharing(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return SharingSutureService{
cfg: cfg.Storage,
}
}
func (s SharingSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.Sharing.Context = ctx
f := &flag.FlagSet{}
cmdFlags := Sharing(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if Sharing(s.cfg).Before != nil {
if err := Sharing(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := Sharing(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,79 @@
package storagedrivers
import (
"github.com/owncloud/ocis/extensions/storage/pkg/config"
)
func MetadataDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Reva.MetadataStorage.EOS.Root,
"shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.MetadataStorage.EOS.UploadsNamespace,
"eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.MetadataStorage.EOS.MasterURL,
"slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol,
"keytab": cfg.Reva.MetadataStorage.EOS.Keytab,
"single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername,
"enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC,
"enable_home": false,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Reva.MetadataStorage.EOS.Root,
"shadow_namespace": cfg.Reva.MetadataStorage.EOS.ShadowNamespace,
"eos_binary": cfg.Reva.MetadataStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.MetadataStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.MetadataStorage.EOS.MasterURL,
"master_grpc_uri": cfg.Reva.MetadataStorage.EOS.GrpcURI,
"slave_url": cfg.Reva.MetadataStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.MetadataStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.MetadataStorage.EOS.SecProtocol,
"keytab": cfg.Reva.MetadataStorage.EOS.Keytab,
"single_username": cfg.Reva.MetadataStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.MetadataStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.MetadataStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.MetadataStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.MetadataStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.MetadataStorage.EOS.UseKeytab,
"enable_home": false,
"gatewaysvc": cfg.Reva.MetadataStorage.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.Local.Root,
},
"ocis": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.OCIS.Root,
"user_layout": cfg.Reva.MetadataStorage.OCIS.UserLayout,
"treetime_accounting": false,
"treesize_accounting": false,
"permissionssvc": cfg.Reva.Permissions.Endpoint,
},
"s3": map[string]interface{}{
"region": cfg.Reva.MetadataStorage.S3.Region,
"access_key": cfg.Reva.MetadataStorage.S3.AccessKey,
"secret_key": cfg.Reva.MetadataStorage.S3.SecretKey,
"endpoint": cfg.Reva.MetadataStorage.S3.Endpoint,
"bucket": cfg.Reva.MetadataStorage.S3.Bucket,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.MetadataStorage.S3NG.Root,
"enable_home": false,
"user_layout": cfg.Reva.MetadataStorage.S3NG.UserLayout,
"treetime_accounting": false,
"treesize_accounting": false,
"permissionssvc": cfg.Reva.Permissions.Endpoint,
"s3.region": cfg.Reva.MetadataStorage.S3NG.Region,
"s3.access_key": cfg.Reva.MetadataStorage.S3NG.AccessKey,
"s3.secret_key": cfg.Reva.MetadataStorage.S3NG.SecretKey,
"s3.endpoint": cfg.Reva.MetadataStorage.S3NG.Endpoint,
"s3.bucket": cfg.Reva.MetadataStorage.S3NG.Bucket,
},
}
}
@@ -0,0 +1,126 @@
package storagedrivers
import (
"github.com/owncloud/ocis/extensions/storage/pkg/config"
)
func UserDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eoshome": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"uploads_namespace": cfg.Reva.UserStorage.EOS.UploadsNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Reva.UserStorage.EOS.Root,
"shadow_namespace": cfg.Reva.UserStorage.EOS.ShadowNamespace,
"share_folder": cfg.Reva.UserStorage.EOS.ShareFolder,
"eos_binary": cfg.Reva.UserStorage.EOS.EosBinary,
"xrdcopy_binary": cfg.Reva.UserStorage.EOS.XrdcopyBinary,
"master_url": cfg.Reva.UserStorage.EOS.MasterURL,
"master_grpc_uri": cfg.Reva.UserStorage.EOS.GrpcURI,
"slave_url": cfg.Reva.UserStorage.EOS.SlaveURL,
"cache_directory": cfg.Reva.UserStorage.EOS.CacheDirectory,
"sec_protocol": cfg.Reva.UserStorage.EOS.SecProtocol,
"keytab": cfg.Reva.UserStorage.EOS.Keytab,
"single_username": cfg.Reva.UserStorage.EOS.SingleUsername,
"user_layout": cfg.Reva.UserStorage.EOS.UserLayout,
"enable_logging": cfg.Reva.UserStorage.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Reva.UserStorage.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Reva.UserStorage.EOS.ForceSingleUserMode,
"use_keytab": cfg.Reva.UserStorage.EOS.UseKeytab,
"enable_home": false,
"gatewaysvc": cfg.Reva.UserStorage.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
},
"localhome": map[string]interface{}{
"root": cfg.Reva.UserStorage.Local.Root,
"share_folder": cfg.Reva.UserStorage.Local.ShareFolder,
"user_layout": cfg.Reva.UserStorage.Local.UserLayout,
},
"owncloudsql": map[string]interface{}{
"datadirectory": cfg.Reva.UserStorage.OwnCloudSQL.Root,
"upload_info_dir": cfg.Reva.UserStorage.OwnCloudSQL.UploadInfoDir,
"share_folder": cfg.Reva.UserStorage.OwnCloudSQL.ShareFolder,
"user_layout": cfg.Reva.UserStorage.OwnCloudSQL.UserLayout,
"enable_home": false,
"dbusername": cfg.Reva.UserStorage.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserStorage.OwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserStorage.OwnCloudSQL.DBHost,
"dbport": cfg.Reva.UserStorage.OwnCloudSQL.DBPort,
"dbname": cfg.Reva.UserStorage.OwnCloudSQL.DBName,
"userprovidersvc": cfg.Reva.Users.Endpoint,
},
"ocis": map[string]interface{}{
"root": cfg.Reva.UserStorage.OCIS.Root,
"user_layout": cfg.Reva.UserStorage.OCIS.UserLayout,
"share_folder": cfg.Reva.UserStorage.OCIS.ShareFolder,
"personalspacealias_template": cfg.Reva.UserStorage.OCIS.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Reva.UserStorage.OCIS.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Reva.Permissions.Endpoint,
},
"s3": map[string]interface{}{
"enable_home": false,
"region": cfg.Reva.UserStorage.S3.Region,
"access_key": cfg.Reva.UserStorage.S3.AccessKey,
"secret_key": cfg.Reva.UserStorage.S3.SecretKey,
"endpoint": cfg.Reva.UserStorage.S3.Endpoint,
"bucket": cfg.Reva.UserStorage.S3.Bucket,
"prefix": cfg.Reva.UserStorage.S3.Root,
},
"s3ng": map[string]interface{}{
"root": cfg.Reva.UserStorage.S3NG.Root,
"user_layout": cfg.Reva.UserStorage.S3NG.UserLayout,
"share_folder": cfg.Reva.UserStorage.S3NG.ShareFolder,
"personalspacealias_template": cfg.Reva.UserStorage.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Reva.UserStorage.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Reva.Permissions.Endpoint,
"s3.region": cfg.Reva.UserStorage.S3NG.Region,
"s3.access_key": cfg.Reva.UserStorage.S3NG.AccessKey,
"s3.secret_key": cfg.Reva.UserStorage.S3NG.SecretKey,
"s3.endpoint": cfg.Reva.UserStorage.S3NG.Endpoint,
"s3.bucket": cfg.Reva.UserStorage.S3NG.Bucket,
},
}
}
@@ -0,0 +1,194 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/command/storagedrivers"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/service/external"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/version"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// StorageMetadata the entrypoint for the storage-storage-metadata command.
//
// It provides a ocis-specific storage store metadata (shares,account,settings...)
func StorageMetadata(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-metadata",
Usage: "start storage-metadata service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-metadata")
},
Category: "extensions",
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := func() (context.Context, context.CancelFunc) {
if cfg.Reva.StorageMetadata.Context == nil {
return context.WithCancel(context.Background())
}
return context.WithCancel(cfg.Reva.StorageMetadata.Context)
}()
defer cancel()
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.Must(uuid.NewV4()).String()+".pid")
rcfg := storageMetadataFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StorageMetadata.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().
Err(err).
Str("server", c.Command.Name+"-debug").
Msg("Failed to initialize server")
return err
}
gr.Add(func() error {
return debugServer.ListenAndServe()
}, func(_ error) {
_ = debugServer.Shutdown(ctx)
cancel()
})
if !cfg.Reva.StorageMetadata.Supervised {
sync.Trap(&gr, cancel)
}
if err := external.RegisterGRPCEndpoint(
ctx,
"com.owncloud.storage.metadata",
uuid.Must(uuid.NewV4()).String(),
cfg.Reva.StorageMetadata.GRPCAddr,
version.String,
logger,
); err != nil {
logger.Fatal().Err(err).Msg("failed to register the grpc endpoint")
}
return gr.Run()
},
}
}
// storageMetadataFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StorageMetadata.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StorageMetadata.GRPCNetwork,
"address": cfg.Reva.StorageMetadata.GRPCAddr,
"interceptors": map[string]interface{}{
"log": map[string]interface{}{},
},
"services": map[string]interface{}{
"storageprovider": map[string]interface{}{
"driver": cfg.Reva.StorageMetadata.Driver,
"drivers": storagedrivers.MetadataDrivers(cfg),
"data_server_url": cfg.Reva.StorageMetadata.DataServerURL,
"tmp_folder": cfg.Reva.StorageMetadata.TempFolder,
},
},
},
"http": map[string]interface{}{
"network": cfg.Reva.StorageMetadata.HTTPNetwork,
"address": cfg.Reva.StorageMetadata.HTTPAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"dataprovider": map[string]interface{}{
"prefix": "data",
"driver": cfg.Reva.StorageMetadata.Driver,
"drivers": storagedrivers.MetadataDrivers(cfg),
"timeout": 86400,
"insecure": cfg.Reva.StorageMetadata.DataProvider.Insecure,
"disable_tus": true,
},
},
},
}
return rcfg
}
// SutureService allows for the storage-metadata command to be embedded and supervised by a suture supervisor tree.
type MetadataSutureService struct {
cfg *config.Config
}
// NewSutureService creates a new storagemetadata.SutureService
func NewStorageMetadata(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return MetadataSutureService{
cfg: cfg.Storage,
}
}
func (s MetadataSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageMetadata.Context = ctx
f := &flag.FlagSet{}
cmdFlags := StorageMetadata(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StorageMetadata(s.cfg).Before != nil {
if err := StorageMetadata(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := StorageMetadata(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,154 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// StoragePublicLink is the entrypoint for the reva-storage-public-link command.
func StoragePublicLink(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-public-link",
Usage: "start storage-public-link service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-public-link")
},
Category: "extensions",
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.Must(uuid.NewV4()).String()+".pid")
rcfg := storagePublicLinkConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StoragePublicLink.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.StoragePublicLink.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// storagePublicLinkConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storagePublicLinkConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StoragePublicLink.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StoragePublicLink.GRPCNetwork,
"address": cfg.Reva.StoragePublicLink.GRPCAddr,
"interceptors": map[string]interface{}{
"log": map[string]interface{}{},
},
"services": map[string]interface{}{
"publicstorageprovider": map[string]interface{}{
"mount_id": cfg.Reva.StoragePublicLink.MountID,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
"authprovider": map[string]interface{}{
"auth_manager": "publicshares",
"auth_managers": map[string]interface{}{
"publicshares": map[string]interface{}{
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
},
},
},
},
}
return rcfg
}
// StoragePublicLinkSutureService allows for the storage-public-link command to be embedded and supervised by a suture supervisor tree.
type StoragePublicLinkSutureService struct {
cfg *config.Config
}
// NewStoragePublicLinkSutureService creates a new storage.StoragePublicLinkSutureService
func NewStoragePublicLink(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return StoragePublicLinkSutureService{
cfg: cfg.Storage,
}
}
func (s StoragePublicLinkSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StoragePublicLink.Context = ctx
f := &flag.FlagSet{}
cmdFlags := StoragePublicLink(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StoragePublicLink(s.cfg).Before != nil {
if err := StoragePublicLink(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := StoragePublicLink(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,153 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// StorageShares is the entrypoint for the storage-shares command.
func StorageShares(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-shares",
Usage: "start storage-shares service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-shares")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := storageSharesConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StorageShares.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.StorageShares.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// storageSharesConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storageSharesConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StorageShares.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StorageShares.GRPCNetwork,
"address": cfg.Reva.StorageShares.GRPCAddr,
"services": map[string]interface{}{
"sharesstorageprovider": map[string]interface{}{
"usershareprovidersvc": cfg.Reva.Sharing.Endpoint,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
},
},
},
}
if cfg.Reva.StorageShares.ReadOnly {
gcfg := rcfg["grpc"].(map[string]interface{})
gcfg["interceptors"] = map[string]interface{}{
"readonly": map[string]interface{}{},
}
}
return rcfg
}
// StorageSharesSutureService allows for the storage-shares command to be embedded and supervised by a suture supervisor tree.
type StorageSharesSutureService struct {
cfg *config.Config
}
// NewStorageShares creates a new storage.StorageSharesSutureService
func NewStorageShares(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return StorageSharesSutureService{
cfg: cfg.Storage,
}
}
func (s StorageSharesSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageShares.Context = ctx
f := &flag.FlagSet{}
cmdFlags := StorageShares(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StorageShares(s.cfg).Before != nil {
if err := StorageShares(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := StorageShares(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
@@ -0,0 +1,182 @@
package command
import (
"context"
"flag"
"os"
"path"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/command/storagedrivers"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// StorageUsers is the entrypoint for the storage-users command.
func StorageUsers(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-users",
Usage: "start storage-users service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-userprovider")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := storageUsersConfigFromStruct(c, cfg)
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StorageUsers.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.StorageUsers.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// storageUsersConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func storageUsersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StorageUsers.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StorageUsers.GRPCNetwork,
"address": cfg.Reva.StorageUsers.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"storageprovider": map[string]interface{}{
"driver": cfg.Reva.StorageUsers.Driver,
"drivers": storagedrivers.UserDrivers(cfg),
"mount_id": cfg.Reva.StorageUsers.MountID,
"expose_data_server": cfg.Reva.StorageUsers.ExposeDataServer,
"data_server_url": cfg.Reva.StorageUsers.DataServerURL,
"tmp_folder": cfg.Reva.StorageUsers.TempFolder,
},
},
"interceptors": map[string]interface{}{
"eventsmiddleware": map[string]interface{}{
"group": "sharing",
"type": "nats",
"address": cfg.Reva.Sharing.Events.Address,
"clusterID": cfg.Reva.Sharing.Events.ClusterID,
},
},
},
"http": map[string]interface{}{
"network": cfg.Reva.StorageUsers.HTTPNetwork,
"address": cfg.Reva.StorageUsers.HTTPAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"dataprovider": map[string]interface{}{
"prefix": cfg.Reva.StorageUsers.HTTPPrefix,
"driver": cfg.Reva.StorageUsers.Driver,
"drivers": storagedrivers.UserDrivers(cfg),
"timeout": 86400,
"insecure": cfg.Reva.StorageUsers.DataProvider.Insecure,
"disable_tus": false,
},
},
},
}
if cfg.Reva.StorageUsers.ReadOnly {
gcfg := rcfg["grpc"].(map[string]interface{})
gcfg["interceptors"] = map[string]interface{}{
"readonly": map[string]interface{}{},
}
}
return rcfg
}
// StorageUsersSutureService allows for the storage-home command to be embedded and supervised by a suture supervisor tree.
type StorageUsersSutureService struct {
cfg *config.Config
}
// NewStorageUsersSutureService creates a new storage.StorageUsersSutureService
func NewStorageUsers(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return StorageUsersSutureService{
cfg: cfg.Storage,
}
}
func (s StorageUsersSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageUsers.Context = ctx
f := &flag.FlagSet{}
cmdFlags := StorageUsers(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StorageUsers(s.cfg).Before != nil {
if err := StorageUsers(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := StorageUsers(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
+197
View File
@@ -0,0 +1,197 @@
package command
import (
"context"
"flag"
"os"
"path"
"path/filepath"
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
// Users is the entrypoint for the sharing command.
func Users(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "users",
Usage: "start users service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-users")
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// precreate folders
if cfg.Reva.Users.Driver == "json" && cfg.Reva.Users.JSON != "" {
if err := os.MkdirAll(filepath.Dir(cfg.Reva.Users.JSON), os.FileMode(0700)); err != nil {
return err
}
}
uuid := uuid.Must(uuid.NewV4())
pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid")
rcfg := usersConfigFromStruct(c, cfg)
logger.Debug().
Str("server", "users").
Interface("reva-config", rcfg).
Msg("config")
if cfg.Reva.Users.Driver == "ldap" {
if err := waitForLDAPCA(logger, &cfg.Reva.LDAP); err != nil {
logger.Error().Err(err).Msg("The configured LDAP CA cert does not exist")
return err
}
}
gr.Add(func() error {
runtime.RunWithOptions(
rcfg,
pidFile,
runtime.WithLogger(&logger.Logger),
)
return nil
}, func(_ error) {
logger.Info().
Str("server", c.Command.Name).
Msg("Shutting down server")
cancel()
})
debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.Users.DebugAddr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
return err
}
gr.Add(debugServer.ListenAndServe, func(_ error) {
cancel()
})
if !cfg.Reva.Users.Supervised {
sync.Trap(&gr, cancel)
}
return gr.Run()
},
}
}
// usersConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
func usersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.Users.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.Users.GRPCNetwork,
"address": cfg.Reva.Users.GRPCAddr,
// TODO build services dynamically
"services": map[string]interface{}{
"userprovider": map[string]interface{}{
"driver": cfg.Reva.Users.Driver,
"drivers": map[string]interface{}{
"json": map[string]interface{}{
"users": cfg.Reva.Users.JSON,
},
"ldap": ldapConfigFromString(cfg),
"rest": map[string]interface{}{
"client_id": cfg.Reva.UserGroupRest.ClientID,
"client_secret": cfg.Reva.UserGroupRest.ClientSecret,
"redis_address": cfg.Reva.UserGroupRest.RedisAddress,
"redis_username": cfg.Reva.UserGroupRest.RedisUsername,
"redis_password": cfg.Reva.UserGroupRest.RedisPassword,
"user_groups_cache_expiration": cfg.Reva.Users.UserGroupsCacheExpiration,
"id_provider": cfg.Reva.UserGroupRest.IDProvider,
"api_base_url": cfg.Reva.UserGroupRest.APIBaseURL,
"oidc_token_endpoint": cfg.Reva.UserGroupRest.OIDCTokenEndpoint,
"target_api": cfg.Reva.UserGroupRest.TargetAPI,
},
"owncloudsql": map[string]interface{}{
"dbusername": cfg.Reva.UserOwnCloudSQL.DBUsername,
"dbpassword": cfg.Reva.UserOwnCloudSQL.DBPassword,
"dbhost": cfg.Reva.UserOwnCloudSQL.DBHost,
"dbport": cfg.Reva.UserOwnCloudSQL.DBPort,
"dbname": cfg.Reva.UserOwnCloudSQL.DBName,
"idp": cfg.Reva.UserOwnCloudSQL.Idp,
"nobody": cfg.Reva.UserOwnCloudSQL.Nobody,
"join_username": cfg.Reva.UserOwnCloudSQL.JoinUsername,
"join_ownclouduuid": cfg.Reva.UserOwnCloudSQL.JoinOwnCloudUUID,
"enable_medial_search": cfg.Reva.UserOwnCloudSQL.EnableMedialSearch,
},
},
},
},
},
}
return rcfg
}
// UserProviderSutureService allows for the storage-userprovider command to be embedded and supervised by a suture supervisor tree.
type UserProviderSutureService struct {
cfg *config.Config
}
// NewUserProviderSutureService creates a new storage.UserProvider
func NewUserProvider(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
return UserProviderSutureService{
cfg: cfg.Storage,
}
}
func (s UserProviderSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.Users.Context = ctx
f := &flag.FlagSet{}
cmdFlags := Users(s.cfg).Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if Users(s.cfg).Before != nil {
if err := Users(s.cfg).Before(cliCtx); err != nil {
return err
}
}
if err := Users(s.cfg).Action(cliCtx); err != nil {
return err
}
return nil
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,462 @@
package defaults
import (
"os"
"path"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/config/defaults"
)
const (
defaultPublicURL = "https://localhost:9200"
defaultShareFolder = "/Shares"
defaultStorageNamespace = "/users/{{.Id.OpaqueId}}"
defaultGatewayAddr = "127.0.0.1:9142"
defaultUserLayout = "{{.Id.OpaqueId}}"
defaultPersonalSpaceAliasTemplate = "{{.SpaceType}}/{{.User.Username | lower}}"
defaultGeneralSpaceAliasTemplate = "{{.SpaceType}}/{{.SpaceName | replace \" \" \"-\" | lower}}"
)
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
Sanitize(cfg)
return cfg
}
func DefaultConfig() *config.Config {
return &config.Config{
// log is inherited
Debug: config.Debug{
Addr: "127.0.0.1:9109",
},
Reva: config.Reva{
JWTSecret: "Pive-Fumkiu4",
SkipUserGroupsInToken: false,
TransferSecret: "replace-me-with-a-transfer-secret",
TransferExpires: 24 * 60 * 60,
OIDC: config.OIDC{
Issuer: defaultPublicURL,
Insecure: false,
IDClaim: "preferred_username",
},
LDAP: config.LDAP{
URI: "ldaps://localhost:9126",
CACert: path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"),
Insecure: false,
UserBaseDN: "dc=ocis,dc=test",
GroupBaseDN: "dc=ocis,dc=test",
UserScope: "sub",
GroupScope: "sub",
LoginAttributes: []string{"cn", "mail"},
UserFilter: "",
GroupFilter: "",
UserObjectClass: "posixAccount",
GroupObjectClass: "posixGroup",
BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test",
BindPassword: "reva",
IDP: defaultPublicURL,
UserSchema: config.LDAPUserSchema{
ID: "ownclouduuid",
Mail: "mail",
DisplayName: "displayname",
Username: "cn",
UIDNumber: "uidnumber",
GIDNumber: "gidnumber",
},
GroupSchema: config.LDAPGroupSchema{
ID: "cn",
Mail: "mail",
DisplayName: "cn",
Groupname: "cn",
Member: "cn",
GIDNumber: "gidnumber",
},
},
UserGroupRest: config.UserGroupRest{
RedisAddress: "localhost:6379",
},
UserOwnCloudSQL: config.UserOwnCloudSQL{
DBUsername: "owncloud",
DBPassword: "secret",
DBHost: "mysql",
DBPort: 3306,
DBName: "owncloud",
Idp: defaultPublicURL,
Nobody: 90,
JoinUsername: false,
JoinOwnCloudUUID: false,
EnableMedialSearch: false,
},
OCDav: config.OCDav{
WebdavNamespace: defaultStorageNamespace,
DavFilesNamespace: defaultStorageNamespace,
},
Archiver: config.Archiver{
MaxNumFiles: 10000,
MaxSize: 1073741824,
ArchiverURL: "/archiver",
},
UserStorage: config.StorageConfig{
EOS: config.DriverEOS{
DriverCommon: config.DriverCommon{
Root: "/eos/dockertest/reva",
ShareFolder: defaultShareFolder,
UserLayout: "{{substr 0 1 .Username}}/{{.Username}}",
},
ShadowNamespace: "", // Defaults to path.Join(c.Namespace, ".shadow")
UploadsNamespace: "", // Defaults to path.Join(c.Namespace, ".uploads")
EosBinary: "/usr/bin/eos",
XrdcopyBinary: "/usr/bin/xrdcopy",
MasterURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
SlaveURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
CacheDirectory: os.TempDir(),
GatewaySVC: defaultGatewayAddr,
},
Local: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "local", "users"),
ShareFolder: defaultShareFolder,
UserLayout: "{{.Username}}",
EnableHome: false,
},
OwnCloudSQL: config.DriverOwnCloudSQL{
DriverCommon: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "owncloud"),
ShareFolder: defaultShareFolder,
UserLayout: "{{.Username}}",
EnableHome: false,
},
UploadInfoDir: path.Join(defaults.BaseDataPath(), "storage", "uploadinfo"),
DBUsername: "owncloud",
DBPassword: "owncloud",
DBHost: "",
DBPort: 3306,
DBName: "owncloud",
},
S3: config.DriverS3{
DriverCommon: config.DriverCommon{},
Region: "default",
AccessKey: "",
SecretKey: "",
Endpoint: "",
Bucket: "",
},
S3NG: config.DriverS3NG{
DriverCommon: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "users"),
ShareFolder: defaultShareFolder,
UserLayout: defaultUserLayout,
PersonalSpaceAliasTemplate: defaultPersonalSpaceAliasTemplate,
GeneralSpaceAliasTemplate: defaultGeneralSpaceAliasTemplate,
EnableHome: false,
},
Region: "default",
AccessKey: "",
SecretKey: "",
Endpoint: "",
Bucket: "",
},
OCIS: config.DriverOCIS{
DriverCommon: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "users"),
ShareFolder: defaultShareFolder,
UserLayout: defaultUserLayout,
PersonalSpaceAliasTemplate: defaultPersonalSpaceAliasTemplate,
GeneralSpaceAliasTemplate: defaultGeneralSpaceAliasTemplate,
},
},
},
MetadataStorage: config.StorageConfig{
EOS: config.DriverEOS{
DriverCommon: config.DriverCommon{
Root: "/eos/dockertest/reva",
ShareFolder: defaultShareFolder,
UserLayout: "{{substr 0 1 .Username}}/{{.Username}}",
EnableHome: false,
},
ShadowNamespace: "",
UploadsNamespace: "",
EosBinary: "/usr/bin/eos",
XrdcopyBinary: "/usr/bin/xrdcopy",
MasterURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
GrpcURI: "",
SlaveURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
CacheDirectory: os.TempDir(),
EnableLogging: false,
ShowHiddenSysFiles: false,
ForceSingleUserMode: false,
UseKeytab: false,
SecProtocol: "",
Keytab: "",
SingleUsername: "",
GatewaySVC: defaultGatewayAddr,
},
Local: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "local", "metadata"),
},
OwnCloudSQL: config.DriverOwnCloudSQL{},
S3: config.DriverS3{
DriverCommon: config.DriverCommon{},
Region: "default",
},
S3NG: config.DriverS3NG{
DriverCommon: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "metadata"),
ShareFolder: "",
UserLayout: defaultUserLayout,
EnableHome: false,
},
Region: "default",
AccessKey: "",
SecretKey: "",
Endpoint: "",
Bucket: "",
},
OCIS: config.DriverOCIS{
DriverCommon: config.DriverCommon{
Root: path.Join(defaults.BaseDataPath(), "storage", "metadata"),
ShareFolder: "",
UserLayout: defaultUserLayout,
EnableHome: false,
},
},
},
Frontend: config.FrontendPort{
Port: config.Port{
MaxCPUs: "",
LogLevel: "",
GRPCNetwork: "",
GRPCAddr: "",
HTTPNetwork: "tcp",
HTTPAddr: "127.0.0.1:9140",
Protocol: "",
Endpoint: "",
DebugAddr: "127.0.0.1:9141",
Services: []string{"datagateway", "ocdav", "ocs", "appprovider"},
Config: nil,
Context: nil,
Supervised: false,
},
AppProviderInsecure: false,
AppProviderPrefix: "",
ArchiverInsecure: false,
ArchiverPrefix: "archiver",
DatagatewayPrefix: "data",
Favorites: false,
ProjectSpaces: true,
OCDavInsecure: false, // true?
OCDavPrefix: "",
OCSPrefix: "ocs",
OCSSharePrefix: defaultShareFolder,
OCSHomeNamespace: defaultStorageNamespace,
PublicURL: defaultPublicURL,
OCSCacheWarmupDriver: "",
OCSAdditionalInfoAttribute: "{{.Mail}}",
OCSResourceInfoCacheTTL: 0,
Middleware: config.Middleware{},
},
DataGateway: config.DataGatewayPort{
Port: config.Port{},
PublicURL: "",
},
Gateway: config.Gateway{
Port: config.Port{
Endpoint: defaultGatewayAddr,
DebugAddr: "127.0.0.1:9143",
GRPCNetwork: "tcp",
GRPCAddr: defaultGatewayAddr,
},
CommitShareToStorageGrant: true,
CommitShareToStorageRef: true,
DisableHomeCreationOnLogin: true,
ShareFolder: "Shares",
LinkGrants: "",
HomeMapping: "",
EtagCacheTTL: 0,
},
StorageRegistry: config.StorageRegistry{
Driver: "spaces",
HomeProvider: "/home", // unused for spaces, static currently not supported
JSON: "",
},
AppRegistry: config.AppRegistry{
Driver: "static",
MimetypesJSON: "",
},
Users: config.Users{
Port: config.Port{
Endpoint: "localhost:9144",
DebugAddr: "127.0.0.1:9145",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9144",
Services: []string{"userprovider"},
},
Driver: "ldap",
UserGroupsCacheExpiration: 5,
},
Groups: config.Groups{
Port: config.Port{
Endpoint: "localhost:9160",
DebugAddr: "127.0.0.1:9161",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9160",
Services: []string{"groupprovider"},
},
Driver: "ldap",
GroupMembersCacheExpiration: 5,
},
AuthProvider: config.Users{
Port: config.Port{},
Driver: "ldap",
UserGroupsCacheExpiration: 0,
},
AuthBasic: config.Port{
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9146",
DebugAddr: "127.0.0.1:9147",
Services: []string{"authprovider"},
Endpoint: "localhost:9146",
},
AuthBearer: config.Port{
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9148",
DebugAddr: "127.0.0.1:9149",
Services: []string{"authprovider"},
Endpoint: "localhost:9148",
},
AuthMachine: config.Port{
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9166",
DebugAddr: "127.0.0.1:9167",
Services: []string{"authprovider"},
Endpoint: "localhost:9166",
},
AuthMachineConfig: config.AuthMachineConfig{
MachineAuthAPIKey: "change-me-please",
},
Sharing: config.Sharing{
Port: config.Port{
Endpoint: "localhost:9150",
DebugAddr: "127.0.0.1:9151",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9150",
Services: []string{"usershareprovider", "publicshareprovider"},
},
CS3ProviderAddr: "127.0.0.1:9215",
CS3ServiceUser: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
CS3ServiceUserIdp: "https://localhost:9200",
UserDriver: "json",
UserJSONFile: path.Join(defaults.BaseDataPath(), "storage", "shares.json"),
UserSQLUsername: "",
UserSQLPassword: "",
UserSQLHost: "",
UserSQLPort: 1433,
UserSQLName: "",
PublicDriver: "json",
PublicJSONFile: path.Join(defaults.BaseDataPath(), "storage", "publicshares.json"),
PublicPasswordHashCost: 11,
PublicEnableExpiredSharesCleanup: true,
PublicJanitorRunInterval: 60,
UserStorageMountID: "",
Events: config.Events{
Address: "127.0.0.1:9233",
ClusterID: "ocis-cluster",
},
},
StorageShares: config.StoragePort{
Port: config.Port{
Endpoint: "localhost:9154",
DebugAddr: "127.0.0.1:9156",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9154",
HTTPNetwork: "tcp",
HTTPAddr: "127.0.0.1:9155",
},
ReadOnly: false,
AlternativeID: "1284d238-aa92-42ce-bdc4-0b0000009154",
MountID: "1284d238-aa92-42ce-bdc4-0b0000009157",
},
StorageUsers: config.StoragePort{
Port: config.Port{
Endpoint: "localhost:9157",
DebugAddr: "127.0.0.1:9159",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9157",
HTTPNetwork: "tcp",
HTTPAddr: "127.0.0.1:9158",
},
MountID: "1284d238-aa92-42ce-bdc4-0b0000009157",
Driver: "ocis",
DataServerURL: "http://localhost:9158/data",
HTTPPrefix: "data",
TempFolder: path.Join(defaults.BaseDataPath(), "tmp", "users"),
},
StoragePublicLink: config.PublicStorage{
StoragePort: config.StoragePort{
Port: config.Port{
Endpoint: "localhost:9178",
DebugAddr: "127.0.0.1:9179",
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9178",
},
MountID: "7993447f-687f-490d-875c-ac95e89a62a4",
},
PublicShareProviderAddr: "",
UserProviderAddr: "",
},
StorageMetadata: config.StoragePort{
Port: config.Port{
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9215",
HTTPNetwork: "tcp",
HTTPAddr: "127.0.0.1:9216",
DebugAddr: "127.0.0.1:9217",
},
Driver: "ocis",
ExposeDataServer: false,
DataServerURL: "http://localhost:9216/data",
TempFolder: path.Join(defaults.BaseDataPath(), "tmp", "metadata"),
DataProvider: config.DataProvider{},
},
AppProvider: config.AppProvider{
Port: config.Port{
GRPCNetwork: "tcp",
GRPCAddr: "127.0.0.1:9164",
DebugAddr: "127.0.0.1:9165",
Endpoint: "localhost:9164",
Services: []string{"appprovider"},
},
ExternalAddr: "127.0.0.1:9164",
WopiDriver: config.WopiDriver{},
AppsURL: "/app/list",
OpenURL: "/app/open",
NewURL: "/app/new",
},
Permissions: config.Port{
Endpoint: "localhost:9191",
},
Configs: nil,
UploadMaxChunkSize: 1e+8,
UploadHTTPMethodOverride: "",
ChecksumSupportedTypes: []string{"sha1", "md5", "adler32"},
ChecksumPreferredUploadType: "",
DefaultUploadProtocol: "tus",
},
Tracing: config.Tracing{
Service: "storage",
Type: "jaeger",
},
Asset: config.Asset{},
}
}
func EnsureDefaults(cfg *config.Config) {
// TODO: IMPLEMENT ME!
}
func Sanitize(cfg *config.Config) {
// TODO: IMPLEMENT ME!
}
@@ -0,0 +1,66 @@
package debug
import (
"context"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Addr string
Logger log.Logger
Context context.Context
Config *config.Config
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Name provides a function to set the name option.
func Name(val string) Option {
return func(o *Options) {
o.Name = val
}
}
// Addr provides a function to set the addr option.
func Addr(val string) Option {
return func(o *Options) {
o.Addr = val
}
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
@@ -0,0 +1,59 @@
package debug
import (
"io"
"net/http"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/service/debug"
"github.com/owncloud/ocis/ocis-pkg/version"
)
// Server initializes the debug service and server.
func Server(opts ...Option) (*http.Server, error) {
options := newOptions(opts...)
return debug.NewService(
debug.Logger(options.Logger),
debug.Name(options.Name),
debug.Version(version.String),
debug.Address(options.Addr),
debug.Token(options.Config.Debug.Token),
debug.Pprof(options.Config.Debug.Pprof),
debug.Zpages(options.Config.Debug.Zpages),
debug.Health(health(options.Config)),
debug.Ready(ready(options.Config)),
), nil
}
// health implements the health check.
func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
// TODO: check if services are up and running
_, err := io.WriteString(w, http.StatusText(http.StatusOK))
// io.WriteString should not fail but if it does we want to know.
if err != nil {
panic(err)
}
}
}
// ready implements the ready check.
func ready(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
// TODO: check if services are up and running
_, err := io.WriteString(w, http.StatusText(http.StatusOK))
// io.WriteString should not fail but if it does we want to know.
if err != nil {
panic(err)
}
}
}
+68
View File
@@ -0,0 +1,68 @@
package external
import (
"context"
"time"
"github.com/owncloud/ocis/ocis-pkg/log"
oregistry "github.com/owncloud/ocis/ocis-pkg/registry"
"go-micro.dev/v4/broker"
"go-micro.dev/v4/registry"
)
// RegisterGRPCEndpoint publishes an arbitrary endpoint to the service-registry. This allows to query nodes of
// non-micro GRPC-services like reva. No health-checks are done, thus the caller is responsible for canceling.
//
func RegisterGRPCEndpoint(ctx context.Context, serviceID, uuid, addr string, version string, logger log.Logger) error {
node := &registry.Node{
Id: serviceID + "-" + uuid,
Address: addr,
Metadata: make(map[string]string),
}
ocisRegistry := oregistry.GetRegistry()
node.Metadata["broker"] = broker.String()
node.Metadata["registry"] = ocisRegistry.String()
node.Metadata["server"] = "grpc"
node.Metadata["transport"] = "grpc"
node.Metadata["protocol"] = "grpc"
service := &registry.Service{
Name: serviceID,
Version: version,
Nodes: []*registry.Node{node},
Endpoints: make([]*registry.Endpoint, 0),
}
logger.Info().Msgf("registering external service %v@%v", node.Id, node.Address)
rOpts := []registry.RegisterOption{registry.RegisterTTL(time.Minute)}
if err := ocisRegistry.Register(service, rOpts...); err != nil {
logger.Fatal().Err(err).Msgf("Registration error for external service %v", serviceID)
}
t := time.NewTicker(time.Second * 30)
go func() {
for {
select {
case <-t.C:
logger.Debug().Interface("service", service).Msg("refreshing external service-registration")
err := ocisRegistry.Register(service, rOpts...)
if err != nil {
logger.Error().Err(err).Msgf("registration error for external service %v", serviceID)
}
case <-ctx.Done():
logger.Debug().Interface("service", service).Msg("unregistering")
t.Stop()
err := ocisRegistry.Deregister(service)
if err != nil {
logger.Err(err).Msgf("Error unregistering external service %v", serviceID)
}
}
}
}()
return nil
}
@@ -0,0 +1,61 @@
package external
//
//import (
// "context"
// "testing"
//
// "github.com/micro/go-micro/v2/registry"
// "github.com/owncloud/ocis/ocis-pkg/log"
//)
//
//func TestRegisterGRPCEndpoint(t *testing.T) {
// ctx, cancel := context.WithCancel(context.Background())
// err := RegisterGRPCEndpoint(ctx, "test", "1234", "192.168.0.1:777", log.Logger{})
// if err != nil {
// t.Errorf("Unexpected error: %v", err)
// }
//
// s, err := registry.GetService("test")
// if err != nil {
// t.Errorf("Unexpected error: %v", err)
// }
//
// if len(s) != 1 {
// t.Errorf("Expected exactly one service to be returned got %v", len(s))
// }
//
// if len(s[0].Nodes) != 1 {
// t.Errorf("Expected exactly one node to be returned got %v", len(s[0].Nodes))
// }
//
// testSvc := s[0]
// if testSvc.Name != "test" {
// t.Errorf("Expected service name to be 'test' got %v", s[0].Name)
// }
//
// testNode := testSvc.Nodes[0]
//
// if testNode.Address != "192.168.0.1:777" {
// t.Errorf("Expected node address to be '192.168.0.1:777' got %v", testNode.Address)
// }
//
// if testNode.Id != "test-1234" {
// t.Errorf("Expected node id to be 'test-1234' got %v", testNode.Id)
// }
//
// cancel()
//
// // When switching over to monorepo this little test fails. We're unsure of what the cause is, but since this test
// // is testing a framework specific behavior, we're better off letting it commented out. There is also no use of
// // com.owncloud.reva anywhere in the codebase, so we're effectively only registering reva as a go-micro service,
// // but not sending any message.
// s, err = registry.GetService("test")
// if err != nil {
// t.Errorf("Unexpected error: %v", err)
// }
//
// if len(s) != 0 {
// t.Errorf("Deregister on cancelation failed. Result-length should be zero, got %v", len(s))
// }
//}
+38
View File
@@ -0,0 +1,38 @@
package tracing
import (
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// Configure for Reva serves only as informational / instructive log messages. Tracing config will be delegated directly
// to Reva services.
func Configure(cfg *config.Config, logger log.Logger) {
if cfg.Tracing.Enabled {
switch t := cfg.Tracing.Type; t {
case "agent":
logger.Error().
Str("type", t).
Msg("Reva only supports the jaeger tracing backend")
case "jaeger":
logger.Info().
Str("type", t).
Msg("configuring storage to use the jaeger tracing backend")
case "zipkin":
logger.Error().
Str("type", t).
Msg("Reva only supports the jaeger tracing backend")
default:
logger.Warn().
Str("type", t).
Msg("Unknown tracing backend")
}
} else {
logger.Debug().
Msg("Tracing is not enabled")
}
}