From 5013d71dac1527406534807a2a9215634b08b047 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Wed, 20 Apr 2022 16:30:32 +0200 Subject: [PATCH] make sharing config similar to other services --- extensions/sharing/pkg/command/command.go | 237 ++++++++++++++++++ extensions/sharing/pkg/config/config.go | 115 +++++++++ .../pkg/config/defaults/defaultconfig.go | 107 ++++++++ extensions/storage/pkg/command/root.go | 2 +- ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/command/storagesharing.go | 4 +- ocis/pkg/runtime/service/service.go | 3 +- 8 files changed, 468 insertions(+), 4 deletions(-) create mode 100644 extensions/sharing/pkg/command/command.go create mode 100644 extensions/sharing/pkg/config/config.go create mode 100644 extensions/sharing/pkg/config/defaults/defaultconfig.go diff --git a/extensions/sharing/pkg/command/command.go b/extensions/sharing/pkg/command/command.go new file mode 100644 index 000000000..614e3e87c --- /dev/null +++ b/extensions/sharing/pkg/command/command.go @@ -0,0 +1,237 @@ +package command + +import ( + "context" + "flag" + "os" + "path" + "path/filepath" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/ocis-pkg/sync" + "github.com/owncloud/ocis/ocis-pkg/tracing" + + "github.com/cs3org/reva/v2/cmd/revad/runtime" + "github.com/gofrs/uuid" + "github.com/oklog/run" + "github.com/owncloud/ocis/extensions/sharing/pkg/config" + "github.com/owncloud/ocis/extensions/storage/pkg/server/debug" + ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/thejerf/suture/v4" + "github.com/urfave/cli/v2" +) + +// Sharing is the entrypoint for the sharing command. +func Sharing(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "sharing", + Usage: "start sharing service", + Action: func(c *cli.Context) error { + logCfg := cfg.Logging + logger := log.NewLogger( + log.Level(logCfg.Level), + log.File(logCfg.File), + log.Pretty(logCfg.Pretty), + log.Color(logCfg.Color), + ) + tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger) + gr := run.Group{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // precreate folders + if cfg.UserSharingDriver == "json" && cfg.UserSharingDrivers.JSON.File != "" { + if err := os.MkdirAll(filepath.Dir(cfg.UserSharingDrivers.JSON.File), os.FileMode(0700)); err != nil { + return err + } + } + if cfg.PublicSharingDriver == "json" && cfg.PublicSharingDrivers.JSON.File != "" { + if err := os.MkdirAll(filepath.Dir(cfg.PublicSharingDrivers.JSON.File), os.FileMode(0700)); err != nil { + return err + } + } + + uuid := uuid.Must(uuid.NewV4()) + pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.String()+".pid") + + rcfg := sharingConfigFromStruct(c, cfg) + + gr.Add(func() error { + runtime.RunWithOptions( + rcfg, + pidFile, + runtime.WithLogger(&logger.Logger), + ) + return nil + }, func(_ error) { + logger.Info(). + Str("server", c.Command.Name). + Msg("Shutting down server") + + cancel() + }) + + debug, err := debug.Server( + debug.Name(c.Command.Name+"-debug"), + debug.Addr(cfg.Debug.Addr), + debug.Logger(logger), + debug.Context(ctx), + debug.Pprof(cfg.Debug.Pprof), + debug.Zpages(cfg.Debug.Zpages), + debug.Token(cfg.Debug.Token), + ) + + if err != nil { + logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server") + return err + } + + gr.Add(debug.ListenAndServe, func(_ error) { + cancel() + }) + + if !cfg.Supervised { + sync.Trap(&gr, cancel) + } + + return gr.Run() + }, + } +} + +// sharingConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service. +func sharingConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} { + rcfg := map[string]interface{}{ + "core": map[string]interface{}{ + "tracing_enabled": cfg.Tracing.Enabled, + "tracing_endpoint": cfg.Tracing.Endpoint, + "tracing_collector": cfg.Tracing.Collector, + "tracing_service_name": c.Command.Name, + }, + "shared": map[string]interface{}{ + "jwt_secret": cfg.JWTSecret, + "gatewaysvc": cfg.GatewayEndpoint, + "skip_user_groups_in_token": cfg.SkipUserGroupsInToken, + }, + "grpc": map[string]interface{}{ + "network": cfg.GRPC.Protocol, + "address": cfg.GRPC.Addr, + // TODO build services dynamically + "services": map[string]interface{}{ + "usershareprovider": map[string]interface{}{ + "driver": cfg.UserSharingDriver, + "drivers": map[string]interface{}{ + "json": map[string]interface{}{ + "file": cfg.UserSharingDrivers.JSON.File, + "gateway_addr": cfg.GatewayEndpoint, + }, + "sql": map[string]interface{}{ // cernbox sql + "db_username": cfg.UserSharingDrivers.SQL.DBUsername, + "db_password": cfg.UserSharingDrivers.SQL.DBPassword, + "db_host": cfg.UserSharingDrivers.SQL.DBHost, + "db_port": cfg.UserSharingDrivers.SQL.DBPort, + "db_name": cfg.UserSharingDrivers.SQL.DBName, + "password_hash_cost": cfg.UserSharingDrivers.SQL.PasswordHashCost, + "enable_expired_shares_cleanup": cfg.UserSharingDrivers.SQL.EnableExpiredSharesCleanup, + "janitor_run_interval": cfg.UserSharingDrivers.SQL.JanitorRunInterval, + }, + "oc10-sql": map[string]interface{}{ + "storage_mount_id": cfg.UserSharingDrivers.SQL.UserStorageMountID, + "db_username": cfg.UserSharingDrivers.SQL.DBUsername, + "db_password": cfg.UserSharingDrivers.SQL.DBPassword, + "db_host": cfg.UserSharingDrivers.SQL.DBHost, + "db_port": cfg.UserSharingDrivers.SQL.DBPort, + "db_name": cfg.UserSharingDrivers.SQL.DBName, + }, + "cs3": map[string]interface{}{ + "provider_addr": cfg.UserSharingDrivers.CS3.ProviderAddr, + "service_user_id": cfg.UserSharingDrivers.CS3.ServiceUserID, + "service_user_idp": cfg.UserSharingDrivers.CS3.ServiceUserIDP, + "machine_auth_apikey": cfg.UserSharingDrivers.CS3.MachineAuthAPIKey, + }, + }, + }, + "publicshareprovider": map[string]interface{}{ + "driver": cfg.PublicSharingDriver, + "drivers": map[string]interface{}{ + "json": map[string]interface{}{ + "file": cfg.PublicSharingDrivers.JSON.File, + }, + "sql": map[string]interface{}{ + "db_username": cfg.PublicSharingDrivers.SQL.DBUsername, + "db_password": cfg.PublicSharingDrivers.SQL.DBPassword, + "db_host": cfg.PublicSharingDrivers.SQL.DBHost, + "db_port": cfg.PublicSharingDrivers.SQL.DBPort, + "db_name": cfg.PublicSharingDrivers.SQL.DBName, + "password_hash_cost": cfg.PublicSharingDrivers.SQL.PasswordHashCost, + "enable_expired_shares_cleanup": cfg.PublicSharingDrivers.SQL.EnableExpiredSharesCleanup, + "janitor_run_interval": cfg.PublicSharingDrivers.SQL.JanitorRunInterval, + }, + "oc10-sql": map[string]interface{}{ + "storage_mount_id": cfg.PublicSharingDrivers.SQL.UserStorageMountID, + "db_username": cfg.PublicSharingDrivers.SQL.DBUsername, + "db_password": cfg.PublicSharingDrivers.SQL.DBPassword, + "db_host": cfg.PublicSharingDrivers.SQL.DBHost, + "db_port": cfg.PublicSharingDrivers.SQL.DBPort, + "db_name": cfg.PublicSharingDrivers.SQL.DBName, + "password_hash_cost": cfg.PublicSharingDrivers.SQL.PasswordHashCost, + "enable_expired_shares_cleanup": cfg.PublicSharingDrivers.SQL.EnableExpiredSharesCleanup, + "janitor_run_interval": cfg.PublicSharingDrivers.SQL.JanitorRunInterval, + }, + "cs3": map[string]interface{}{ + "provider_addr": cfg.PublicSharingDrivers.CS3.ProviderAddr, + "service_user_id": cfg.PublicSharingDrivers.CS3.ServiceUserID, + "service_user_idp": cfg.PublicSharingDrivers.CS3.ServiceUserIDP, + "machine_auth_apikey": cfg.PublicSharingDrivers.CS3.MachineAuthAPIKey, + }, + }, + }, + }, + "interceptors": map[string]interface{}{ + "eventsmiddleware": map[string]interface{}{ + "group": "sharing", + "type": "nats", + "address": cfg.Events.Addr, + "clusterID": cfg.Events.ClusterID, + }, + }, + }, + } + return rcfg +} + +// SharingSutureService allows for the storage-sharing command to be embedded and supervised by a suture supervisor tree. +type SharingSutureService struct { + cfg *config.Config +} + +// NewSharingSutureService creates a new store.SharingSutureService +func NewSharing(cfg *ociscfg.Config) suture.Service { + cfg.Sharing.Commons = cfg.Commons + return SharingSutureService{ + cfg: cfg.Sharing, + } +} + +func (s SharingSutureService) Serve(ctx context.Context) error { + // s.cfg.Reva.Sharing.Context = ctx + cmd := Sharing(s.cfg) + f := &flag.FlagSet{} + cmdFlags := cmd.Flags + for k := range cmdFlags { + if err := cmdFlags[k].Apply(f); err != nil { + return err + } + } + cliCtx := cli.NewContext(nil, f, nil) + if cmd.Before != nil { + if err := cmd.Before(cliCtx); err != nil { + return err + } + } + if err := cmd.Action(cliCtx); err != nil { + return err + } + + return nil +} diff --git a/extensions/sharing/pkg/config/config.go b/extensions/sharing/pkg/config/config.go new file mode 100644 index 000000000..760848cff --- /dev/null +++ b/extensions/sharing/pkg/config/config.go @@ -0,0 +1,115 @@ +package config + +import "github.com/owncloud/ocis/ocis-pkg/shared" + +type Config struct { + *shared.Commons `yaml:"-"` + Service Service `yaml:"-"` + Tracing *Tracing `yaml:"tracing"` + Logging *Logging `yaml:"log"` + Debug Debug `yaml:"debug"` + Supervised bool + + GRPC GRPCConfig `yaml:"grpc"` + + JWTSecret string + GatewayEndpoint string + SkipUserGroupsInToken bool + UserSharingDriver string + UserSharingDrivers UserSharingDrivers + PublicSharingDriver string + PublicSharingDrivers PublicSharingDrivers + Events Events +} +type Tracing struct { + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;SHARING_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;SHARING_TRACING_TYPE"` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;SHARING_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;SHARING_TRACING_COLLECTOR"` +} + +type Logging struct { + Level string `yaml:"level" env:"OCIS_LOG_LEVEL;SHARING_LOG_LEVEL" desc:"The log level."` + Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;SHARING_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `yaml:"color" env:"OCIS_LOG_COLOR;SHARING_LOG_COLOR" desc:"Activates colorized log output."` + File string `yaml:"file" env:"OCIS_LOG_FILE;SHARING_LOG_FILE" desc:"The target log file."` +} + +type Service struct { + Name string `yaml:"-"` +} + +type Debug struct { + Addr string `yaml:"addr" env:"SHARING_DEBUG_ADDR"` + Token string `yaml:"token" env:"SHARING_DEBUG_TOKEN"` + Pprof bool `yaml:"pprof" env:"SHARING_DEBUG_PPROF"` + Zpages bool `yaml:"zpages" env:"SHARING_DEBUG_ZPAGES"` +} + +type GRPCConfig struct { + Addr string `yaml:"addr" env:"SHARING_GRPC_ADDR" desc:"The address of the grpc service."` + Protocol string `yaml:"protocol" env:"SHARING_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` +} + +type UserSharingDrivers struct { + JSON UserSharingJSONDriver + SQL UserSharingSQLDriver + CS3 UserSharingCS3Driver +} + +type UserSharingJSONDriver struct { + File string +} + +type UserSharingSQLDriver struct { + DBUsername string + DBPassword string + DBHost string + DBPort int + DBName string + PasswordHashCost int + EnableExpiredSharesCleanup bool + JanitorRunInterval int + UserStorageMountID string +} + +type UserSharingCS3Driver struct { + ProviderAddr string + ServiceUserID string + ServiceUserIDP string + MachineAuthAPIKey string +} + +type PublicSharingDrivers struct { + JSON PublicSharingJSONDriver + SQL PublicSharingSQLDriver + CS3 PublicSharingCS3Driver +} + +type PublicSharingJSONDriver struct { + File string +} + +type PublicSharingSQLDriver struct { + DBUsername string + DBPassword string + DBHost string + DBPort int + DBName string + PasswordHashCost int + EnableExpiredSharesCleanup bool + JanitorRunInterval int + UserStorageMountID string +} + +type PublicSharingCS3Driver struct { + ProviderAddr string + ServiceUserID string + ServiceUserIDP string + MachineAuthAPIKey string +} + +type Events struct { + Addr string + ClusterID string +} diff --git a/extensions/sharing/pkg/config/defaults/defaultconfig.go b/extensions/sharing/pkg/config/defaults/defaultconfig.go new file mode 100644 index 000000000..deed960ee --- /dev/null +++ b/extensions/sharing/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,107 @@ +package defaults + +import ( + "path/filepath" + + "github.com/owncloud/ocis/extensions/sharing/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/defaults" +) + +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + + return cfg +} + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9151", + Token: "", + Pprof: false, + Zpages: false, + }, + GRPC: config.GRPCConfig{ + Addr: "127.0.0.1:9150", + Protocol: "tcp", + }, + Service: config.Service{ + Name: "sharing", + }, + GatewayEndpoint: "127.0.0.1:9142", + JWTSecret: "Pive-Fumkiu4", + UserSharingDriver: "json", + UserSharingDrivers: config.UserSharingDrivers{ + JSON: config.UserSharingJSONDriver{ + File: filepath.Join(defaults.BaseDataPath(), "storage", "shares.json"), + }, + SQL: config.UserSharingSQLDriver{ + DBUsername: "", + DBPassword: "", + DBHost: "", + DBPort: 1433, + DBName: "", + PasswordHashCost: 11, + EnableExpiredSharesCleanup: true, + JanitorRunInterval: 60, + }, + CS3: config.UserSharingCS3Driver{ + ProviderAddr: "127.0.0.1:9215", + ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + ServiceUserIDP: "https://localhost:9200", + }, + }, + PublicSharingDriver: "json", + PublicSharingDrivers: config.PublicSharingDrivers{ + JSON: config.PublicSharingJSONDriver{ + File: filepath.Join(defaults.BaseDataPath(), "storage", "publicshares.json"), + }, + SQL: config.PublicSharingSQLDriver{ + DBUsername: "", + DBPassword: "", + DBHost: "", + DBPort: 1433, + DBName: "", + PasswordHashCost: 11, + EnableExpiredSharesCleanup: true, + JanitorRunInterval: 60, + }, + CS3: config.PublicSharingCS3Driver{ + ProviderAddr: "127.0.0.1:9215", + ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + ServiceUserIDP: "https://localhost:9200", + }, + }, + Events: config.Events{ + Addr: "127.0.0.1:9233", + ClusterID: "ocis-cluster", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Logging == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Logging = &config.Logging{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Logging == nil { + cfg.Logging = &config.Logging{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} diff --git a/extensions/storage/pkg/command/root.go b/extensions/storage/pkg/command/root.go index 5cc4afb3e..180f2e3fc 100644 --- a/extensions/storage/pkg/command/root.go +++ b/extensions/storage/pkg/command/root.go @@ -20,7 +20,7 @@ func GetCommands(cfg *config.Config) cli.Commands { // AuthBasic(cfg), // AuthBearer(cfg), AuthMachine(cfg), - Sharing(cfg), + // Sharing(cfg), StoragePublicLink(cfg), StorageShares(cfg), StorageUsers(cfg), diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 75b342a11..47e8d6026 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -19,6 +19,7 @@ import ( ocs "github.com/owncloud/ocis/extensions/ocs/pkg/config" proxy "github.com/owncloud/ocis/extensions/proxy/pkg/config" settings "github.com/owncloud/ocis/extensions/settings/pkg/config" + sharing "github.com/owncloud/ocis/extensions/sharing/pkg/config" storage "github.com/owncloud/ocis/extensions/storage/pkg/config" store "github.com/owncloud/ocis/extensions/store/pkg/config" thumbnails "github.com/owncloud/ocis/extensions/thumbnails/pkg/config" @@ -83,6 +84,7 @@ type Config struct { User *user.Config `yaml:"user"` Group *group.Config `yaml:"group"` AppProvider *appprovider.Config `yaml:"app_provider"` + Sharing *sharing.Config `yaml:"app_provider"` Store *store.Config `yaml:"store"` Thumbnails *thumbnails.Config `yaml:"thumbnails"` WebDAV *webdav.Config `yaml:"webdav"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index dbf684f6d..ba07fe766 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -17,6 +17,7 @@ import ( ocs "github.com/owncloud/ocis/extensions/ocs/pkg/config/defaults" proxy "github.com/owncloud/ocis/extensions/proxy/pkg/config/defaults" settings "github.com/owncloud/ocis/extensions/settings/pkg/config/defaults" + sharing "github.com/owncloud/ocis/extensions/sharing/pkg/config/defaults" storage "github.com/owncloud/ocis/extensions/storage/pkg/config/defaults" store "github.com/owncloud/ocis/extensions/store/pkg/config/defaults" thumbnails "github.com/owncloud/ocis/extensions/thumbnails/pkg/config/defaults" @@ -55,6 +56,7 @@ func DefaultConfig() *Config { AuthBearer: authbearer.FullDefaultConfig(), User: user.FullDefaultConfig(), Group: group.FullDefaultConfig(), + Sharing: sharing.FullDefaultConfig(), AppProvider: appprovider.FullDefaultConfig(), } } diff --git a/ocis/pkg/command/storagesharing.go b/ocis/pkg/command/storagesharing.go index 4a70baa80..84d8b7b1d 100644 --- a/ocis/pkg/command/storagesharing.go +++ b/ocis/pkg/command/storagesharing.go @@ -1,7 +1,7 @@ package command import ( - "github.com/owncloud/ocis/extensions/storage/pkg/command" + "github.com/owncloud/ocis/extensions/sharing/pkg/command" "github.com/owncloud/ocis/ocis-pkg/config" "github.com/owncloud/ocis/ocis/pkg/register" "github.com/urfave/cli/v2" @@ -18,7 +18,7 @@ func StorageSharingCommand(cfg *config.Config) *cli.Command { return ParseStorageCommon(ctx, cfg) }, Action: func(c *cli.Context) error { - origCmd := command.Sharing(cfg.Storage) + origCmd := command.Sharing(cfg.Sharing) return handleOriginalAction(c, origCmd) }, } diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 689882c7c..8760e18e1 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -34,6 +34,7 @@ import ( ocs "github.com/owncloud/ocis/extensions/ocs/pkg/command" proxy "github.com/owncloud/ocis/extensions/proxy/pkg/command" settings "github.com/owncloud/ocis/extensions/settings/pkg/command" + sharing "github.com/owncloud/ocis/extensions/sharing/pkg/command" storage "github.com/owncloud/ocis/extensions/storage/pkg/command" store "github.com/owncloud/ocis/extensions/store/pkg/command" thumbnails "github.com/owncloud/ocis/extensions/thumbnails/pkg/command" @@ -126,7 +127,7 @@ func NewService(options ...Option) (*Service, error) { s.ServicesRegistry["notifications"] = notifications.NewSutureService // populate delayed services - s.Delayed["storage-sharing"] = storage.NewSharing + s.Delayed["storage-sharing"] = sharing.NewSharing s.Delayed["accounts"] = accounts.NewSutureService s.Delayed["proxy"] = proxy.NewSutureService