sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make storage metadata config similar to other services

Browse Source
This commit is contained in:
David Christofas
2022-04-25 15:43:43 +02:00
parent 5013d71dac
commit d303bb6f55
9 changed files with 424 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/storage-metadata/pkg/config/config.go
+147
View File
@@ -0,0 +1,147 @@
package config
import (
"context"
"github.com/owncloud/ocis/ocis-pkg/shared"
)
type Config struct {
*shared.Commons `yaml:"-"`
Service Service `yaml:"-"`
Tracing *Tracing `yaml:"tracing"`
Logging *Logging `yaml:"log"`
Debug Debug `yaml:"debug"`
Supervised bool
GRPC GRPCConfig `yaml:"grpc"`
HTTP HTTPConfig `yaml:"http"`
Context context.Context
JWTSecret string
GatewayEndpoint string
SkipUserGroupsInToken bool
Driver string `yaml:"driver" env:"STORAGE_METADATA_AUTH_PROVIDER" desc:"The auth provider which should be used by the service"`
Drivers Drivers `yaml:"drivers"`
DataServerURL string
TempFolder string
DataProviderInsecure bool
}
type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
}
type Logging struct {
Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
}
type Service struct {
Name string `yaml:"-"`
}
type Debug struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
Token string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
Pprof bool `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
Zpages bool `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
}
type GRPCConfig struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}
type HTTPConfig struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}
type Drivers struct {
EOS EOSDriver
Local LocalDriver
OCIS OCISDriver
S3 S3Driver
S3NG S3NGDriver
}
type EOSDriver struct {
// Root is the absolute path to the location of the data
Root string `yaml:"root"`
// ShadowNamespace for storing shadow data
ShadowNamespace string `yaml:"shadow_namespace"`
// UploadsNamespace for storing upload data
UploadsNamespace string `yaml:"uploads_namespace"`
// Location of the eos binary.
// Default is /usr/bin/eos.
EosBinary string `yaml:"eos_binary"`
// Location of the xrdcopy binary.
// Default is /usr/bin/xrdcopy.
XrdcopyBinary string `yaml:"xrd_copy_binary"`
// URL of the Master EOS MGM.
// Default is root://eos-example.org
MasterURL string `yaml:"master_url"`
// URL of the Slave EOS MGM.
// Default is root://eos-example.org
SlaveURL string `yaml:"slave_url"`
// Location on the local fs where to store reads.
// Defaults to os.TempDir()
CacheDirectory string `yaml:"cache_directory"`
// SecProtocol specifies the xrootd security protocol to use between the server and EOS.
SecProtocol string `yaml:"sec_protocol"`
// Keytab specifies the location of the keytab to use to authenticate to EOS.
Keytab string `yaml:"keytab"`
// SingleUsername is the username to use when SingleUserMode is enabled
SingleUsername string `yaml:"single_username"`
// Enables logging of the commands executed
// Defaults to false
EnableLogging bool `yaml:"enable_logging"`
// ShowHiddenSysFiles shows internal EOS files like
// .sys.v# and .sys.a# files.
ShowHiddenSysFiles bool `yaml:"shadow_hidden_files"`
// ForceSingleUserMode will force connections to EOS to use SingleUsername
ForceSingleUserMode bool `yaml:"force_single_user_mode"`
// UseKeyTabAuth changes will authenticate requests by using an EOS keytab.
UseKeytab bool `yaml:"user_keytab"`
// gateway service to use for uid lookups
GatewaySVC string `yaml:"gateway_svc"`
GRPCURI string
UserLayout string
}
type LocalDriver struct {
// Root is the absolute path to the location of the data
Root string `yaml:"root"`
}
type OCISDriver struct {
// Root is the absolute path to the location of the data
Root string `yaml:"root"`
UserLayout string
PermissionsEndpoint string
}
type S3Driver struct {
Region string `yaml:"region"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
}
type S3NGDriver struct {
// Root is the absolute path to the location of the data
Root string `yaml:"root"`
UserLayout string
PermissionsEndpoint string
Region string `yaml:"region"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
}
extensions/storage-metadata/pkg/config/defaults/defaultconfig.go
+106
View File
@@ -0,0 +1,106 @@
package defaults
import (
"os"
"path/filepath"
"github.com/owncloud/ocis/extensions/storage-metadata/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/config/defaults"
)
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
return cfg
}
func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9217",
Token: "",
Pprof: false,
Zpages: false,
},
GRPC: config.GRPCConfig{
Addr: "127.0.0.1:9215",
Protocol: "tcp",
},
HTTP: config.HTTPConfig{
Addr: "127.0.0.1:9216",
Protocol: "tcp",
},
Service: config.Service{
Name: "storage-metadata",
},
GatewayEndpoint: "127.0.0.1:9142",
JWTSecret: "Pive-Fumkiu4",
TempFolder: filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"),
DataServerURL: "http://localhost:9216/data",
Driver: "ocis",
Drivers: config.Drivers{
EOS: config.EOSDriver{
Root: "/eos/dockertest/reva",
UserLayout: "{{substr 0 1 .Username}}/{{.Username}}",
ShadowNamespace: "",
UploadsNamespace: "",
EosBinary: "/usr/bin/eos",
XrdcopyBinary: "/usr/bin/xrdcopy",
MasterURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
GRPCURI: "",
SlaveURL: "root://eos-mgm1.eoscluster.cern.ch:1094",
CacheDirectory: os.TempDir(),
EnableLogging: false,
ShowHiddenSysFiles: false,
ForceSingleUserMode: false,
UseKeytab: false,
SecProtocol: "",
Keytab: "",
SingleUsername: "",
GatewaySVC: "127.0.0.1:9142",
},
Local: config.LocalDriver{
Root: filepath.Join(defaults.BaseDataPath(), "storage", "local", "metadata"),
},
S3: config.S3Driver{
Region: "default",
},
S3NG: config.S3NGDriver{
Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
UserLayout: "{{.Id.OpaqueId}}",
Region: "default",
},
OCIS: config.OCISDriver{
Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
UserLayout: "{{.Id.OpaqueId}}",
},
},
}
}
func EnsureDefaults(cfg *config.Config) {
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Logging == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Logging = &config.Logging{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Logging == nil {
cfg.Logging = &config.Logging{}
}
// provide with defaults for shared tracing, since we need a valid destination address for BindEnv.
if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil {
cfg.Tracing = &config.Tracing{
Enabled: cfg.Commons.Tracing.Enabled,
Type: cfg.Commons.Tracing.Type,
Endpoint: cfg.Commons.Tracing.Endpoint,
Collector: cfg.Commons.Tracing.Collector,
}
} else if cfg.Tracing == nil {
cfg.Tracing = &config.Tracing{}
}
}
extensions/storage-metadata/pkg/config/metadata.go
+75
View File
@@ -0,0 +1,75 @@
package config
func MetadataDrivers(cfg *Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Drivers.EOS.Root,
"shadow_namespace": cfg.Drivers.EOS.ShadowNamespace,
"uploads_namespace": cfg.Drivers.EOS.UploadsNamespace,
"eos_binary": cfg.Drivers.EOS.EosBinary,
"xrdcopy_binary": cfg.Drivers.EOS.XrdcopyBinary,
"master_url": cfg.Drivers.EOS.MasterURL,
"slave_url": cfg.Drivers.EOS.SlaveURL,
"cache_directory": cfg.Drivers.EOS.CacheDirectory,
"sec_protocol": cfg.Drivers.EOS.SecProtocol,
"keytab": cfg.Drivers.EOS.Keytab,
"single_username": cfg.Drivers.EOS.SingleUsername,
"enable_logging": cfg.Drivers.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Drivers.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Drivers.EOS.ForceSingleUserMode,
"use_keytab": cfg.Drivers.EOS.UseKeytab,
"gatewaysvc": cfg.Drivers.EOS.GatewaySVC,
"enable_home": false,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Drivers.EOS.Root,
"shadow_namespace": cfg.Drivers.EOS.ShadowNamespace,
"eos_binary": cfg.Drivers.EOS.EosBinary,
"xrdcopy_binary": cfg.Drivers.EOS.XrdcopyBinary,
"master_url": cfg.Drivers.EOS.MasterURL,
"master_grpc_uri": cfg.Drivers.EOS.GRPCURI,
"slave_url": cfg.Drivers.EOS.SlaveURL,
"cache_directory": cfg.Drivers.EOS.CacheDirectory,
"sec_protocol": cfg.Drivers.EOS.SecProtocol,
"keytab": cfg.Drivers.EOS.Keytab,
"single_username": cfg.Drivers.EOS.SingleUsername,
"user_layout": cfg.Drivers.EOS.UserLayout,
"enable_logging": cfg.Drivers.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Drivers.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Drivers.EOS.ForceSingleUserMode,
"use_keytab": cfg.Drivers.EOS.UseKeytab,
"enable_home": false,
"gatewaysvc": cfg.Drivers.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Drivers.Local.Root,
},
"ocis": map[string]interface{}{
"root": cfg.Drivers.OCIS.Root,
"user_layout": cfg.Drivers.OCIS.UserLayout,
"treetime_accounting": false,
"treesize_accounting": false,
"permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint,
},
"s3": map[string]interface{}{
"region": cfg.Drivers.S3.Region,
"access_key": cfg.Drivers.S3.AccessKey,
"secret_key": cfg.Drivers.S3.SecretKey,
"endpoint": cfg.Drivers.S3.Endpoint,
"bucket": cfg.Drivers.S3.Bucket,
},
"s3ng": map[string]interface{}{
"root": cfg.Drivers.S3NG.Root,
"enable_home": false,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"treetime_accounting": false,
"treesize_accounting": false,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
},
}
}