sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

switch all other services to struct tag based env config

Browse Source
This commit is contained in:
Willy Kloucek
2022-01-07 15:47:47 +00:00
committed by Jörn Friedrich Dreyer
parent 288d6c469e
commit 6990e7d660
116 changed files with 1024 additions and 1991 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxy/pkg/command/health.go
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"github.com/owncloud/ocis/proxy/pkg/config"
"github.com/owncloud/ocis/proxy/pkg/logging"
"github.com/urfave/cli/v2"
)
@@ -15,7 +16,7 @@ func Health(cfg *config.Config) *cli.Command {
Usage: "Check health status",
//Flags: flagset.HealthWithConfig(cfg),
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
logger := logging.Configure(cfg.Service.Name, cfg.Log)
resp, err := http.Get(
fmt.Sprintf(
proxy/pkg/command/root.go
+28 -35
View File
@@ -4,13 +4,13 @@ import (
"context"
"os"
"github.com/imdario/mergo"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/ocis-pkg/shared"
"github.com/owncloud/ocis/ocis-pkg/version"
"github.com/owncloud/ocis/proxy/pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
"github.com/wkloucek/envdecode"
)
// Execute is the entry point for the ocis-proxy command.
@@ -26,10 +26,12 @@ func Execute(cfg *config.Config) error {
Email: "support@owncloud.com",
},
},
Before: func(c *cli.Context) error {
cfg.Service.Version = version.String
return nil
return ParseConfig(c, cfg)
},
Commands: []*cli.Command{
Server(cfg),
Health(cfg),
@@ -50,35 +52,37 @@ func Execute(cfg *config.Config) error {
return app.Run(os.Args)
}
// ParseConfig loads proxy configuration. Loading will first attempt to parse config files in the expected locations
// and then parses environment variables. In the context of oCIS env variables will always overwrite values set
// in a config file.
// If this extension is run as a subcommand (i.e: ocis proxy) then there are 2 levels of config parsing:
// 1. ocis.yaml (if any)
// 2. proxy.yaml (if any)
// 3. environment variables.
// ParseConfig loads accounts configuration from known paths.
func ParseConfig(c *cli.Context, cfg *config.Config) error {
conf, err := ociscfg.BindSourcesToStructs("proxy", cfg)
_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &shared.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil && cfg.Commons == nil {
cfg.Log = &shared.Log{}
//if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
// cfg.Log = &shared.Log{
// Level: cfg.Commons.Log.Level,
// Pretty: cfg.Commons.Log.Pretty,
// Color: cfg.Commons.Log.Color,
// File: cfg.Commons.Log.File,
// }
//} else if cfg.Log == nil && cfg.Commons == nil {
// cfg.Log = &shared.Log{}
//}
// load all env variables relevant to the config in the current context.
envCfg := config.Config{}
if err := envdecode.Decode(&envCfg); err != nil && err.Error() != "none of the target fields were set from environment variables" {
return err
}
conf.LoadOSEnv(config.GetEnv(cfg), false)
// merge environment variable config on top of the current config
if err := mergo.Merge(cfg, envCfg, mergo.WithOverride); err != nil {
return err
}
bindings := config.StructMappings(cfg)
return ociscfg.BindEnv(conf, bindings)
return nil
}
// SutureService allows for the proxy command to be embedded and supervised by a suture supervisor tree.
@@ -88,7 +92,7 @@ type SutureService struct {
// NewSutureService creates a new proxy.SutureService
func NewSutureService(cfg *ociscfg.Config) suture.Service {
cfg.Proxy.Commons = cfg.Commons
//cfg.Proxy.Commons = cfg.Commons
return SutureService{
cfg: cfg.Proxy,
}
@@ -102,14 +106,3 @@ func (s SutureService) Serve(ctx context.Context) error {
return nil
}
// NewLogger initializes a service-specific logger instance.
func NewLogger(cfg *config.Config) log.Logger {
return log.NewLogger(
log.Name("proxy"),
log.Level(cfg.Log.Level),
log.Pretty(cfg.Log.Pretty),
log.Color(cfg.Log.Color),
log.File(cfg.Log.File),
)
}
proxy/pkg/command/server.go
+4 -3
View File
@@ -20,6 +20,7 @@ import (
"github.com/owncloud/ocis/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/proxy/pkg/config"
"github.com/owncloud/ocis/proxy/pkg/cs3"
"github.com/owncloud/ocis/proxy/pkg/logging"
"github.com/owncloud/ocis/proxy/pkg/metrics"
"github.com/owncloud/ocis/proxy/pkg/middleware"
"github.com/owncloud/ocis/proxy/pkg/proxy"
@@ -62,9 +63,9 @@ func Server(cfg *config.Config) *cli.Command {
return nil
},
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
if err := tracing.Configure(cfg); err != nil {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
err := tracing.Configure(cfg)
if err != nil {
return err
}
proxy/pkg/config/config.go
+57 -61
View File
@@ -8,45 +8,45 @@ import (
"github.com/owncloud/ocis/ocis-pkg/shared"
)
// Log defines the available logging configuration.
// Log defines the available log configuration.
type Log struct {
Level string `ocisConfig:"level"`
Pretty bool `ocisConfig:"pretty"`
Color bool `ocisConfig:"color"`
File string `ocisConfig:"file"`
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;PROXY_LOG_LEVEL"`
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;PROXY_LOG_PRETTY"`
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;PROXY_LOG_COLOR"`
File string `mapstructure:"file" env:"OCIS_LOG_FILE;PROXY_LOG_FILE"`
}
// Debug defines the available debug configuration.
type Debug struct {
Addr string `ocisConfig:"addr"`
Token string `ocisConfig:"token"`
Pprof bool `ocisConfig:"pprof"`
Zpages bool `ocisConfig:"zpages"`
Addr string `ocisConfig:"addr" env:"PROXY_DEBUG_ADDR"`
Token string `ocisConfig:"token" env:"PROXY_DEBUG_TOKEN"`
Pprof bool `ocisConfig:"pprof" env:"PROXY_DEBUG_PPROF"`
Zpages bool `ocisConfig:"zpages" env:"PROXY_DEBUG_ZPAGES"`
}
// HTTP defines the available http configuration.
type HTTP struct {
Addr string `ocisConfig:"addr"`
Root string `ocisConfig:"root"`
Addr string `ocisConfig:"addr" env:"PROXY_HTTP_ADDR"`
Root string `ocisConfig:"root" env:"PROXY_HTTP_ROOT"`
Namespace string
TLSCert string `ocisConfig:"tls_cert"`
TLSKey string `ocisConfig:"tls_key"`
TLS bool `ocisConfig:"tls"`
TLSCert string `ocisConfig:"tls_cert" env:"PROXY_TRANSPORT_TLS_CERT"`
TLSKey string `ocisConfig:"tls_key" env:"PROXY_TRANSPORT_TLS_KEY"`
TLS bool `ocisConfig:"tls" env:"PROXY_TLS"`
}
// Service defines the available service configuration.
type Service struct {
Name string `ocisConfig:"name"`
Version string `ocisConfig:"version"`
Name string
Version string
}
// Tracing defines the available tracing configuration.
type Tracing struct {
Enabled bool `ocisConfig:"enabled"`
Type string `ocisConfig:"type"`
Endpoint string `ocisConfig:"endpoint"`
Collector string `ocisConfig:"collector"`
Service string `ocisConfig:"service"`
Enabled bool `ocisConfig:"enabled" env:"OCIS_TRACING_ENABLED;PROXY_TRACING_ENABLED"`
Type string `ocisConfig:"type" env:"OCIS_TRACING_TYPE;PROXY_TRACING_TYPE"`
Endpoint string `ocisConfig:"endpoint" env:"OCIS_TRACING_ENDPOINT;PROXY_TRACING_ENDPOINT"`
Collector string `ocisConfig:"collector" env:"OCIS_TRACING_COLLECTOR;PROXY_TRACING_COLLECTOR"`
Service string `ocisConfig:"service" env:"PROXY_TRACING_SERVICE"` //TODO: should this be an ID? or the same as Service.Name?
}
// Policy enables us to use multiple directors.
@@ -84,7 +84,7 @@ var (
// Reva defines all available REVA configuration.
type Reva struct {
Address string `ocisConfig:"address"`
Address string `ocisConfig:"address" env:"REVA_GATEWAY"`
Middleware Middleware `ocisConfig:"middleware"`
}
@@ -98,34 +98,31 @@ type Auth struct {
CredentialsByUserAgent map[string]string `ocisConfig:""`
}
// Cache is a TTL cache configuration.
type Cache struct {
Size int `ocisConfig:"size"`
TTL int `ocisConfig:"ttl"`
}
// Config combines all available configuration parts.
type Config struct {
*shared.Commons
Log *shared.Log `ocisConfig:"log"`
Debug Debug `ocisConfig:"debug"`
HTTP HTTP `ocisConfig:"http"`
Service Service `ocisConfig:"service"`
Tracing Tracing `ocisConfig:"tracing"`
Service Service `ocisConfig:"service"`
Tracing Tracing `ocisConfig:"tracing"`
Log Log `ocisConfig:"log"`
Debug Debug `ocisConfig:"debug"`
HTTP HTTP `ocisConfig:"http"`
Policies []Policy `ocisConfig:"policies"`
OIDC OIDC `ocisConfig:"oidc"`
TokenManager TokenManager `ocisConfig:"token_manager"`
PolicySelector *PolicySelector `ocisConfig:"policy_selector"`
Reva Reva `ocisConfig:"reva"`
PreSignedURL PreSignedURL `ocisConfig:"pre_signed_url"`
AccountBackend string `ocisConfig:"account_backend"`
UserOIDCClaim string `ocisConfig:"user_oidc_claim"`
UserCS3Claim string `ocisConfig:"user_cs3_claim"`
MachineAuthAPIKey string `ocisConfig:"machine_auth_api_key"`
AutoprovisionAccounts bool `ocisConfig:"auto_provision_accounts"`
EnableBasicAuth bool `ocisConfig:"enable_basic_auth"`
InsecureBackends bool `ocisConfig:"insecure_backends"`
AccountBackend string `ocisConfig:"account_backend" env:"PROXY_ACCOUNT_BACKEND_TYPE"`
UserOIDCClaim string `ocisConfig:"user_oidc_claim" env:"PROXY_USER_OIDC_CLAIM"`
UserCS3Claim string `ocisConfig:"user_cs3_claim" env:"PROXY_USER_CS3_CLAIM"`
MachineAuthAPIKey string `ocisConfig:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;PROXY_MACHINE_AUTH_API_KEY"`
AutoprovisionAccounts bool `ocisConfig:"auto_provision_accounts" env:"PROXY_AUTOPROVISION_ACCOUNTS"`
EnableBasicAuth bool `ocisConfig:"enable_basic_auth" env:"PROXY_ENABLE_BASIC_AUTH"`
InsecureBackends bool `ocisConfig:"insecure_backends" env:"PROXY_INSECURE_BACKENDS"`
Context context.Context
Supervised bool
@@ -134,9 +131,15 @@ type Config struct {
// OIDC is the config for the OpenID-Connect middleware. If set the proxy will try to authenticate every request
// with the configured oidc-provider
type OIDC struct {
Issuer string `ocisConfig:"issuer"`
Insecure bool `ocisConfig:"insecure"`
UserinfoCache Cache `ocisConfig:"user_info_cache"`
Issuer string `ocisConfig:"issuer" env:"OCIS_URL;PROXY_OIDC_ISSUER"`
Insecure bool `ocisConfig:"insecure" env:"OCIS_INSECURE;PROXY_OIDC_INSECURE"`
UserinfoCache UserinfoCache `ocisConfig:"user_info_cache"`
}
// UserinfoCache is a TTL cache configuration.
type UserinfoCache struct {
Size int `ocisConfig:"size" env:"PROXY_OIDC_USERINFO_CACHE_SIZE"`
TTL int `ocisConfig:"ttl" env:"PROXY_OIDC_USERINFO_CACHE_TTL"`
}
// PolicySelector is the toplevel-configuration for different selectors
@@ -154,13 +157,13 @@ type StaticSelectorConf struct {
// TokenManager is the config for using the reva token manager
type TokenManager struct {
JWTSecret string `ocisConfig:"jwt_secret"`
JWTSecret string `ocisConfig:"jwt_secret" env:"OCIS_JWT_SECRET;PROXY_JWT_SECRET"`
}
// PreSignedURL is the config for the presigned url middleware
type PreSignedURL struct {
AllowedHTTPMethods []string `ocisConfig:"allowed_http_methods"`
Enabled bool `ocisConfig:"enabled"`
Enabled bool `ocisConfig:"enabled" env:"PROXY_ENABLE_PRESIGNEDURLS"`
}
// MigrationSelectorConf is the config for the migration-selector
@@ -192,13 +195,6 @@ type RegexRuleConf struct {
Policy string `ocisConfig:"policy"`
}
// New initializes a new configuration
func New() *Config {
return &Config{
HTTP: HTTP{},
}
}
// DefaultConfig provides with a working local configuration for a proxy service.
func DefaultConfig() *Config {
return &Config{
@@ -227,7 +223,7 @@ func DefaultConfig() *Config {
Issuer: "https://localhost:9200",
Insecure: true,
//Insecure: true,
UserinfoCache: Cache{
UserinfoCache: UserinfoCache{
Size: 1024,
TTL: 10,
},
@@ -243,14 +239,14 @@ func DefaultConfig() *Config {
AllowedHTTPMethods: []string{"GET"},
Enabled: true,
},
AccountBackend: "accounts",
UserOIDCClaim: "email",
UserCS3Claim: "mail",
MachineAuthAPIKey: "change-me-please",
//AutoprovisionAccounts: false,
//EnableBasicAuth: false,
//InsecureBackends: false,
Context: nil,
AccountBackend: "accounts",
UserOIDCClaim: "email",
UserCS3Claim: "mail",
MachineAuthAPIKey: "change-me-please",
AutoprovisionAccounts: false,
EnableBasicAuth: false,
InsecureBackends: false,
// TODO: enable
//Policies: defaultPolicies(),
}
}
proxy/pkg/config/mappings.go
-182
View File
@@ -1,182 +0,0 @@
package config
import "github.com/owncloud/ocis/ocis-pkg/shared"
// GetEnv fetches a list of known env variables for this extension. It is to be used by gookit, as it provides a list
// with all the environment variables an extension supports.
func GetEnv(cfg *Config) []string {
var r = make([]string, len(structMappings(cfg)))
for i := range structMappings(cfg) {
r = append(r, structMappings(cfg)[i].EnvVars...)
}
return r
}
// StructMappings binds a set of environment variables to a destination on cfg. Iterating over this set and editing the
// Destination value of a binding will alter the original value, as it is a pointer to its memory address. This lets
// us propagate changes easier.
func StructMappings(cfg *Config) []shared.EnvBinding {
return structMappings(cfg)
}
func structMappings(cfg *Config) []shared.EnvBinding {
return []shared.EnvBinding{
// Logging
{
EnvVars: []string{"OCIS_LOG_LEVEL", "PROXY_LOG_LEVEL"},
Destination: &cfg.Log.Level,
},
{
EnvVars: []string{"OCIS_LOG_COLOR", "PROXY_LOG_COLOR"},
Destination: &cfg.Log.Color,
},
{
EnvVars: []string{"OCIS_LOG_PRETTY", "PROXY_LOG_PRETTY"},
Destination: &cfg.Log.Pretty,
},
{
EnvVars: []string{"OCIS_LOG_FILE", "PROXY_LOG_FILE"},
Destination: &cfg.Log.File,
},
// Basic auth
{
EnvVars: []string{"PROXY_ENABLE_BASIC_AUTH"},
Destination: &cfg.EnableBasicAuth,
},
// Debug (health)
{
EnvVars: []string{"PROXY_DEBUG_ADDR"},
Destination: &cfg.Debug.Addr,
},
// Tracing
{
EnvVars: []string{"OCIS_TRACING_ENABLED", "PROXY_TRACING_ENABLED"},
Destination: &cfg.Tracing.Enabled,
},
{
EnvVars: []string{"OCIS_TRACING_TYPE", "PROXY_TRACING_TYPE"},
Destination: &cfg.Tracing.Type,
},
{
EnvVars: []string{"OCIS_TRACING_ENDPOINT", "PROXY_TRACING_ENDPOINT"},
Destination: &cfg.Tracing.Endpoint,
},
{
EnvVars: []string{"OCIS_TRACING_COLLECTOR", "PROXY_TRACING_COLLECTOR"},
Destination: &cfg.Tracing.Collector,
},
{
EnvVars: []string{"PROXY_TRACING_SERVICE"},
Destination: &cfg.Tracing.Service,
},
// Debug
{
EnvVars: []string{"PROXY_DEBUG_ADDR"},
Destination: &cfg.Debug.Addr,
},
{
EnvVars: []string{"PROXY_DEBUG_TOKEN"},
Destination: &cfg.Debug.Token,
},
{
EnvVars: []string{"PROXY_DEBUG_PPROF"},
Destination: &cfg.Debug.Pprof,
},
{
EnvVars: []string{"PROXY_DEBUG_ZPAGES"},
Destination: &cfg.Debug.Zpages,
},
// HTTP
{
EnvVars: []string{"PROXY_HTTP_ADDR"},
Destination: &cfg.HTTP.Addr,
},
{
EnvVars: []string{"PROXY_HTTP_ROOT"},
Destination: &cfg.HTTP.Root,
},
{
EnvVars: []string{"PROXY_HTTP_NAMESPACE"},
Destination: &cfg.HTTP.Namespace,
},
{
EnvVars: []string{"PROXY_TRANSPORT_TLS_CERT"},
Destination: &cfg.HTTP.TLSCert,
},
{
EnvVars: []string{"PROXY_TRANSPORT_TLS_KEY"},
Destination: &cfg.HTTP.TLSKey,
},
{
EnvVars: []string{"PROXY_TLS"},
Destination: &cfg.HTTP.TLS,
},
// Other
{
EnvVars: []string{"OCIS_JWT_SECRET", "PROXY_JWT_SECRET"},
Destination: &cfg.TokenManager.JWTSecret,
},
{
EnvVars: []string{"REVA_GATEWAY"},
Destination: &cfg.Reva.Address,
},
{
EnvVars: []string{"PROXY_INSECURE_BACKENDS"},
Destination: &cfg.InsecureBackends,
},
{
EnvVars: []string{"OCIS_URL", "PROXY_OIDC_ISSUER"},
Destination: &cfg.OIDC.Issuer,
},
{
EnvVars: []string{"OCIS_INSECURE", "PROXY_OIDC_INSECURE"},
Destination: &cfg.OIDC.Insecure,
},
{
EnvVars: []string{"PROXY_OIDC_USERINFO_CACHE_TTL"},
Destination: &cfg.OIDC.UserinfoCache.TTL,
},
{
EnvVars: []string{"PROXY_OIDC_USERINFO_CACHE_SIZE"},
Destination: &cfg.OIDC.UserinfoCache.Size,
},
{
EnvVars: []string{"PROXY_AUTOPROVISION_ACCOUNTS"},
Destination: &cfg.AutoprovisionAccounts,
},
{
EnvVars: []string{"PROXY_USER_OIDC_CLAIM"},
Destination: &cfg.UserOIDCClaim,
},
{
EnvVars: []string{"PROXY_USER_CS3_CLAIM"},
Destination: &cfg.UserCS3Claim,
},
{
EnvVars: []string{"PROXY_ENABLE_PRESIGNEDURLS"},
Destination: &cfg.PreSignedURL.Enabled,
},
{
EnvVars: []string{"PROXY_ACCOUNT_BACKEND_TYPE"},
Destination: &cfg.AccountBackend,
},
{
EnvVars: []string{"OCIS_MACHINE_AUTH_API_KEY", "PROXY_MACHINE_AUTH_API_KEY"},
Destination: &cfg.MachineAuthAPIKey,
},
// there are 2 missing bindings:
// EnvVars: []string{"PROXY_MIDDLEWARE_AUTH_CREDENTIALS_BY_USER_AGENT"},
// EnvVars: []string{"PRESIGNEDURL_ALLOWED_METHODS"},
// since they both have no destination
// see https://github.com/owncloud/ocis/blob/52e5effa4fa05a1626d46f7d4cb574dde3a54593/proxy/pkg/flagset/flagset.go#L256-L261
// and https://github.com/owncloud/ocis/blob/52e5effa4fa05a1626d46f7d4cb574dde3a54593/proxy/pkg/flagset/flagset.go#L295-L300
}
}
proxy/pkg/cs3/client.go
+2 -1
View File
@@ -10,7 +10,7 @@ import (
func newConn(endpoint string) (*grpc.ClientConn, error) {
conn, err := grpc.Dial(
endpoint,
grpc.WithInsecure(),
grpc.WithInsecure(), //TODO: depreciated
grpc.WithUnaryInterceptor(
otelgrpc.UnaryClientInterceptor(
otelgrpc.WithTracerProvider(
@@ -28,6 +28,7 @@ func newConn(endpoint string) (*grpc.ClientConn, error) {
// GetGatewayServiceClient returns a new cs3 gateway client
func GetGatewayServiceClient(endpoint string) (gateway.GatewayAPIClient, error) {
// TODO: check connection pooling
conn, err := newConn(endpoint)
if err != nil {
return nil, err
proxy/pkg/logging/logging.go
+17
View File
@@ -0,0 +1,17 @@
package logging
import (
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/proxy/pkg/config"
)
// LoggerFromConfig initializes a service-specific logger instance.
func Configure(name string, cfg config.Log) log.Logger {
return log.NewLogger(
log.Name(name),
log.Level(cfg.Level),
log.Pretty(cfg.Pretty),
log.Color(cfg.Color),
log.File(cfg.File),
)
}