Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+126
View File
@@ -0,0 +1,126 @@
package config
import (
"github.com/qsfera/server/pkg/shared"
activitylog "github.com/qsfera/server/services/activitylog/pkg/config"
antivirus "github.com/qsfera/server/services/antivirus/pkg/config"
appProvider "github.com/qsfera/server/services/app-provider/pkg/config"
appRegistry "github.com/qsfera/server/services/app-registry/pkg/config"
audit "github.com/qsfera/server/services/audit/pkg/config"
authapp "github.com/qsfera/server/services/auth-app/pkg/config"
authbasic "github.com/qsfera/server/services/auth-basic/pkg/config"
authbearer "github.com/qsfera/server/services/auth-bearer/pkg/config"
authmachine "github.com/qsfera/server/services/auth-machine/pkg/config"
authservice "github.com/qsfera/server/services/auth-service/pkg/config"
clientlog "github.com/qsfera/server/services/clientlog/pkg/config"
collaboration "github.com/qsfera/server/services/collaboration/pkg/config"
eventhistory "github.com/qsfera/server/services/eventhistory/pkg/config"
frontend "github.com/qsfera/server/services/frontend/pkg/config"
gateway "github.com/qsfera/server/services/gateway/pkg/config"
graph "github.com/qsfera/server/services/graph/pkg/config"
groups "github.com/qsfera/server/services/groups/pkg/config"
idm "github.com/qsfera/server/services/idm/pkg/config"
idp "github.com/qsfera/server/services/idp/pkg/config"
invitations "github.com/qsfera/server/services/invitations/pkg/config"
nats "github.com/qsfera/server/services/nats/pkg/config"
notifications "github.com/qsfera/server/services/notifications/pkg/config"
ocm "github.com/qsfera/server/services/ocm/pkg/config"
ocs "github.com/qsfera/server/services/ocs/pkg/config"
policies "github.com/qsfera/server/services/policies/pkg/config"
postprocessing "github.com/qsfera/server/services/postprocessing/pkg/config"
proxy "github.com/qsfera/server/services/proxy/pkg/config"
search "github.com/qsfera/server/services/search/pkg/config"
settings "github.com/qsfera/server/services/settings/pkg/config"
sharing "github.com/qsfera/server/services/sharing/pkg/config"
sse "github.com/qsfera/server/services/sse/pkg/config"
storagepublic "github.com/qsfera/server/services/storage-publiclink/pkg/config"
storageshares "github.com/qsfera/server/services/storage-shares/pkg/config"
storagesystem "github.com/qsfera/server/services/storage-system/pkg/config"
storageusers "github.com/qsfera/server/services/storage-users/pkg/config"
thumbnails "github.com/qsfera/server/services/thumbnails/pkg/config"
userlog "github.com/qsfera/server/services/userlog/pkg/config"
users "github.com/qsfera/server/services/users/pkg/config"
web "github.com/qsfera/server/services/web/pkg/config"
webdav "github.com/qsfera/server/services/webdav/pkg/config"
webfinger "github.com/qsfera/server/services/webfinger/pkg/config"
)
type Mode int
// Runtime configures the КуСфера runtime when running in supervised mode.
type Runtime struct {
Port string `yaml:"port" env:"OC_RUNTIME_PORT" desc:"The TCP port at which КуСфера will be available" introductionVersion:"1.0.0"`
Host string `yaml:"host" env:"OC_RUNTIME_HOST" desc:"The host at which КуСфера will be available" introductionVersion:"1.0.0"`
Services []string `yaml:"services" env:"OC_RUN_EXTENSIONS;OC_RUN_SERVICES" desc:"A comma-separated list of service names. Will start only the listed services." introductionVersion:"1.0.0"`
Disabled []string `yaml:"disabled_services" env:"OC_EXCLUDE_RUN_SERVICES" desc:"A comma-separated list of service names. Will start all default services except of the ones listed. Has no effect when OC_RUN_SERVICES is set." introductionVersion:"1.0.0"`
Additional []string `yaml:"add_services" env:"OC_ADD_RUN_SERVICES" desc:"A comma-separated list of service names. Will add the listed services to the default configuration. Has no effect when OC_RUN_SERVICES is set. Note that one can add services not started by the default list and exclude services from the default list by using both envvars at the same time." introductionVersion:"1.0.0"`
ShutdownOrder []string `yaml:"shutdown_order" env:"OC_SHUTDOWN_ORDER" desc:"A comma-separated list of service names defining the order in which services are shut down. Services not listed will be stopped after the listed ones in random order." introductionVersion:"4.0.0"`
}
// Config combines all available configuration parts.
type Config struct {
*shared.Commons `yaml:"shared"`
Log *shared.Log `yaml:"log"`
Cache *shared.Cache `yaml:"cache"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
GRPCServiceTLS *shared.GRPCServiceTLS `yaml:"grpc_service_tls"`
HTTPServiceTLS shared.HTTPServiceTLS `yaml:"http_service_tls"`
Reva *shared.Reva `yaml:"reva"`
Mode Mode // DEPRECATED
File string
QsferaURL string `yaml:"qsfera_url" env:"OC_URL" desc:"URL, where КуСфера is reachable for users." introductionVersion:"1.0.0"`
Registry string `yaml:"registry"`
TokenManager *shared.TokenManager `yaml:"token_manager"`
MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OC_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services." introductionVersion:"1.0.0"`
TransferSecret string `yaml:"transfer_secret" env:"OC_TRANSFER_SECRET" desc:"Transfer secret for signing file up- and download requests." introductionVersion:"1.0.0"`
URLSigningSecret string `yaml:"url_signing_secret" env:"OC_URL_SIGNING_SECRET" desc:"The shared secret used to sign URLs e.g. for image downloads by the web office suite." introductionVersion:"4.0.0"`
SystemUserID string `yaml:"system_user_id" env:"OC_SYSTEM_USER_ID" desc:"ID of the КуСфера storage-system system user. Admins need to set the ID for the storage-system system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format." introductionVersion:"1.0.0"`
SystemUserAPIKey string `yaml:"system_user_api_key" env:"OC_SYSTEM_USER_API_KEY" desc:"API key for the storage-system system user." introductionVersion:"1.0.0"`
AdminUserID string `yaml:"admin_user_id" env:"OC_ADMIN_USER_ID" desc:"ID of a user, that should receive admin privileges. Consider that the UUID can be encoded in some LDAP deployment configurations like in .ldif files. These need to be decoded beforehand." introductionVersion:"1.0.0"`
Runtime Runtime `yaml:"runtime"`
Activitylog *activitylog.Config `yaml:"activitylog"`
Antivirus *antivirus.Config `yaml:"antivirus"`
AppProvider *appProvider.Config `yaml:"app_provider"`
AppRegistry *appRegistry.Config `yaml:"app_registry"`
Audit *audit.Config `yaml:"audit"`
AuthApp *authapp.Config `yaml:"auth_app"`
AuthBasic *authbasic.Config `yaml:"auth_basic"`
AuthBearer *authbearer.Config `yaml:"auth_bearer"`
AuthMachine *authmachine.Config `yaml:"auth_machine"`
AuthService *authservice.Config `yaml:"auth_service"`
Clientlog *clientlog.Config `yaml:"clientlog"`
Collaboration *collaboration.Config `yaml:"collaboration"`
EventHistory *eventhistory.Config `yaml:"eventhistory"`
Frontend *frontend.Config `yaml:"frontend"`
Gateway *gateway.Config `yaml:"gateway"`
Graph *graph.Config `yaml:"graph"`
Groups *groups.Config `yaml:"groups"`
IDM *idm.Config `yaml:"idm"`
IDP *idp.Config `yaml:"idp"`
Invitations *invitations.Config `yaml:"invitations"`
Nats *nats.Config `yaml:"nats"`
Notifications *notifications.Config `yaml:"notifications"`
OCM *ocm.Config `yaml:"ocm"`
OCS *ocs.Config `yaml:"ocs"`
Postprocessing *postprocessing.Config `yaml:"postprocessing"`
Policies *policies.Config `yaml:"policies"`
Proxy *proxy.Config `yaml:"proxy"`
Settings *settings.Config `yaml:"settings"`
Sharing *sharing.Config `yaml:"sharing"`
SSE *sse.Config `yaml:"sse"`
StorageSystem *storagesystem.Config `yaml:"storage_system"`
StoragePublicLink *storagepublic.Config `yaml:"storage_public"`
StorageShares *storageshares.Config `yaml:"storage_shares"`
StorageUsers *storageusers.Config `yaml:"storage_users"`
Thumbnails *thumbnails.Config `yaml:"thumbnails"`
Userlog *userlog.Config `yaml:"userlog"`
Users *users.Config `yaml:"users"`
Web *web.Config `yaml:"web"`
WebDAV *webdav.Config `yaml:"webdav"`
Webfinger *webfinger.Config `yaml:"webfinger"`
Search *search.Config `yaml:"search"`
}
+13
View File
@@ -0,0 +1,13 @@
package config_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestConfig(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Config Suite")
}
+16
View File
@@ -0,0 +1,16 @@
package config_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/qsfera/server/pkg/config"
"gopkg.in/yaml.v2"
)
var _ = Describe("Config", func() {
It("Success generating the default config", func() {
cfg := config.DefaultConfig()
_, err := yaml.Marshal(cfg)
Expect(err).To(BeNil())
})
})
+30
View File
@@ -0,0 +1,30 @@
package configlog
import (
"fmt"
"os"
)
// Error logs the error
func Error(err error) {
if err != nil {
fmt.Printf("%v\n", err)
}
}
// ReturnError logs the error and returns it unchanged
func ReturnError(err error) error {
if err != nil {
fmt.Printf("%v\n", err)
}
return err
}
// ReturnFatal logs the error and calls os.Exit(1) and returns nil if no error is passed
func ReturnFatal(err error) error {
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
return nil
}
+102
View File
@@ -0,0 +1,102 @@
package config
import (
"github.com/qsfera/server/pkg/shared"
activitylog "github.com/qsfera/server/services/activitylog/pkg/config/defaults"
antivirus "github.com/qsfera/server/services/antivirus/pkg/config/defaults"
appProvider "github.com/qsfera/server/services/app-provider/pkg/config/defaults"
appRegistry "github.com/qsfera/server/services/app-registry/pkg/config/defaults"
audit "github.com/qsfera/server/services/audit/pkg/config/defaults"
authapp "github.com/qsfera/server/services/auth-app/pkg/config/defaults"
authbasic "github.com/qsfera/server/services/auth-basic/pkg/config/defaults"
authbearer "github.com/qsfera/server/services/auth-bearer/pkg/config/defaults"
authmachine "github.com/qsfera/server/services/auth-machine/pkg/config/defaults"
authservice "github.com/qsfera/server/services/auth-service/pkg/config/defaults"
clientlog "github.com/qsfera/server/services/clientlog/pkg/config/defaults"
collaboration "github.com/qsfera/server/services/collaboration/pkg/config/defaults"
eventhistory "github.com/qsfera/server/services/eventhistory/pkg/config/defaults"
frontend "github.com/qsfera/server/services/frontend/pkg/config/defaults"
gateway "github.com/qsfera/server/services/gateway/pkg/config/defaults"
graph "github.com/qsfera/server/services/graph/pkg/config/defaults"
groups "github.com/qsfera/server/services/groups/pkg/config/defaults"
idm "github.com/qsfera/server/services/idm/pkg/config/defaults"
idp "github.com/qsfera/server/services/idp/pkg/config/defaults"
invitations "github.com/qsfera/server/services/invitations/pkg/config/defaults"
nats "github.com/qsfera/server/services/nats/pkg/config/defaults"
notifications "github.com/qsfera/server/services/notifications/pkg/config/defaults"
ocm "github.com/qsfera/server/services/ocm/pkg/config/defaults"
ocs "github.com/qsfera/server/services/ocs/pkg/config/defaults"
policies "github.com/qsfera/server/services/policies/pkg/config/defaults"
postprocessing "github.com/qsfera/server/services/postprocessing/pkg/config/defaults"
proxy "github.com/qsfera/server/services/proxy/pkg/config/defaults"
search "github.com/qsfera/server/services/search/pkg/config/defaults"
settings "github.com/qsfera/server/services/settings/pkg/config/defaults"
sharing "github.com/qsfera/server/services/sharing/pkg/config/defaults"
sse "github.com/qsfera/server/services/sse/pkg/config/defaults"
storagepublic "github.com/qsfera/server/services/storage-publiclink/pkg/config/defaults"
storageshares "github.com/qsfera/server/services/storage-shares/pkg/config/defaults"
storageSystem "github.com/qsfera/server/services/storage-system/pkg/config/defaults"
storageusers "github.com/qsfera/server/services/storage-users/pkg/config/defaults"
thumbnails "github.com/qsfera/server/services/thumbnails/pkg/config/defaults"
userlog "github.com/qsfera/server/services/userlog/pkg/config/defaults"
users "github.com/qsfera/server/services/users/pkg/config/defaults"
web "github.com/qsfera/server/services/web/pkg/config/defaults"
webdav "github.com/qsfera/server/services/webdav/pkg/config/defaults"
webfinger "github.com/qsfera/server/services/webfinger/pkg/config/defaults"
)
func DefaultConfig() *Config {
return &Config{
QsferaURL: "https://localhost:9200",
Runtime: Runtime{
Port: "9250",
Host: "localhost",
ShutdownOrder: []string{"proxy"},
},
Reva: &shared.Reva{
Address: "qsfera.api.gateway",
},
Activitylog: activitylog.DefaultConfig(),
Antivirus: antivirus.DefaultConfig(),
AppProvider: appProvider.DefaultConfig(),
AppRegistry: appRegistry.DefaultConfig(),
Audit: audit.DefaultConfig(),
AuthApp: authapp.DefaultConfig(),
AuthBasic: authbasic.DefaultConfig(),
AuthBearer: authbearer.DefaultConfig(),
AuthMachine: authmachine.DefaultConfig(),
AuthService: authservice.DefaultConfig(),
Clientlog: clientlog.DefaultConfig(),
Collaboration: collaboration.DefaultConfig(),
EventHistory: eventhistory.DefaultConfig(),
Frontend: frontend.DefaultConfig(),
Gateway: gateway.DefaultConfig(),
Graph: graph.DefaultConfig(),
Groups: groups.DefaultConfig(),
IDM: idm.DefaultConfig(),
IDP: idp.DefaultConfig(),
Invitations: invitations.DefaultConfig(),
Nats: nats.DefaultConfig(),
Notifications: notifications.DefaultConfig(),
OCM: ocm.DefaultConfig(),
OCS: ocs.DefaultConfig(),
Postprocessing: postprocessing.DefaultConfig(),
Policies: policies.DefaultConfig(),
Proxy: proxy.DefaultConfig(),
Search: search.DefaultConfig(),
Settings: settings.DefaultConfig(),
Sharing: sharing.DefaultConfig(),
SSE: sse.DefaultConfig(),
StoragePublicLink: storagepublic.DefaultConfig(),
StorageShares: storageshares.DefaultConfig(),
StorageSystem: storageSystem.DefaultConfig(),
StorageUsers: storageusers.DefaultConfig(),
Thumbnails: thumbnails.DefaultConfig(),
Userlog: userlog.DefaultConfig(),
Users: users.DefaultConfig(),
Web: web.DefaultConfig(),
WebDAV: webdav.DefaultConfig(),
Webfinger: webfinger.DefaultConfig(),
}
}
+73
View File
@@ -0,0 +1,73 @@
package defaults
import (
"log"
"os"
"path"
)
var (
// switch between modes
BaseDataPathType = "homedir" // or "path"
// default data path
BaseDataPathValue = "/var/lib/qsfera"
)
func BaseDataPath() string {
// It is not nice to have hidden / secrete configuration options
// But how can we update the base path for every occurrence with a flagset option?
// This is currently not possible and needs a new configuration concept
p := os.Getenv("OC_BASE_DATA_PATH")
if p != "" {
return p
}
switch BaseDataPathType {
case "homedir":
dir, err := os.UserHomeDir()
if err != nil {
// fallback to BaseDatapathValue for users without home
return BaseDataPathValue
}
return path.Join(dir, ".qsfera")
case "path":
return BaseDataPathValue
default:
log.Fatalf("BaseDataPathType %s not found", BaseDataPathType)
return ""
}
}
var (
// switch between modes
BaseConfigPathType = "homedir" // or "path"
// default config path
BaseConfigPathValue = "/etc/qsfera"
)
func BaseConfigPath() string {
// It is not nice to have hidden / secrete configuration options
// But how can we update the base path for every occurrence with a flagset option?
// This is currently not possible and needs a new configuration concept
p := os.Getenv("OC_CONFIG_DIR")
if p != "" {
return p
}
switch BaseConfigPathType {
case "homedir":
dir, err := os.UserHomeDir()
if err != nil {
// fallback to BaseConfigPathValue for users without home
return BaseConfigPathValue
}
return path.Join(dir, ".qsfera", "config")
case "path":
return BaseConfigPathValue
default:
log.Fatalf("BaseConfigPathType %s not found", BaseConfigPathType)
return ""
}
}
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 Joe Shaw
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+88
View File
@@ -0,0 +1,88 @@
`envdecode` is a Go package for populating structs from environment
variables. It's basically a fork of https://github.com/joeshaw/envdecode,
but changed to support multiple environment variables (precedence).
`envdecode` uses struct tags to map environment variables to fields,
allowing you to use any names you want for environment variables.
`envdecode` will recurse into nested structs, including pointers to
nested structs, but it will not allocate new pointers to structs.
## API
Full API docs are available on
[godoc.org](https://godoc.org/github.com/qsfera/server/pkg/config/envdecode).
Define a struct with `env` struct tags:
```go
type Config struct {
Hostname string `env:"SERVER_HOSTNAME,default=localhost"`
Port uint16 `env:"HTTP_PORT;SERVER_PORT,default=8080"`
AWS struct {
ID string `env:"AWS_ACCESS_KEY_ID"`
Secret string `env:"AWS_SECRET_ACCESS_KEY,required"`
SnsTopics []string `env:"AWS_SNS_TOPICS"`
}
Timeout time.Duration `env:"TIMEOUT,default=1m,strict"`
}
```
Fields _must be exported_ (i.e. begin with a capital letter) in order
for `envdecode` to work with them. An error will be returned if a
struct with no exported fields is decoded (including one that contains
no `env` tags at all).
Default values may be provided by appending ",default=value" to the
struct tag. Required values may be marked by appending ",required" to the
struct tag. Strict values may be marked by appending ",strict" which will
return an error on Decode if there is an error while parsing.
Then call `envdecode.Decode`:
```go
var cfg Config
err := envdecode.Decode(&cfg)
```
If you want all fields to act `strict`, you may use `envdecode.StrictDecode`:
```go
var cfg Config
err := envdecode.StrictDecode(&cfg)
```
All parse errors will fail fast and return an error in this mode.
## Supported types
- Structs (and pointer to structs)
- Slices of defined types below, separated by semicolon
- `bool`
- `float32`, `float64`
- `int`, `int8`, `int16`, `int32`, `int64`
- `uint`, `uint8`, `uint16`, `uint32`, `uint64`
- `string`
- `time.Duration`, using the [`time.ParseDuration()` format](http://golang.org/pkg/time/#ParseDuration)
- `*url.URL`, using [`url.Parse()`](https://godoc.org/net/url#Parse)
- Types those implement a `Decoder` interface
## Custom `Decoder`
If you want a field to be decoded with custom behavior, you may implement the interface `Decoder` for the filed type.
```go
type Config struct {
IPAddr IP `env:"IP_ADDR"`
}
type IP net.IP
// Decode implements the interface `envdecode.Decoder`
func (i *IP) Decode(repl string) error {
*i = net.ParseIP(repl)
return nil
}
```
`Decoder` is the interface implemented by an object that can decode an environment variable string representation of itself.
+436
View File
@@ -0,0 +1,436 @@
// Package envdecode is a package for populating structs from environment
// variables, using struct tags.
package envdecode
import (
"encoding"
"errors"
"fmt"
"log"
"net/url"
"os"
"reflect"
"sort"
"strconv"
"strings"
"time"
)
// ErrInvalidTarget indicates that the target value passed to
// Decode is invalid. Target must be a non-nil pointer to a struct.
var ErrInvalidTarget = errors.New("target must be non-nil pointer to struct that has at least one exported field with a valid env tag")
var ErrNoTargetFieldsAreSet = errors.New("none of the target fields were set from environment variables")
// FailureFunc is called when an error is encountered during a MustDecode
// operation. It prints the error and terminates the process.
//
// This variable can be assigned to another function of the user-programmer's
// design, allowing for graceful recovery of the problem, such as loading
// from a backup configuration file.
var FailureFunc = func(err error) {
log.Fatalf("envdecode: an error was encountered while decoding: %v\n", err)
}
// Decoder is the interface implemented by an object that can decode an
// environment variable string representation of itself.
type Decoder interface {
Decode(string) error
}
// Decode environment variables into the provided target. The target
// must be a non-nil pointer to a struct. Fields in the struct must
// be exported, and tagged with an "env" struct tag with a value
// containing the name of the environment variable. An error is
// returned if there are no exported members tagged.
//
// Default values may be provided by appending ",default=value" to the
// struct tag. Required values may be marked by appending ",required"
// to the struct tag. It is an error to provide both "default" and
// "required". Strict values may be marked by appending ",strict" which
// will return an error on Decode if there is an error while parsing.
// If everything must be strict, consider using StrictDecode instead.
//
// All primitive types are supported, including bool, floating point,
// signed and unsigned integers, and string. Boolean and numeric
// types are decoded using the standard strconv Parse functions for
// those types. Structs and pointers to structs are decoded
// recursively. time.Duration is supported via the
// time.ParseDuration() function and *url.URL is supported via the
// url.Parse() function. Slices are supported for all above mentioned
// primitive types. Semicolon is used as delimiter in environment variables.
func Decode(target any) error {
nFields, err := decode(target, false)
if err != nil {
return err
}
// if we didn't do anything - the user probably did something
// wrong like leave all fields unexported.
if nFields == 0 {
return ErrNoTargetFieldsAreSet
}
return nil
}
// StrictDecode is similar to Decode except all fields will have an implicit
// ",strict" on all fields.
func StrictDecode(target any) error {
nFields, err := decode(target, true)
if err != nil {
return err
}
// if we didn't do anything - the user probably did something
// wrong like leave all fields unexported.
if nFields == 0 {
return ErrInvalidTarget
}
return nil
}
func decode(target any, strict bool) (int, error) {
s := reflect.ValueOf(target)
if s.Kind() != reflect.Ptr || s.IsNil() {
return 0, ErrInvalidTarget
}
s = s.Elem()
if s.Kind() != reflect.Struct {
return 0, ErrInvalidTarget
}
t := s.Type()
setFieldCount := 0
for i := 0; i < s.NumField(); i++ {
// Localize the umbrella `strict` value to the specific field.
strict := strict
f := s.Field(i)
switch f.Kind() {
case reflect.Ptr:
if f.Elem().Kind() != reflect.Struct {
break
}
f = f.Elem()
fallthrough
case reflect.Struct:
if !f.Addr().CanInterface() {
continue
}
ss := f.Addr().Interface()
_, custom := ss.(Decoder)
if custom {
break
}
n, err := decode(ss, strict)
if err != nil {
return 0, err
}
setFieldCount += n
}
if !f.CanSet() {
continue
}
tag := t.Field(i).Tag.Get("env")
if tag == "" {
continue
}
parts := strings.Split(tag, ",")
overrides := strings.Split(parts[0], `;`)
var env string
var envSet bool
for _, override := range overrides {
if v, set := os.LookupEnv(override); set {
env = v
envSet = true
}
}
required := false
hasDefault := false
defaultValue := ""
for _, o := range parts[1:] {
if !required {
required = strings.HasPrefix(o, "required")
}
if strings.HasPrefix(o, "default=") {
hasDefault = true
defaultValue = o[8:]
}
if !strict {
strict = strings.HasPrefix(o, "strict")
}
}
if required && hasDefault {
panic(`envdecode: "default" and "required" may not be specified in the same annotation`)
}
if !envSet && required {
return 0, fmt.Errorf("the environment variable \"%s\" is missing", parts[0])
}
if !envSet {
env = defaultValue
}
if !envSet && env == "" {
continue
}
setFieldCount++
unmarshaler, implementsUnmarshaler := f.Addr().Interface().(encoding.TextUnmarshaler)
decoder, implmentsDecoder := f.Addr().Interface().(Decoder)
if implmentsDecoder {
if err := decoder.Decode(env); err != nil {
return 0, err
}
} else if implementsUnmarshaler {
if err := unmarshaler.UnmarshalText([]byte(env)); err != nil {
return 0, err
}
} else if f.Kind() == reflect.Slice {
if err := decodeSlice(&f, env); err != nil {
return 0, err
}
} else {
if err := decodePrimitiveType(&f, env); err != nil && strict {
return 0, err
}
}
}
return setFieldCount, nil
}
func decodeSlice(f *reflect.Value, env string) error {
parts := strings.Split(env, ",")
values := parts[:0]
for _, x := range parts {
if x != "" {
values = append(values, strings.TrimSpace(x))
}
}
valuesCount := len(values)
slice := reflect.MakeSlice(f.Type(), valuesCount, valuesCount)
if valuesCount > 0 {
for i := 0; i < valuesCount; i++ {
e := slice.Index(i)
err := decodePrimitiveType(&e, values[i])
if err != nil {
return err
}
}
}
f.Set(slice)
return nil
}
func decodePrimitiveType(f *reflect.Value, env string) error {
switch f.Kind() {
case reflect.Bool:
v, err := strconv.ParseBool(env)
if err != nil {
return err
}
f.SetBool(v)
case reflect.Float32, reflect.Float64:
bits := f.Type().Bits()
v, err := strconv.ParseFloat(env, bits)
if err != nil {
return err
}
f.SetFloat(v)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
if t := f.Type(); t.PkgPath() == "time" && t.Name() == "Duration" {
v, err := time.ParseDuration(env)
if err != nil {
return err
}
f.SetInt(int64(v))
} else {
bits := f.Type().Bits()
v, err := strconv.ParseInt(env, 0, bits)
if err != nil {
return err
}
f.SetInt(v)
}
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
bits := f.Type().Bits()
v, err := strconv.ParseUint(env, 0, bits)
if err != nil {
return err
}
f.SetUint(v)
case reflect.String:
f.SetString(env)
case reflect.Ptr:
if t := f.Type().Elem(); t.Kind() == reflect.Struct && t.PkgPath() == "net/url" && t.Name() == "URL" {
v, err := url.Parse(env)
if err != nil {
return err
}
f.Set(reflect.ValueOf(v))
}
}
return nil
}
// MustDecode calls Decode and terminates the process if any errors
// are encountered.
func MustDecode(target any) {
if err := Decode(target); err != nil {
FailureFunc(err)
}
}
// MustStrictDecode calls StrictDecode and terminates the process if any errors
// are encountered.
func MustStrictDecode(target any) {
if err := StrictDecode(target); err != nil {
FailureFunc(err)
}
}
//// Configuration info for Export
type ConfigInfo struct {
Field string
EnvVar string
Value string
DefaultValue string
HasDefault bool
Required bool
UsesEnv bool
}
type ConfigInfoSlice []*ConfigInfo
func (c ConfigInfoSlice) Less(i, j int) bool {
return c[i].EnvVar < c[j].EnvVar
}
func (c ConfigInfoSlice) Len() int {
return len(c)
}
func (c ConfigInfoSlice) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
// Returns a list of final configuration metadata sorted by envvar name
func Export(target any) ([]*ConfigInfo, error) {
s := reflect.ValueOf(target)
if s.Kind() != reflect.Ptr || s.IsNil() {
return nil, ErrInvalidTarget
}
cfg := []*ConfigInfo{}
s = s.Elem()
if s.Kind() != reflect.Struct {
return nil, ErrInvalidTarget
}
t := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
fName := t.Field(i).Name
fElem := f
if f.Kind() == reflect.Ptr {
fElem = f.Elem()
}
if fElem.Kind() == reflect.Struct {
ss := fElem.Addr().Interface()
subCfg, err := Export(ss)
if err != ErrInvalidTarget {
f = fElem
for _, v := range subCfg {
v.Field = fmt.Sprintf("%s.%s", fName, v.Field)
cfg = append(cfg, v)
}
}
}
tag := t.Field(i).Tag.Get("env")
if tag == "" {
continue
}
parts := strings.Split(tag, ",")
ci := &ConfigInfo{
Field: fName,
EnvVar: parts[0],
UsesEnv: os.Getenv(parts[0]) != "",
}
for _, o := range parts[1:] {
if strings.HasPrefix(o, "default=") {
ci.HasDefault = true
ci.DefaultValue = o[8:]
} else if strings.HasPrefix(o, "required") {
ci.Required = true
}
}
if f.Kind() == reflect.Ptr && f.IsNil() {
ci.Value = ""
} else if stringer, ok := f.Interface().(fmt.Stringer); ok {
ci.Value = stringer.String()
} else {
switch f.Kind() {
case reflect.Bool:
ci.Value = strconv.FormatBool(f.Bool())
case reflect.Float32, reflect.Float64:
bits := f.Type().Bits()
ci.Value = strconv.FormatFloat(f.Float(), 'f', -1, bits)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
ci.Value = strconv.FormatInt(f.Int(), 10)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
ci.Value = strconv.FormatUint(f.Uint(), 10)
case reflect.String:
ci.Value = f.String()
case reflect.Slice:
ci.Value = fmt.Sprintf("%v", f.Interface())
default:
// Unable to determine string format for value
return nil, ErrInvalidTarget
}
}
cfg = append(cfg, ci)
}
// No configuration tags found, assume invalid input
if len(cfg) == 0 {
return nil, ErrInvalidTarget
}
sort.Sort(ConfigInfoSlice(cfg))
return cfg, nil
}
@@ -0,0 +1,790 @@
package envdecode
import (
"encoding/json"
"fmt"
"math"
"net/url"
"os"
"reflect"
"sort"
"strconv"
"testing"
"time"
)
type nested struct {
String string `env:"TEST_STRING"`
}
type testConfig struct {
String string `env:"TEST_STRING"`
Int64 int64 `env:"TEST_INT64"`
Uint16 uint16 `env:"TEST_UINT16"`
Float64 float64 `env:"TEST_FLOAT64"`
Bool bool `env:"TEST_BOOL"`
Duration time.Duration `env:"TEST_DURATION"`
URL *url.URL `env:"TEST_URL"`
StringSlice []string `env:"TEST_STRING_SLICE"`
Int64Slice []int64 `env:"TEST_INT64_SLICE"`
Uint16Slice []uint16 `env:"TEST_UINT16_SLICE"`
Float64Slice []float64 `env:"TEST_FLOAT64_SLICE"`
BoolSlice []bool `env:"TEST_BOOL_SLICE"`
DurationSlice []time.Duration `env:"TEST_DURATION_SLICE"`
URLSlice []*url.URL `env:"TEST_URL_SLICE"`
UnsetString string `env:"TEST_UNSET_STRING"`
UnsetInt64 int64 `env:"TEST_UNSET_INT64"`
UnsetDuration time.Duration `env:"TEST_UNSET_DURATION"`
UnsetURL *url.URL `env:"TEST_UNSET_URL"`
UnsetSlice []string `env:"TEST_UNSET_SLICE"`
InvalidInt64 int64 `env:"TEST_INVALID_INT64"`
UnusedField string
unexportedField string
IgnoredPtr *bool `env:"TEST_BOOL"`
Nested nested
NestedPtr *nested
DecoderStruct decoderStruct `env:"TEST_DECODER_STRUCT"`
DecoderStructPtr *decoderStruct `env:"TEST_DECODER_STRUCT_PTR"`
DecoderString decoderString `env:"TEST_DECODER_STRING"`
UnmarshalerNumber unmarshalerNumber `env:"TEST_UNMARSHALER_NUMBER"`
DefaultInt int `env:"TEST_UNSET,asdf=asdf,default=1234"`
DefaultSliceInt []int `env:"TEST_UNSET,asdf=asdf,default=1"`
DefaultDuration time.Duration `env:"TEST_UNSET,asdf=asdf,default=24h"`
DefaultURL *url.URL `env:"TEST_UNSET,default=http://example.com"`
}
type testConfigNoSet struct {
Some string `env:"TEST_THIS_ENV_WILL_NOT_BE_SET"`
}
type testConfigRequired struct {
Required string `env:"TEST_REQUIRED,required"`
}
type testConfigRequiredDefault struct {
RequiredDefault string `env:"TEST_REQUIRED_DEFAULT,required,default=test"`
}
type testConfigOverride struct {
OverrideString string `env:"TEST_OVERRIDE_A;TEST_OVERRIDE_B,default=override_default"`
}
type testNoExportedFields struct {
// following unexported fields are used for tests
aString string `env:"TEST_STRING"` //nolint:structcheck,unused
anInt64 int64 `env:"TEST_INT64"` //nolint:structcheck,unused
aUint16 uint16 `env:"TEST_UINT16"` //nolint:structcheck,unused
aFloat64 float64 `env:"TEST_FLOAT64"` //nolint:structcheck,unused
aBool bool `env:"TEST_BOOL"` //nolint:structcheck,unused
}
type testNoTags struct {
String string
}
type decoderStruct struct {
String string
}
func (d *decoderStruct) Decode(env string) error {
return json.Unmarshal([]byte(env), &d)
}
type decoderString string
func (d *decoderString) Decode(env string) error {
r, l := []rune(env), len(env)
for i := 0; i < l/2; i++ {
r[i], r[l-1-i] = r[l-1-i], r[i]
}
*d = decoderString(r)
return nil
}
type unmarshalerNumber uint8
func (o *unmarshalerNumber) UnmarshalText(raw []byte) error {
n, err := strconv.ParseUint(string(raw), 8, 8) // parse text as octal number
if err != nil {
return err
}
*o = unmarshalerNumber(n)
return nil
}
func TestDecode(t *testing.T) {
int64Val := int64(-(1 << 50))
int64AsString := fmt.Sprintf("%d", int64Val)
piAsString := fmt.Sprintf("%.48f", math.Pi)
os.Setenv("TEST_STRING", "foo")
os.Setenv("TEST_INT64", int64AsString)
os.Setenv("TEST_UINT16", "60000")
os.Setenv("TEST_FLOAT64", piAsString)
os.Setenv("TEST_BOOL", "true")
os.Setenv("TEST_DURATION", "10m")
os.Setenv("TEST_URL", "https://example.com")
os.Setenv("TEST_INVALID_INT64", "asdf")
os.Setenv("TEST_STRING_SLICE", "foo,bar")
os.Setenv("TEST_INT64_SLICE", int64AsString+","+int64AsString)
os.Setenv("TEST_UINT16_SLICE", "60000,50000")
os.Setenv("TEST_FLOAT64_SLICE", piAsString+","+piAsString)
os.Setenv("TEST_BOOL_SLICE", "true, false, true")
os.Setenv("TEST_DURATION_SLICE", "10m, 20m")
os.Setenv("TEST_URL_SLICE", "https://example.com")
os.Setenv("TEST_DECODER_STRUCT", "{\"string\":\"foo\"}")
os.Setenv("TEST_DECODER_STRUCT_PTR", "{\"string\":\"foo\"}")
os.Setenv("TEST_DECODER_STRING", "oof")
os.Setenv("TEST_UNMARSHALER_NUMBER", "07")
var tc testConfig
tc.NestedPtr = &nested{}
tc.DecoderStructPtr = &decoderStruct{}
err := Decode(&tc)
if err != nil {
t.Fatal(err)
}
if tc.String != "foo" {
t.Fatalf(`Expected "foo", got "%s"`, tc.String)
}
if tc.Int64 != -(1 << 50) {
t.Fatalf("Expected %d, got %d", -(1 << 50), tc.Int64)
}
if tc.Uint16 != 60000 {
t.Fatalf("Expected 60000, got %d", tc.Uint16)
}
if tc.Float64 != math.Pi {
t.Fatalf("Expected %.48f, got %.48f", math.Pi, tc.Float64)
}
if !tc.Bool {
t.Fatal("Expected true, got false")
}
duration, _ := time.ParseDuration("10m")
if tc.Duration != duration {
t.Fatalf("Expected %d, got %d", duration, tc.Duration)
}
if tc.URL == nil {
t.Fatalf("Expected https://example.com, got nil")
} else if tc.URL.String() != "https://example.com" {
t.Fatalf("Expected https://example.com, got %s", tc.URL.String())
}
expectedStringSlice := []string{"foo", "bar"}
if !reflect.DeepEqual(tc.StringSlice, expectedStringSlice) {
t.Fatalf("Expected %s, got %s", expectedStringSlice, tc.StringSlice)
}
expectedInt64Slice := []int64{int64Val, int64Val}
if !reflect.DeepEqual(tc.Int64Slice, expectedInt64Slice) {
t.Fatalf("Expected %#v, got %#v", expectedInt64Slice, tc.Int64Slice)
}
expectedUint16Slice := []uint16{60000, 50000}
if !reflect.DeepEqual(tc.Uint16Slice, expectedUint16Slice) {
t.Fatalf("Expected %#v, got %#v", expectedUint16Slice, tc.Uint16Slice)
}
expectedFloat64Slice := []float64{math.Pi, math.Pi}
if !reflect.DeepEqual(tc.Float64Slice, expectedFloat64Slice) {
t.Fatalf("Expected %#v, got %#v", expectedFloat64Slice, tc.Float64Slice)
}
expectedBoolSlice := []bool{true, false, true}
if !reflect.DeepEqual(tc.BoolSlice, expectedBoolSlice) {
t.Fatalf("Expected %#v, got %#v", expectedBoolSlice, tc.BoolSlice)
}
duration2, _ := time.ParseDuration("20m")
expectedDurationSlice := []time.Duration{duration, duration2}
if !reflect.DeepEqual(tc.DurationSlice, expectedDurationSlice) {
t.Fatalf("Expected %s, got %s", expectedDurationSlice, tc.DurationSlice)
}
urlVal, _ := url.Parse("https://example.com")
expectedURLSlice := []*url.URL{urlVal}
if !reflect.DeepEqual(tc.URLSlice, expectedURLSlice) {
t.Fatalf("Expected %s, got %s", expectedURLSlice, tc.URLSlice)
}
if tc.UnsetString != "" {
t.Fatal("Got non-empty string unexpectedly")
}
if tc.UnsetInt64 != 0 {
t.Fatal("Got non-zero int unexpectedly")
}
if tc.UnsetDuration != time.Duration(0) {
t.Fatal("Got non-zero time.Duration unexpectedly")
}
if tc.UnsetURL != nil {
t.Fatal("Got non-zero *url.URL unexpectedly")
}
if len(tc.UnsetSlice) > 0 {
t.Fatal("Got not-empty string slice unexpectedly")
}
if tc.InvalidInt64 != 0 {
t.Fatal("Got non-zero int unexpectedly")
}
if tc.UnusedField != "" {
t.Fatal("Expected empty field")
}
if tc.unexportedField != "" {
t.Fatal("Expected empty field")
}
if tc.IgnoredPtr != nil {
t.Fatal("Expected nil pointer")
}
if tc.Nested.String != "foo" {
t.Fatalf(`Expected "foo", got "%s"`, tc.Nested.String)
}
if tc.NestedPtr.String != "foo" {
t.Fatalf(`Expected "foo", got "%s"`, tc.NestedPtr.String)
}
if tc.DefaultInt != 1234 {
t.Fatalf("Expected 1234, got %d", tc.DefaultInt)
}
expectedDefaultSlice := []int{1}
if !reflect.DeepEqual(tc.DefaultSliceInt, expectedDefaultSlice) {
t.Fatalf("Expected %d, got %d", expectedDefaultSlice, tc.DefaultSliceInt)
}
defaultDuration, _ := time.ParseDuration("24h")
if tc.DefaultDuration != defaultDuration {
t.Fatalf("Expected %d, got %d", defaultDuration, tc.DefaultInt)
}
if tc.DefaultURL.String() != "http://example.com" {
t.Fatalf("Expected http://example.com, got %s", tc.DefaultURL.String())
}
if tc.DecoderStruct.String != "foo" {
t.Fatalf("Expected foo, got %s", tc.DecoderStruct.String)
}
if tc.DecoderStructPtr.String != "foo" {
t.Fatalf("Expected foo, got %s", tc.DecoderStructPtr.String)
}
if tc.DecoderString != "foo" {
t.Fatalf("Expected foo, got %s", tc.DecoderString)
}
if tc.UnmarshalerNumber != 07 {
t.Fatalf("Expected 07, got %04o", tc.UnmarshalerNumber)
}
os.Setenv("TEST_REQUIRED", "required")
var tcr testConfigRequired
err = Decode(&tcr)
if err != nil {
t.Fatal(err)
}
if tcr.Required != "required" {
t.Fatalf("Expected \"required\", got %s", tcr.Required)
}
_, err = Export(&tcr)
if err != nil {
t.Fatal(err)
}
var tco testConfigOverride
err = Decode(&tco)
if err != nil {
t.Fatal(err)
}
if tco.OverrideString != "override_default" {
t.Fatalf(`Expected "override_default" but got %s`, tco.OverrideString)
}
os.Setenv("TEST_OVERRIDE_A", "override_a")
tco = testConfigOverride{}
err = Decode(&tco)
if err != nil {
t.Fatal(err)
}
if tco.OverrideString != "override_a" {
t.Fatalf(`Expected "override_a" but got %s`, tco.OverrideString)
}
os.Setenv("TEST_OVERRIDE_B", "override_b")
tco = testConfigOverride{}
err = Decode(&tco)
if err != nil {
t.Fatal(err)
}
if tco.OverrideString != "override_b" {
t.Fatalf(`Expected "override_b" but got %s`, tco.OverrideString)
}
}
func TestDecodeErrors(t *testing.T) {
var b bool
err := Decode(&b)
if err != ErrInvalidTarget {
t.Fatal("Should have gotten an error decoding into a bool")
}
var tc testConfig
err = Decode(tc) //nolint:govet
if err != ErrInvalidTarget {
t.Fatal("Should have gotten an error decoding into a non-pointer")
}
var tcp *testConfig
err = Decode(tcp)
if err != ErrInvalidTarget {
t.Fatal("Should have gotten an error decoding to a nil pointer")
}
var tnt testNoTags
err = Decode(&tnt)
if err != ErrNoTargetFieldsAreSet {
t.Fatal("Should have gotten an error decoding a struct with no tags")
}
var tcni testNoExportedFields
err = Decode(&tcni)
if err != ErrNoTargetFieldsAreSet {
t.Fatal("Should have gotten an error decoding a struct with no unexported fields")
}
var tcr testConfigRequired
os.Clearenv()
err = Decode(&tcr)
if err == nil {
t.Fatal("An error was expected but recieved:", err)
}
var tcns testConfigNoSet
err = Decode(&tcns)
if err != ErrNoTargetFieldsAreSet {
t.Fatal("Should have gotten an error decoding when no env variables are set")
}
missing := false
FailureFunc = func(err error) {
missing = true
}
MustDecode(&tcr)
if !missing {
t.Fatal("The FailureFunc should have been called but it was not")
}
var tcrd testConfigRequiredDefault
defer func() {
_ = recover()
}()
_ = Decode(&tcrd)
t.Fatal("This should not have been reached. A panic should have occured.")
}
func TestOnlyNested(t *testing.T) {
os.Setenv("TEST_STRING", "foo")
// No env vars in the outer level are ok, as long as they're
// in the inner struct.
var o struct {
Inner nested
}
if err := Decode(&o); err != nil {
t.Fatalf("Expected no error, got %s", err)
}
// No env vars in the inner levels are ok, as long as they're
// in the outer struct.
var o2 struct {
Inner noConfig
X string `env:"TEST_STRING"`
}
if err := Decode(&o2); err != nil {
t.Fatalf("Expected no error, got %s", err)
}
// No env vars in either outer or inner levels should result
// in error
var o3 struct {
Inner noConfig
}
if err := Decode(&o3); err != ErrNoTargetFieldsAreSet {
t.Fatalf("Expected ErrInvalidTarget, got %s", err)
}
}
func ExampleDecode() {
type Example struct {
// A string field, without any default
String string `env:"EXAMPLE_STRING"`
// A uint16 field, with a default value of 100
Uint16 uint16 `env:"EXAMPLE_UINT16,default=100"`
}
os.Setenv("EXAMPLE_STRING", "an example!")
var e Example
if err := Decode(&e); err != nil {
panic(err)
}
// If TEST_STRING is set, e.String will contain its value
fmt.Println(e.String)
// If TEST_UINT16 is set, e.Uint16 will contain its value.
// Otherwise, it will contain the default value, 100.
fmt.Println(e.Uint16)
// Output:
// an example!
// 100
}
//// Export tests
type testConfigExport struct {
String string `env:"TEST_STRING"`
Int64 int64 `env:"TEST_INT64"`
Uint16 uint16 `env:"TEST_UINT16"`
Float64 float64 `env:"TEST_FLOAT64"`
Bool bool `env:"TEST_BOOL"`
Duration time.Duration `env:"TEST_DURATION"`
URL *url.URL `env:"TEST_URL"`
StringSlice []string `env:"TEST_STRING_SLICE"`
UnsetString string `env:"TEST_UNSET_STRING"`
UnsetInt64 int64 `env:"TEST_UNSET_INT64"`
UnsetDuration time.Duration `env:"TEST_UNSET_DURATION"`
UnsetURL *url.URL `env:"TEST_UNSET_URL"`
UnusedField string
unexportedField string //nolint:structcheck,unused
IgnoredPtr *bool `env:"TEST_IGNORED_POINTER"`
Nested nestedConfigExport
NestedPtr *nestedConfigExportPointer
NestedPtrUnset *nestedConfigExportPointer
NestedTwice nestedTwiceConfig
NoConfig noConfig
NoConfigPtr *noConfig
NoConfigPtrSet *noConfig
RequiredInt int `env:"TEST_REQUIRED_INT,required"`
DefaultBool bool `env:"TEST_DEFAULT_BOOL,default=true"`
DefaultInt int `env:"TEST_DEFAULT_INT,default=1234"`
DefaultDuration time.Duration `env:"TEST_DEFAULT_DURATION,default=24h"`
DefaultURL *url.URL `env:"TEST_DEFAULT_URL,default=http://example.com"`
DefaultIntSet int `env:"TEST_DEFAULT_INT_SET,default=99"`
DefaultIntSlice []int `env:"TEST_DEFAULT_INT_SLICE,default=99"`
}
type nestedConfigExport struct {
String string `env:"TEST_NESTED_STRING"`
}
type nestedConfigExportPointer struct {
String string `env:"TEST_NESTED_STRING_POINTER"`
}
type noConfig struct {
Int int
}
type nestedTwiceConfig struct {
Nested nestedConfigInner
}
type nestedConfigInner struct {
String string `env:"TEST_NESTED_TWICE_STRING"`
}
type testConfigStrict struct {
InvalidInt64Strict int64 `env:"TEST_INVALID_INT64,strict,default=1"`
InvalidInt64Implicit int64 `env:"TEST_INVALID_INT64_IMPLICIT,default=1"`
Nested struct {
InvalidInt64Strict int64 `env:"TEST_INVALID_INT64_NESTED,strict,required"`
InvalidInt64Implicit int64 `env:"TEST_INVALID_INT64_NESTED_IMPLICIT,required"`
}
}
func TestInvalidStrict(t *testing.T) {
cases := []struct {
decoder func(any) error
rootValue string
nestedValue string
rootValueImplicit string
nestedValueImplicit string
pass bool
}{
{Decode, "1", "1", "1", "1", true},
{Decode, "1", "1", "1", "asdf", true},
{Decode, "1", "1", "asdf", "1", true},
{Decode, "1", "1", "asdf", "asdf", true},
{Decode, "1", "asdf", "1", "1", false},
{Decode, "asdf", "1", "1", "1", false},
{Decode, "asdf", "asdf", "1", "1", false},
{StrictDecode, "1", "1", "1", "1", true},
{StrictDecode, "asdf", "1", "1", "1", false},
{StrictDecode, "1", "asdf", "1", "1", false},
{StrictDecode, "1", "1", "asdf", "1", false},
{StrictDecode, "1", "1", "1", "asdf", false},
{StrictDecode, "asdf", "asdf", "1", "1", false},
{StrictDecode, "1", "asdf", "asdf", "1", false},
{StrictDecode, "1", "1", "asdf", "asdf", false},
{StrictDecode, "1", "asdf", "asdf", "asdf", false},
{StrictDecode, "asdf", "asdf", "asdf", "asdf", false},
}
for _, test := range cases {
os.Setenv("TEST_INVALID_INT64", test.rootValue)
os.Setenv("TEST_INVALID_INT64_NESTED", test.nestedValue)
os.Setenv("TEST_INVALID_INT64_IMPLICIT", test.rootValueImplicit)
os.Setenv("TEST_INVALID_INT64_NESTED_IMPLICIT", test.nestedValueImplicit)
var tc testConfigStrict
if err := test.decoder(&tc); test.pass != (err == nil) {
t.Fatalf("Have err=%s wanted pass=%v", err, test.pass)
}
}
}
func TestExport(t *testing.T) {
testFloat64 := fmt.Sprintf("%.48f", math.Pi)
testFloat64Output := strconv.FormatFloat(math.Pi, 'f', -1, 64)
testInt64 := fmt.Sprintf("%d", -(1 << 50))
os.Setenv("TEST_STRING", "foo")
os.Setenv("TEST_INT64", testInt64)
os.Setenv("TEST_UINT16", "60000")
os.Setenv("TEST_FLOAT64", testFloat64)
os.Setenv("TEST_BOOL", "true")
os.Setenv("TEST_DURATION", "10m")
os.Setenv("TEST_URL", "https://example.com")
os.Setenv("TEST_STRING_SLICE", "foo,bar")
os.Setenv("TEST_NESTED_STRING", "nest_foo")
os.Setenv("TEST_NESTED_STRING_POINTER", "nest_foo_ptr")
os.Setenv("TEST_NESTED_TWICE_STRING", "nest_twice_foo")
os.Setenv("TEST_REQUIRED_INT", "101")
os.Setenv("TEST_DEFAULT_INT_SET", "102")
os.Setenv("TEST_DEFAULT_INT_SLICE", "1,2,3")
var tc testConfigExport
tc.NestedPtr = &nestedConfigExportPointer{}
tc.NoConfigPtrSet = &noConfig{}
if err := Decode(&tc); err != nil {
t.Fatal(err)
}
rc, err := Export(&tc)
if err != nil {
t.Fatal(err)
}
expected := []*ConfigInfo{
&ConfigInfo{
Field: "String",
EnvVar: "TEST_STRING",
Value: "foo",
UsesEnv: true,
},
&ConfigInfo{
Field: "Int64",
EnvVar: "TEST_INT64",
Value: testInt64,
UsesEnv: true,
},
&ConfigInfo{
Field: "Uint16",
EnvVar: "TEST_UINT16",
Value: "60000",
UsesEnv: true,
},
&ConfigInfo{
Field: "Float64",
EnvVar: "TEST_FLOAT64",
Value: testFloat64Output,
UsesEnv: true,
},
&ConfigInfo{
Field: "Bool",
EnvVar: "TEST_BOOL",
Value: "true",
UsesEnv: true,
},
&ConfigInfo{
Field: "Duration",
EnvVar: "TEST_DURATION",
Value: "10m0s",
UsesEnv: true,
},
&ConfigInfo{
Field: "URL",
EnvVar: "TEST_URL",
Value: "https://example.com",
UsesEnv: true,
},
&ConfigInfo{
Field: "StringSlice",
EnvVar: "TEST_STRING_SLICE",
Value: "[foo bar]",
UsesEnv: true,
},
&ConfigInfo{
Field: "UnsetString",
EnvVar: "TEST_UNSET_STRING",
Value: "",
},
&ConfigInfo{
Field: "UnsetInt64",
EnvVar: "TEST_UNSET_INT64",
Value: "0",
},
&ConfigInfo{
Field: "UnsetDuration",
EnvVar: "TEST_UNSET_DURATION",
Value: "0s",
},
&ConfigInfo{
Field: "UnsetURL",
EnvVar: "TEST_UNSET_URL",
Value: "",
},
&ConfigInfo{
Field: "IgnoredPtr",
EnvVar: "TEST_IGNORED_POINTER",
Value: "",
},
&ConfigInfo{
Field: "Nested.String",
EnvVar: "TEST_NESTED_STRING",
Value: "nest_foo",
UsesEnv: true,
},
&ConfigInfo{
Field: "NestedPtr.String",
EnvVar: "TEST_NESTED_STRING_POINTER",
Value: "nest_foo_ptr",
UsesEnv: true,
},
&ConfigInfo{
Field: "NestedTwice.Nested.String",
EnvVar: "TEST_NESTED_TWICE_STRING",
Value: "nest_twice_foo",
UsesEnv: true,
},
&ConfigInfo{
Field: "RequiredInt",
EnvVar: "TEST_REQUIRED_INT",
Value: "101",
UsesEnv: true,
Required: true,
},
&ConfigInfo{
Field: "DefaultBool",
EnvVar: "TEST_DEFAULT_BOOL",
Value: "true",
DefaultValue: "true",
HasDefault: true,
},
&ConfigInfo{
Field: "DefaultInt",
EnvVar: "TEST_DEFAULT_INT",
Value: "1234",
DefaultValue: "1234",
HasDefault: true,
},
&ConfigInfo{
Field: "DefaultDuration",
EnvVar: "TEST_DEFAULT_DURATION",
Value: "24h0m0s",
DefaultValue: "24h",
HasDefault: true,
},
&ConfigInfo{
Field: "DefaultURL",
EnvVar: "TEST_DEFAULT_URL",
Value: "http://example.com",
DefaultValue: "http://example.com",
HasDefault: true,
},
&ConfigInfo{
Field: "DefaultIntSet",
EnvVar: "TEST_DEFAULT_INT_SET",
Value: "102",
DefaultValue: "99",
HasDefault: true,
UsesEnv: true,
},
&ConfigInfo{
Field: "DefaultIntSlice",
EnvVar: "TEST_DEFAULT_INT_SLICE",
Value: "[1 2 3]",
DefaultValue: "99",
HasDefault: true,
UsesEnv: true,
},
}
sort.Sort(ConfigInfoSlice(expected))
if len(rc) != len(expected) {
t.Fatalf("Have %d results, expected %d", len(rc), len(expected))
}
for n, v := range rc {
ci := expected[n]
if *ci != *v {
t.Fatalf("have %+v, expected %+v", v, ci)
}
}
}
+69
View File
@@ -0,0 +1,69 @@
package config
import (
"io/fs"
"os"
"path"
"strings"
gofig "github.com/gookit/config/v2"
gooyaml "github.com/gookit/config/v2/yaml"
"github.com/qsfera/server/pkg/config/defaults"
)
var (
// decoderConfigTagName sets the tag name to be used from the config structs
// currently we only support "yaml" because we only support config loading
// from yaml files and the yaml parser has no simple way to set a custom tag name to use
decoderConfigTagName = "yaml"
)
// BindSourcesToStructs assigns any config value from a config file / env variable to struct `dst`.
func BindSourcesToStructs(service string, dst any) error {
fileSystem := os.DirFS("/")
filePath := strings.TrimLeft(path.Join(defaults.BaseConfigPath(), service+".yaml"), "/")
return bindSourcesToStructs(fileSystem, filePath, service, dst)
}
func bindSourcesToStructs(fileSystem fs.FS, filePath, service string, dst any) error {
cnf := gofig.NewWithOptions(service)
cnf.WithOptions(func(options *gofig.Options) {
options.ParseEnv = true
options.DecoderConfig.TagName = decoderConfigTagName
})
cnf.AddDriver(gooyaml.Driver)
yamlContent, err := fs.ReadFile(fileSystem, filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
_ = cnf.LoadSources("yaml", yamlContent)
err = cnf.BindStruct("", &dst)
if err != nil {
return err
}
return nil
}
// LocalEndpoint returns the local endpoint for a given protocol and address.
// Use it when configuring the reva runtime to get a service endpoint in the same
// runtime, e.g. a gateway talking to an authregistry service.
func LocalEndpoint(protocol, addr string) string {
localEndpoint := addr
switch protocol {
case "tcp":
parts := strings.SplitN(addr, ":", 2)
if len(parts) == 2 {
localEndpoint = "dns:127.0.0.1:" + parts[1]
}
case "unix":
localEndpoint = "unix:" + addr
}
return localEndpoint
}
+189
View File
@@ -0,0 +1,189 @@
package config
import (
"testing"
"testing/fstest"
"gotest.tools/v3/assert"
)
type TestConfig struct {
A string `yaml:"a"`
B string `yaml:"b"`
C string `yaml:"c"`
}
func TestBindSourcesToStructs(t *testing.T) {
// setup test env
yaml := `
a: "${FOO_VAR|no-foo}"
b: "${BAR_VAR|no-bar}"
c: "${CODE_VAR|code}"
`
filePath := "etc/qsfera/foo.yaml"
fs := fstest.MapFS{
filePath: {Data: []byte(yaml)},
}
// perform test
c := TestConfig{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "no-foo")
assert.Equal(t, c.B, "no-bar")
assert.Equal(t, c.C, "code")
}
func TestBindSourcesToStructs_UnknownFile(t *testing.T) {
// setup test env
filePath := "etc/qsfera/foo.yaml"
fs := fstest.MapFS{}
// perform test
c := TestConfig{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "")
assert.Equal(t, c.B, "")
assert.Equal(t, c.C, "")
}
func TestBindSourcesToStructs_NoEnvVar(t *testing.T) {
// setup test env
yaml := `
token_manager:
jwt_secret: f%LovwC6xnKkHhc.!.Lp4ZYpQDIO7=d@
machine_auth_api_key: jG&%ZCmCSYqT#Yi$9y28o5u84ZMo2UBf
system_user_api_key: wqxH7FZHv5gifuLIzxqdyaZOCo2s^yl1
transfer_secret: $1^2xspR1WHussV16knaJ$x@X*XLPL%y
system_user_id: 4d0bf32c-83ee-4703-bd43-5e0d6b78215b
admin_user_id: e2fca2b3-992b-47d5-8ecd-3312418ed3d7
graph:
application:
id: 4fdff90c-d13c-47ab-8227-bbd3e6dbee3c
events:
tls_insecure: true
spaces:
insecure: true
identity:
ldap:
bind_password: $ZZ8fSJR&YA02jBBPx6IRCzW0kVZ#cBO
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
idp:
ldap:
bind_password: kWJGC6WRY1wQ+e8Bmt--=-3r6gp0CNVS
idm:
service_user_passwords:
admin_password: admin
idm_password: $ZZ8fSJR&YA02jBBPx6IRCzW0kVZ#cBO
reva_password: c68JL=V$c@0GHs!%eSb8r&Ps3rgzKnXJ
idp_password: kWJGC6WRY1wQ+e8Bmt--=-3r6gp0CNVS
proxy:
oidc:
insecure: true
insecure_backends: true
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
frontend:
app_handler:
insecure: true
archiver:
insecure: true
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
ocdav:
insecure: true
auth_basic:
auth_providers:
ldap:
bind_password: c68JL=V$c@0GHs!%eSb8r&Ps3rgzKnXJ
auth_bearer:
auth_providers:
oidc:
insecure: true
users:
drivers:
ldap:
bind_password: c68JL=V$c@0GHs!%eSb8r&Ps3rgzKnXJ
groups:
drivers:
ldap:
bind_password: c68JL=V$c@0GHs!%eSb8r&Ps3rgzKnXJ
ocm:
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
thumbnails:
thumbnail:
transfer_secret: 0N05@YXB.h3e@lsVfksL4YxwQC9aE5A.
webdav_allow_insecure: true
cs3_allow_insecure: true
search:
events:
tls_insecure: true
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
audit:
events:
tls_insecure: true
settings:
service_account_ids:
- c05389b2-d94c-4d01-a9b5-a2f97952cc14
sharing:
events:
tls_insecure: true
storage_users:
events:
tls_insecure: true
mount_id: 64fdfb03-22ff-4788-be4d-d7731a475683
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
notifications:
notifications:
events:
tls_insecure: true
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
nats:
nats:
tls_skip_verify_client_cert: true
gateway:
storage_registry:
storage_users_mount_id: 64fdfb03-22ff-4788-be4d-d7731a475683
userlog:
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
auth_service:
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
clientlog:
service_account:
service_account_id: c05389b2-d94c-4d01-a9b5-a2f97952cc14
service_account_secret: GW5.x1vDM&+NPRi++eV@.P7Tms4vj!=s
`
filePath := "etc/qsfera/foo.yaml"
fs := fstest.MapFS{
filePath: {Data: []byte(yaml)},
}
// perform test
c := Config{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.Graph.Identity.LDAP.BindPassword, "$ZZ8fSJR&YA02jBBPx6IRCzW0kVZ#cBO")
}
+148
View File
@@ -0,0 +1,148 @@
package parser
import (
"errors"
"github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/pkg/config/envdecode"
"github.com/qsfera/server/pkg/shared"
"github.com/qsfera/server/pkg/structs"
)
// ParseConfig loads the КуСфера configuration and
// copies applicable parts into the commons part, from
// where the services can copy it into their own config
func ParseConfig(cfg *config.Config, skipValidate bool) error {
err := config.BindSourcesToStructs("qsfera", cfg)
if err != nil {
return err
}
EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
EnsureCommons(cfg)
if skipValidate {
return nil
}
return Validate(cfg)
}
// EnsureDefaults ensures that all pointers in the
// КуСфера config (not the services configs) are initialized
func EnsureDefaults(cfg *config.Config) {
if cfg.Log == nil {
cfg.Log = &shared.Log{}
}
if cfg.TokenManager == nil {
cfg.TokenManager = &shared.TokenManager{}
}
if cfg.Cache == nil {
cfg.Cache = &shared.Cache{}
}
if cfg.GRPCClientTLS == nil {
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
}
if cfg.GRPCServiceTLS == nil {
cfg.GRPCServiceTLS = &shared.GRPCServiceTLS{}
}
if cfg.Reva == nil {
cfg.Reva = &shared.Reva{}
}
}
// EnsureCommons copies applicable parts of the КуСфера config into the commons part
func EnsureCommons(cfg *config.Config) {
// ensure the commons part is initialized
if cfg.Commons == nil {
cfg.Commons = &shared.Commons{}
}
cfg.Commons.Log = structs.CopyOrZeroValue(cfg.Log)
cfg.Commons.Cache = structs.CopyOrZeroValue(cfg.Cache)
if cfg.GRPCClientTLS != nil {
cfg.Commons.GRPCClientTLS = cfg.GRPCClientTLS
}
if cfg.GRPCServiceTLS != nil {
cfg.Commons.GRPCServiceTLS = cfg.GRPCServiceTLS
}
cfg.Commons.HTTPServiceTLS = cfg.HTTPServiceTLS
cfg.Commons.TokenManager = structs.CopyOrZeroValue(cfg.TokenManager)
// copy machine auth api key to the commons part if set
if cfg.MachineAuthAPIKey != "" {
cfg.Commons.MachineAuthAPIKey = cfg.MachineAuthAPIKey
}
if cfg.SystemUserAPIKey != "" {
cfg.Commons.SystemUserAPIKey = cfg.SystemUserAPIKey
}
// copy transfer secret to the commons part if set
if cfg.TransferSecret != "" {
cfg.Commons.TransferSecret = cfg.TransferSecret
}
// make sure url signing secret is set and copy it to the commons part
// fall back to transfer secret for url signing secret to avoid
// issues when upgrading from an older release
if cfg.URLSigningSecret == "" {
cfg.URLSigningSecret = cfg.TransferSecret
}
cfg.Commons.URLSigningSecret = cfg.URLSigningSecret
// copy metadata user id to the commons part if set
if cfg.SystemUserID != "" {
cfg.Commons.SystemUserID = cfg.SystemUserID
}
// copy admin user id to the commons part if set
if cfg.AdminUserID != "" {
cfg.Commons.AdminUserID = cfg.AdminUserID
}
if cfg.QsferaURL != "" {
cfg.Commons.QsferaURL = cfg.QsferaURL
}
cfg.Commons.Reva = structs.CopyOrZeroValue(cfg.Reva)
}
// Validate checks that all required configs are set. If a required config value
// is missing an error will be returned.
func Validate(cfg *config.Config) error {
if cfg.TokenManager.JWTSecret == "" {
return shared.MissingJWTTokenError("qsfera")
}
if cfg.TransferSecret == "" {
return shared.MissingRevaTransferSecretError("qsfera")
}
if cfg.URLSigningSecret == "" {
return shared.MissingURLSigningSecret("qsfera")
}
if cfg.MachineAuthAPIKey == "" {
return shared.MissingMachineAuthApiKeyError("qsfera")
}
if cfg.SystemUserID == "" {
return shared.MissingSystemUserID("qsfera")
}
return nil
}