diff --git a/go.mod b/go.mod index 8dffa4741..11d438cc6 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.4.0 github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 - github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960 + github.com/cs3org/reva/v2 v2.12.1-0.20230427075231-7842414d18e1 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 690987bf6..00ecd3d2f 100644 --- a/go.sum +++ b/go.sum @@ -627,8 +627,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960 h1:QEQ9tu4S835dmRUryRj0JTQJPBg1AgPxXyVHrPtOY4U= -github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960/go.mod h1:VxBmpOvIKlgKLPOsHun+fABopzX+3ZELPAp3N5bQMsM= +github.com/cs3org/reva/v2 v2.12.1-0.20230427075231-7842414d18e1 h1:p563+4bqdVSYPtDeo6ikOEbciU/mjbYhrei2zCjzxkw= +github.com/cs3org/reva/v2 v2.12.1-0.20230427075231-7842414d18e1/go.mod h1:VxBmpOvIKlgKLPOsHun+fABopzX+3ZELPAp3N5bQMsM= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/services/frontend/pkg/config/config.go b/services/frontend/pkg/config/config.go index bf0e6e445..a9019e7f7 100644 --- a/services/frontend/pkg/config/config.go +++ b/services/frontend/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "context" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/shared" ) @@ -123,12 +124,12 @@ type OCS struct { SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of an ocis resource. Note that the path must start with '/'."` HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE;FRONTEND_OCS_HOME_NAMESPACE" desc:"Homespace namespace identifier." deprecationVersion:"3.0" removalVersion:"4.0.0" deprecationInfo:"FRONTEND_OCS_HOME_NAMESPACE changing name for consistency" deprecationReplacement:"FRONTEND_OCS_PERSONAL_NAMESPACE"` AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}."` - ResourceInfoCacheType string `yaml:"resource_info_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_RESOURCE_INFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - ResourceInfoCacheNodes []string `yaml:"resource_info_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_RESOURCE_INFO_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` - ResourceInfoCacheDatabase string `yaml:"resource_info_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - ResourceInfoCacheTable string `yaml:"resource_info_cache_table" env:"FRONTEND_OCS_RESOURCE_INFO_CACHE_TABLE" desc:"The database table the store should use."` - ResourceInfoCacheTTL int `yaml:"resource_info_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_RESOURCE_INFO_CACHE_TTL" desc:"Max TTL in seconds for the resource info cache."` - ResourceInfoCacheSize int `yaml:"resource_info_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_RESOURCE_INFO_CACHE_SIZE" desc:"Max number of entries to hold in the cache."` + StatCacheType string `yaml:"stat_cache_type" env:"OCIS_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` + StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use."` + StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` + StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;FRONTEND_OCS_STAT_CACHE_SIZE" desc:"Max number of entries to hold in the cache."` CacheWarmupDriver string `yaml:"cache_warmup_driver,omitempty"` // not supported by the oCIS product, therefore not part of docs CacheWarmupDrivers CacheWarmupDrivers `yaml:"cache_warmup_drivers,omitempty"` // not supported by the oCIS product, therefore not part of docs EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders."` diff --git a/services/frontend/pkg/config/defaults/defaultconfig.go b/services/frontend/pkg/config/defaults/defaultconfig.go index 50fa0ca0b..48f03545b 100644 --- a/services/frontend/pkg/config/defaults/defaultconfig.go +++ b/services/frontend/pkg/config/defaults/defaultconfig.go @@ -1,6 +1,8 @@ package defaults import ( + "time" + "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/owncloud/ocis/v2/ocis-pkg/structs" "github.com/owncloud/ocis/v2/services/frontend/pkg/config" @@ -102,13 +104,13 @@ func DefaultConfig() *config.Config { Prefix: "data", }, OCS: config.OCS{ - Prefix: "ocs", - SharePrefix: "/Shares", - HomeNamespace: "/users/{{.Id.OpaqueId}}", - AdditionalInfoAttribute: "{{.Mail}}", - ResourceInfoCacheType: "memory", - ResourceInfoCacheDatabase: "ocis", - ResourceInfoCacheTTL: 0, + Prefix: "ocs", + SharePrefix: "/Shares", + HomeNamespace: "/users/{{.Id.OpaqueId}}", + AdditionalInfoAttribute: "{{.Mail}}", + StatCacheType: "noop", + StatCacheDatabase: "ocis", + StatCacheTTL: 300 * time.Second, }, Middleware: config.Middleware{ Auth: config.Auth{ diff --git a/services/frontend/pkg/revaconfig/config.go b/services/frontend/pkg/revaconfig/config.go index cda78110e..adff5e1bd 100644 --- a/services/frontend/pkg/revaconfig/config.go +++ b/services/frontend/pkg/revaconfig/config.go @@ -4,6 +4,7 @@ import ( "net/url" "path" "strconv" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/version" "github.com/owncloud/ocis/v2/services/frontend/pkg/config" @@ -146,20 +147,20 @@ func FrontendConfigFromStruct(cfg *config.Config) (map[string]interface{}, error "insecure": true, }, "ocs": map[string]interface{}{ - "storage_registry_svc": cfg.Reva.Address, - "share_prefix": cfg.OCS.SharePrefix, - "home_namespace": cfg.OCS.HomeNamespace, - "resource_info_cache_ttl": cfg.OCS.ResourceInfoCacheTTL, - "resource_info_cache_size": cfg.OCS.ResourceInfoCacheSize, - "resource_info_cache_store": cfg.OCS.ResourceInfoCacheType, - "resource_info_cache_nodes": cfg.OCS.ResourceInfoCacheNodes, - "resource_info_cache_database": cfg.OCS.ResourceInfoCacheDatabase, - "resource_info_cache_table": cfg.OCS.ResourceInfoCacheTable, - "prefix": cfg.OCS.Prefix, - "additional_info_attribute": cfg.OCS.AdditionalInfoAttribute, - "machine_auth_apikey": cfg.MachineAuthAPIKey, - "enable_denials": cfg.OCS.EnableDenials, - "cache_warmup_driver": cfg.OCS.CacheWarmupDriver, + "storage_registry_svc": cfg.Reva.Address, + "share_prefix": cfg.OCS.SharePrefix, + "home_namespace": cfg.OCS.HomeNamespace, + "stat_cache_ttl": cfg.OCS.StatCacheTTL / time.Second, + "stat_cache_size": cfg.OCS.StatCacheSize, + "stat_cache_store": cfg.OCS.StatCacheType, + "stat_cache_nodes": cfg.OCS.StatCacheNodes, + "stat_cache_database": cfg.OCS.StatCacheDatabase, + "stat_cache_table": cfg.OCS.StatCacheTable, + "prefix": cfg.OCS.Prefix, + "additional_info_attribute": cfg.OCS.AdditionalInfoAttribute, + "machine_auth_apikey": cfg.MachineAuthAPIKey, + "enable_denials": cfg.OCS.EnableDenials, + "cache_warmup_driver": cfg.OCS.CacheWarmupDriver, "cache_warmup_drivers": map[string]interface{}{ "cbox": map[string]interface{}{ "db_username": cfg.OCS.CacheWarmupDrivers.CBOX.DBUsername, diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index fc20663dd..2527173d5 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -89,13 +89,19 @@ type StorageRegistry struct { // Cache holds cache config type Cache struct { - Store string `yaml:"store" env:"OCIS_CACHE_STORE;GATEWAY_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` - Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` - Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."` - StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` - ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."` - ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` - CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."` - CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` + StatCacheStore string `yaml:"stat_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_STAT_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` + StatCacheDatabase string `yaml:"stat_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` + StatCacheSize int `yaml:"stat_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` + ProviderCacheStore string `yaml:"provider_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_PROVIDER_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + ProviderCacheNodes []string `yaml:"provider_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_PROVIDER_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` + ProviderCacheDatabase string `yaml:"provider_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + ProviderCacheTTL time.Duration `yaml:"provider_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_PROVIDER_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` + ProviderCacheSize int `yaml:"provider_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_PROVIDER_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` + CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OCIS_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OCIS_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` + CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OCIS_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` + CreateHomeCacheSize int `yaml:"create_home_cache_size" env:"OCIS_CACHE_SIZE;GATEWAY_CREATE_HOME_CACHE_SIZE" desc:"The maximum quantity of items in the cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` } diff --git a/services/gateway/pkg/config/defaults/defaultconfig.go b/services/gateway/pkg/config/defaults/defaultconfig.go index ca71c5b3d..f776c55a2 100644 --- a/services/gateway/pkg/config/defaults/defaultconfig.go +++ b/services/gateway/pkg/config/defaults/defaultconfig.go @@ -1,6 +1,8 @@ package defaults import ( + "time" + "github.com/owncloud/ocis/v2/ocis-pkg/shared" "github.com/owncloud/ocis/v2/ocis-pkg/structs" "github.com/owncloud/ocis/v2/services/gateway/pkg/config" @@ -37,11 +39,15 @@ func DefaultConfig() *config.Config { DisableHomeCreationOnLogin: true, TransferExpires: 24 * 60 * 60, Cache: config.Cache{ - Store: "memory", - Database: "ocis", - StatCacheTTL: 300, - ProviderCacheTTL: 300, - CreateHomeCacheTTL: 300, + StatCacheStore: "noop", + StatCacheDatabase: "ocis", + StatCacheTTL: 300 * time.Second, + ProviderCacheStore: "noop", + ProviderCacheDatabase: "ocis", + ProviderCacheTTL: 300 * time.Second, + CreateHomeCacheStore: "noop", + CreateHomeCacheDatabase: "ocis", + CreateHomeCacheTTL: 300 * time.Second, }, FrontendPublicURL: "https://localhost:9200", diff --git a/services/gateway/pkg/revaconfig/config.go b/services/gateway/pkg/revaconfig/config.go index 058551726..dd48181c2 100644 --- a/services/gateway/pkg/revaconfig/config.go +++ b/services/gateway/pkg/revaconfig/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "strings" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/log" @@ -58,15 +59,21 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]i "transfer_shared_secret": cfg.TransferSecret, "transfer_expires": cfg.TransferExpires, // cache and TTLs - "cache_store": cfg.Cache.Store, - "cache_nodes": cfg.Cache.Nodes, - "cache_database": cfg.Cache.Database, - "stat_cache_ttl": cfg.Cache.StatCacheTTL, - "stat_cache_size": cfg.Cache.StatCacheSize, - "provider_cache_ttl": cfg.Cache.ProviderCacheTTL, - "provider_cache_size": cfg.Cache.ProviderCacheSize, - "create_home_cache_ttl": cfg.Cache.CreateHomeCacheTTL, - "create_home_cache_size": cfg.Cache.CreateHomeCacheSize, + "stat_cache_store": cfg.Cache.StatCacheStore, + "stat_cache_nodes": cfg.Cache.StatCacheNodes, + "stat_cache_database": cfg.Cache.StatCacheDatabase, + "stat_cache_ttl": cfg.Cache.StatCacheTTL / time.Second, + "stat_cache_size": cfg.Cache.StatCacheSize, + "provider_cache_store": cfg.Cache.ProviderCacheStore, + "provider_cache_nodes": cfg.Cache.ProviderCacheNodes, + "provider_cache_database": cfg.Cache.ProviderCacheDatabase, + "provider_cache_ttl": cfg.Cache.ProviderCacheTTL / time.Second, + "provider_cache_size": cfg.Cache.ProviderCacheSize, + "create_home_cache_store": cfg.Cache.CreateHomeCacheStore, + "create_home_cache_nodes": cfg.Cache.CreateHomeCacheNodes, + "create_home_cache_database": cfg.Cache.CreateHomeCacheDatabase, + "create_home_cache_ttl": cfg.Cache.CreateHomeCacheTTL / time.Second, + "create_home_cache_size": cfg.Cache.CreateHomeCacheSize, }, "authregistry": map[string]interface{}{ "driver": "static", diff --git a/services/graph/pkg/config/defaults/defaultconfig.go b/services/graph/pkg/config/defaults/defaultconfig.go index db1cbf5a1..eff3b8477 100644 --- a/services/graph/pkg/config/defaults/defaultconfig.go +++ b/services/graph/pkg/config/defaults/defaultconfig.go @@ -93,7 +93,7 @@ func DefaultConfig() *config.Config { }, Cache: &config.Cache{ Store: "memory", - Database: "graph", + Database: "ocis", Table: "roles", TTL: time.Hour * 336, }, diff --git a/services/storage-system/pkg/config/config.go b/services/storage-system/pkg/config/config.go index 170729a80..ccc4ac1c9 100644 --- a/services/storage-system/pkg/config/config.go +++ b/services/storage-system/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/shared" ) +// Config holds Config config type Config struct { Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service Service Service `yaml:"-"` @@ -24,14 +25,16 @@ type Config struct { SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` - Cache Cache `yaml:"cache"` - Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service."` - Drivers Drivers `yaml:"drivers"` - DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service."` + FileMetadataCache Cache `yaml:"cache"` + Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service."` + Drivers Drivers `yaml:"drivers"` + DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service."` Supervised bool `yaml:"-"` Context context.Context `yaml:"-"` } + +// Tracing holds Tracing config type Tracing struct { Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_SYSTEM_TRACING_ENABLED" desc:"Activates tracing."` Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_SYSTEM_TRACING_TYPE" desc:"The type of tracing. Defaults to \"\", which is the same as \"jaeger\". Allowed tracing types are \"jaeger\" and \"\" as of now."` @@ -39,6 +42,7 @@ type Tracing struct { Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_SYSTEM_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."` } +// Log holds Log config type Log struct { Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_SYSTEM_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."` Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_SYSTEM_LOG_PRETTY" desc:"Activates pretty log output."` @@ -46,10 +50,12 @@ type Log struct { File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_SYSTEM_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` } +// Service holds Service config type Service struct { Name string `yaml:"-"` } +// Debug holds Debug config type Debug struct { Addr string `yaml:"addr" env:"STORAGE_SYSTEM_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` Token string `yaml:"token" env:"STORAGE_SYSTEM_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint"` @@ -57,6 +63,7 @@ type Debug struct { Zpages bool `yaml:"zpages" env:"STORAGE_SYSTEM_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` } +// GRPCConfig holds GRPCConfig config type GRPCConfig struct { Addr string `yaml:"addr" env:"STORAGE_SYSTEM_GRPC_ADDR" desc:"The bind address of the GRPC service."` TLS *shared.GRPCServiceTLS `yaml:"tls"` @@ -64,16 +71,19 @@ type GRPCConfig struct { Protocol string `yaml:"protocol" env:"STORAGE_SYSTEM_GRPC_PROTOCOL" desc:"The transport protocol of the GPRC service."` } +// HTTPConfig holds HTTPConfig config type HTTPConfig struct { Addr string `yaml:"addr" env:"STORAGE_SYSTEM_HTTP_ADDR" desc:"The bind address of the HTTP service."` Namespace string `yaml:"-"` Protocol string `yaml:"protocol" env:"STORAGE_SYSTEM_HTTP_PROTOCOL" desc:"The transport protocol of the HTTP service."` } +// Drivers holds Drivers config type Drivers struct { OCIS OCISDriver `yaml:"ocis"` } +// OCISDriver holds ocis Driver config type OCISDriver struct { MetadataBackend string `yaml:"metadata_backend" env:"OCIS_DECOMPOSEDFS_METADATA_BACKEND;STORAGE_SYSTEM_OCIS_METADATA_BACKEND" desc:"The backend to use for storing metadata. Supported values are 'xattrs' and 'ini'. The setting 'xattrs' uses extended attributes to store file metadata while 'ini' uses a dedicated file to store file metadata. Defaults to 'xattrs'."` // Root is the absolute path to the location of the data diff --git a/services/storage-system/pkg/config/defaults/defaultconfig.go b/services/storage-system/pkg/config/defaults/defaultconfig.go index 0f53d22ac..0d98d676f 100644 --- a/services/storage-system/pkg/config/defaults/defaultconfig.go +++ b/services/storage-system/pkg/config/defaults/defaultconfig.go @@ -2,6 +2,7 @@ package defaults import ( "path/filepath" + "time" "github.com/owncloud/ocis/v2/ocis-pkg/config/defaults" "github.com/owncloud/ocis/v2/ocis-pkg/shared" @@ -50,9 +51,10 @@ func DefaultConfig() *config.Config { LockCycleDurationFactor: 30, }, }, - Cache: config.Cache{ + FileMetadataCache: config.Cache{ Store: "memory", Database: "ocis", + TTL: 24 * 60 * time.Second, }, } } diff --git a/services/storage-system/pkg/revaconfig/config.go b/services/storage-system/pkg/revaconfig/config.go index 1c89fb27a..c170893aa 100644 --- a/services/storage-system/pkg/revaconfig/config.go +++ b/services/storage-system/pkg/revaconfig/config.go @@ -1,6 +1,8 @@ package revaconfig import ( + "time" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/owncloud/ocis/v2/services/storage-system/pkg/config" ) @@ -166,11 +168,11 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} { "cache_database": "system", }, "filemetadatacache": map[string]interface{}{ - "cache_store": cfg.Cache.Store, - "cache_nodes": cfg.Cache.Nodes, - "cache_database": cfg.Cache.Database, - "cache_ttl": cfg.Cache.TTL, - "cache_size": cfg.Cache.Size, + "cache_store": cfg.FileMetadataCache.Store, + "cache_nodes": cfg.FileMetadataCache.Nodes, + "cache_database": cfg.FileMetadataCache.Database, + "cache_ttl": cfg.FileMetadataCache.TTL / time.Second, + "cache_size": cfg.FileMetadataCache.Size, }, }, } diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index fa529c15e..fd160cef3 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -175,7 +175,7 @@ type StatCache struct { Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_STAT_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_STAT_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_STAT_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` } @@ -184,7 +184,7 @@ type FilemetadataCache struct { Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_FILEMETADATA_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_FILEMETADATA_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` - TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '10s' (10 seconds)."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_FILEMETADATA_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '24h' (24 hours)."` Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` } diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 3a75c2eed..93313a3c5 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -90,12 +90,14 @@ func DefaultConfig() *config.Config { EnableTLS: false, }, StatCache: config.StatCache{ - Store: "memory", + Store: "noop", Database: "ocis", + TTL: 300 * time.Second, }, FilemetadataCache: config.FilemetadataCache{ Store: "memory", Database: "ocis", + TTL: 24 * 60 * time.Second, }, Tasks: config.Tasks{ PurgeTrashBin: config.PurgeTrashBin{ diff --git a/services/storage-users/pkg/revaconfig/config.go b/services/storage-users/pkg/revaconfig/config.go index 5fe55991e..7e77553a2 100644 --- a/services/storage-users/pkg/revaconfig/config.go +++ b/services/storage-users/pkg/revaconfig/config.go @@ -1,6 +1,8 @@ package revaconfig import ( + "time" + "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" ) @@ -76,7 +78,7 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, "cache_table": "stat", }, @@ -84,7 +86,7 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, "cache_table": "stat", }, @@ -92,7 +94,7 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, "cache_table": "stat", }, diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index 234e79cb8..d0e152035 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -1,6 +1,10 @@ package revaconfig -import "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" +import ( + "time" + + "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" +) // EOS is the config mapping for the EOS storage driver func EOS(cfg *config.Config) map[string]interface{} { @@ -130,14 +134,14 @@ func Ocis(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, }, "filemetadatacache": map[string]interface{}{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, "events": map[string]interface{}{ @@ -177,14 +181,14 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, }, "filemetadatacache": map[string]interface{}{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, } @@ -229,14 +233,14 @@ func S3NG(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, }, "filemetadatacache": map[string]interface{}{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, "events": map[string]interface{}{ @@ -280,14 +284,14 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} { "cache_store": cfg.StatCache.Store, "cache_nodes": cfg.StatCache.Nodes, "cache_database": cfg.StatCache.Database, - "cache_ttl": cfg.StatCache.TTL, + "cache_ttl": cfg.StatCache.TTL / time.Second, "cache_size": cfg.StatCache.Size, }, "filemetadatacache": map[string]interface{}{ "cache_store": cfg.FilemetadataCache.Store, "cache_nodes": cfg.FilemetadataCache.Nodes, "cache_database": cfg.FilemetadataCache.Database, - "cache_ttl": cfg.FilemetadataCache.TTL, + "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, } diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go index 44d6d9524..2dc2d44da 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/gateway.go @@ -68,15 +68,21 @@ type config struct { DataTransfersFolder string `mapstructure:"data_transfers_folder"` TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent - CacheStore string `mapstructure:"cache_store"` - CacheNodes []string `mapstructure:"cache_nodes"` - CacheDatabase string `mapstructure:"cache_database"` - CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"` - CreateHomeCacheSize int `mapstructure:"create_home_cache_size"` - ProviderCacheTTL int `mapstructure:"provider_cache_ttl"` - ProviderCacheSize int `mapstructure:"provider_cache_size"` + StatCacheStore string `mapstructure:"stat_cache_store"` + StatCacheNodes []string `mapstructure:"stat_cache_nodes"` + StatCacheDatabase string `mapstructure:"stat_cache_database"` StatCacheTTL int `mapstructure:"stat_cache_ttl"` StatCacheSize int `mapstructure:"stat_cache_size"` + CreateHomeCacheStore string `mapstructure:"create_home_cache_store"` + CreateHomeCacheNodes []string `mapstructure:"create_home_cache_nodes"` + CreateHomeCacheDatabase string `mapstructure:"create_home_cache_database"` + CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"` + CreateHomeCacheSize int `mapstructure:"create_home_cache_size"` + ProviderCacheStore string `mapstructure:"provider_cache_store"` + ProviderCacheNodes []string `mapstructure:"provider_cache_nodes"` + ProviderCacheDatabase string `mapstructure:"provider_cache_database"` + ProviderCacheTTL int `mapstructure:"provider_cache_ttl"` + ProviderCacheSize int `mapstructure:"provider_cache_size"` UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"` } @@ -124,12 +130,28 @@ func (c *config) init() { } // caching needs to be explicitly enabled - if c.CacheStore == "" { - c.CacheStore = "noop" + if c.StatCacheStore == "" { + c.StatCacheStore = "noop" } - if c.CacheDatabase == "" { - c.CacheDatabase = "reva" + if c.StatCacheDatabase == "" { + c.StatCacheDatabase = "reva" + } + + if c.ProviderCacheStore == "" { + c.ProviderCacheStore = "noop" + } + + if c.ProviderCacheDatabase == "" { + c.ProviderCacheDatabase = "reva" + } + + if c.CreateHomeCacheStore == "" { + c.CreateHomeCacheStore = "noop" + } + + if c.CreateHomeCacheDatabase == "" { + c.CreateHomeCacheDatabase = "reva" } } @@ -169,10 +191,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { c: c, dataGatewayURL: *u, tokenmgr: tokenManager, - statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize), - providerCache: cache.GetProviderCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize), - createHomeCache: cache.GetCreateHomeCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), - createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), + statCache: cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize), + providerCache: cache.GetProviderCache(c.ProviderCacheStore, c.ProviderCacheNodes, c.ProviderCacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize), + createHomeCache: cache.GetCreateHomeCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), + createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), } return s, nil diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/config/config.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/config/config.go index 91da5fc04..9cb7883e6 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/config/config.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/config/config.go @@ -37,12 +37,12 @@ type Config struct { AdditionalInfoAttribute string `mapstructure:"additional_info_attribute"` CacheWarmupDriver string `mapstructure:"cache_warmup_driver"` CacheWarmupDrivers map[string]map[string]interface{} `mapstructure:"cache_warmup_drivers"` - ResourceInfoCacheStore string `mapstructure:"resource_info_cache_store"` - ResourceInfoCacheNodes []string `mapstructure:"resource_info_cache_nodes"` - ResourceInfoCacheDatabase string `mapstructure:"resource_info_cache_database"` - ResourceInfoCacheTable string `mapstructure:"resource_info_cache_table"` - ResourceInfoCacheTTL int `mapstructure:"resource_info_cache_ttl"` - ResourceInfoCacheSize int `mapstructure:"resource_info_cache_size"` + StatCacheStore string `mapstructure:"stat_cache_store"` + StatCacheNodes []string `mapstructure:"stat_cache_nodes"` + StatCacheDatabase string `mapstructure:"stat_cache_database"` + StatCacheTable string `mapstructure:"stat_cache_table"` + StatCacheTTL int `mapstructure:"stat_cache_ttl"` + StatCacheSize int `mapstructure:"stat_cache_size"` UserIdentifierCacheTTL int `mapstructure:"user_identifier_cache_ttl"` MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` SkipUpdatingExistingSharesMountpoints bool `mapstructure:"skip_updating_existing_shares_mountpoint"` diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 51666b764..417550879 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -130,7 +130,7 @@ func (h *Handler) Init(c *config.Config) { h.deniable = c.EnableDenials h.resharing = resharing(c) - h.statCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize) + h.statCache = cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize) if c.CacheWarmupDriver != "" { cwm, err := getCacheWarmupManager(c) if err == nil { diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go index 016f49372..152bddca4 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/ocs.go @@ -67,9 +67,9 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - if conf.CacheWarmupDriver == "first-request" && conf.ResourceInfoCacheTTL > 0 { + if conf.CacheWarmupDriver == "first-request" && conf.StatCacheStore != "noop" { s.warmupCacheTracker = ttlcache.NewCache() - _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.ResourceInfoCacheTTL)) + _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.StatCacheTTL)) } return s, nil diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go index 58be25188..9f8828ccc 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/filemetadata.go @@ -19,7 +19,6 @@ package cache import ( - "strings" "time" ) @@ -40,15 +39,5 @@ func NewFileMetadataCache(store string, nodes []string, database, table string, // RemoveMetadata removes a reference from the metadata cache func (c *fileMetadataCache) RemoveMetadata(path string) error { - keys, err := c.List() - if err != nil { - return err - } - - for _, key := range keys { - if strings.HasPrefix(key, path) { - _ = c.Delete(key) - } - } - return nil + return c.s.Delete(path) } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go index f7ad3e1d0..23b8a3f82 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/provider.go @@ -19,11 +19,12 @@ package cache import ( - "strings" + "sync" "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "go-micro.dev/v4/store" ) // ProviderCache can invalidate all provider related cache entries @@ -47,20 +48,22 @@ func (c providerCache) RemoveListStorageProviders(res *provider.ResourceId) { if res == nil { return } - sid := res.SpaceId - keys, err := c.List() + keys, err := c.List(store.ListSuffix(res.SpaceId), store.ListLimit(100)) if err != nil { // FIXME log error return } - // FIXME add context option to List, Read and Write to upstream + + wg := sync.WaitGroup{} for _, key := range keys { - if strings.Contains(key, sid) { - _ = c.Delete(key) - continue - } + wg.Add(1) + go func(k string) { + defer wg.Done() + _ = c.Delete(k) + }(key) } + wg.Wait() } func (c providerCache) GetKey(userID *userpb.UserId, spaceID string) string { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go index 414fe77f3..cf990a863 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/cache/stat.go @@ -20,10 +20,12 @@ package cache import ( "strings" + "sync" "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "go-micro.dev/v4/store" ) // NewStatCache creates a new StatCache @@ -50,27 +52,27 @@ func (c statCache) RemoveStat(userID *userpb.UserId, res *provider.ResourceId) { oid = "oid:" + res.OpaqueId } - keys, err := c.List() - if err != nil { - // FIXME handle error - return - } - for _, key := range keys { - if strings.Contains(key, uid) { - _ = c.Delete(key) - continue - } + // TODO currently, invalidating the stat cache is inefficient and should be disabled. Storage providers / drivers can more selectively invalidate stat cache entries. + // This shotgun invalidation wipes all cache entries for the user, space, and nodeid of a changed resource, which means the stat cache is mostly empty, anyway. + prefixes := []string{uid, "*" + sid, "*" + oid} - if sid != "" && strings.Contains(key, sid) { - _ = c.Delete(key) - continue - } - - if oid != "" && strings.Contains(key, oid) { - _ = c.Delete(key) - continue - } + wg := sync.WaitGroup{} + for _, prefix := range prefixes { + wg.Add(1) + go func(p string) { + defer wg.Done() + keys, _ := c.List(store.ListPrefix(p), store.ListLimit(100)) + for _, key := range keys { + wg.Add(1) + go func(k string) { + defer wg.Done() + _ = c.Delete(k) + }(key) + } + }(prefix) } + + wg.Wait() } // generates a user specific key pointing to ref - used for statcache diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 012b7af85..fc051372e 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -48,7 +48,6 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/chunking" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/migrator" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" @@ -589,16 +588,6 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) return } - if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - // mark the home node as the end of propagation - if err = n.SetXattrString(prefixes.PropagationAttr, "1"); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") - - // FIXME: This does not return an error at all, but results in a severe situation that the - // part tree is not marked for propagation - return - } - } return } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index b414ede94..4152e56fb 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -57,15 +57,11 @@ func (MessagePackBackend) Name() string { return "messagepack" } // All reads all extended attributes for a node func (b MessagePackBackend) All(path string) (map[string][]byte, error) { - path = b.MetadataPath(path) - return b.loadAttributes(path, nil) } // Get an extended attribute value for the given key func (b MessagePackBackend) Get(path, key string) ([]byte, error) { - path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path, nil) if err != nil { return []byte{}, err @@ -79,8 +75,6 @@ func (b MessagePackBackend) Get(path, key string) ([]byte, error) { // GetInt64 reads a string as int64 from the xattrs func (b MessagePackBackend) GetInt64(path, key string) (int64, error) { - path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path, nil) if err != nil { return 0, err @@ -99,8 +93,6 @@ func (b MessagePackBackend) GetInt64(path, key string) (int64, error) { // List retrieves a list of names of extended attributes associated with the // given path in the file system. func (b MessagePackBackend) List(path string) ([]string, error) { - path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path, nil) if err != nil { return nil, err @@ -130,7 +122,6 @@ func (b MessagePackBackend) Remove(path, key string) error { // AllWithLockedSource reads all extended attributes from the given reader (if possible). // The path argument is used for storing the data in the cache func (b MessagePackBackend) AllWithLockedSource(path string, source io.Reader) (map[string][]byte, error) { - path = b.MetadataPath(path) return b.loadAttributes(path, source) } @@ -139,11 +130,11 @@ func (b MessagePackBackend) saveAttributes(path string, setAttribs map[string][] f readWriteCloseSeekTruncater err error ) - path = b.MetadataPath(path) + metaPath := b.MetadataPath(path) if acquireLock { - f, err = lockedfile.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) + f, err = lockedfile.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0600) } else { - f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) + f, err = os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0600) } if err != nil { return err @@ -151,7 +142,7 @@ func (b MessagePackBackend) saveAttributes(path string, setAttribs map[string][] defer f.Close() // Invalidate cache early - _ = b.metaCache.RemoveMetadata(path) + _ = b.metaCache.RemoveMetadata(b.cacheKey(path)) // Read current state msgBytes, err := io.ReadAll(f) @@ -204,15 +195,16 @@ func (b MessagePackBackend) loadAttributes(path string, source io.Reader) (map[s return attribs, err } + metaPath := b.MetadataPath(path) if source == nil { - source, err = lockedfile.Open(path) + source, err = lockedfile.Open(metaPath) // // No cached entry found. Read from storage and store in cache if err != nil { if os.IsNotExist(err) { // some of the caller rely on ENOTEXISTS to be returned when the // actual file (not the metafile) does not exist in order to // determine whether a node exists or not -> stat the actual node - _, err := os.Stat(strings.TrimSuffix(path, ".mpk")) + _, err := os.Stat(path) if err != nil { return nil, err } @@ -254,13 +246,15 @@ func (b MessagePackBackend) Purge(path string) error { // Rename moves the data for a given path to a new path func (b MessagePackBackend) Rename(oldPath, newPath string) error { - data := map[string]string{} - _ = b.metaCache.PullFromCache(b.cacheKey(oldPath), &data) - err := b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) - if err != nil { - return err + data := map[string][]byte{} + err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data) + if err == nil { + err = b.metaCache.PushToCache(b.cacheKey(newPath), data) + if err != nil { + return err + } } - err = b.metaCache.PushToCache(b.cacheKey(newPath), data) + err = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) if err != nil { return err } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go index ea682ab22..c802ebfa1 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/node.go @@ -164,48 +164,22 @@ func (n *Node) SetType(t provider.ResourceType) { n.nodeType = &t } -// ChangeOwner sets the owner of n to newOwner -func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { - n.SpaceRoot.owner = new - - attribs := Attributes{} - attribs.SetString(prefixes.OwnerIDAttr, new.OpaqueId) - attribs.SetString(prefixes.OwnerIDPAttr, new.Idp) - attribs.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(new.Type)) - - if err := n.SpaceRoot.SetXattrs(attribs, true); err != nil { - return err - } - - return -} - -// WriteAllNodeMetadata writes the Node metadata to disk -func (n *Node) WriteAllNodeMetadata(ctx context.Context) (err error) { +// NodeMetadata writes the Node metadata to disk and allows passing additional attributes +func (n *Node) NodeMetadata() Attributes { attribs := Attributes{} attribs.SetInt64(prefixes.TypeAttr, int64(n.Type())) attribs.SetString(prefixes.ParentidAttr, n.ParentID) attribs.SetString(prefixes.NameAttr, n.Name) - attribs.SetString(prefixes.BlobIDAttr, n.BlobID) - attribs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) - - return n.SetXattrs(attribs, true) + if n.Type() == provider.ResourceType_RESOURCE_TYPE_FILE { + attribs.SetString(prefixes.BlobIDAttr, n.BlobID) + attribs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) + } + return attribs } -// WriteOwner writes the space owner -func (n *Node) WriteOwner(owner *userpb.UserId) error { +// SetOwner sets the space owner on the node +func (n *Node) SetOwner(owner *userpb.UserId) { n.SpaceRoot.owner = owner - - attribs := Attributes{} - attribs.SetString(prefixes.OwnerIDAttr, owner.OpaqueId) - attribs.SetString(prefixes.OwnerIDPAttr, owner.Idp) - attribs.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(owner.Type)) - - if err := n.SpaceRoot.SetXattrs(attribs, true); err != nil { - return err - } - n.SpaceRoot.owner = owner - return nil } // SpaceOwnerOrManager returns the space owner of the space. If no owner is set @@ -354,11 +328,13 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis if revisionSuffix == "" { n.BlobID = attrs.String(prefixes.BlobIDAttr) - blobSize, err := attrs.Int64(prefixes.BlobsizeAttr) - if err != nil { - return nil, err + if n.BlobID != "" { + blobSize, err := attrs.Int64(prefixes.BlobsizeAttr) + if err != nil { + return nil, err + } + n.Blobsize = blobSize } - n.Blobsize = blobSize } else { n.BlobID, err = lu.ReadBlobIDAttr(nodePath + revisionSuffix) if err != nil { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go index 96a884132..4fced4f99 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go @@ -101,28 +101,18 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return nil, errors.Wrap(err, "Decomposedfs: error creating node") } - if err := root.WriteAllNodeMetadata(ctx); err != nil { - return nil, err - } - var owner *userv1beta1.UserId if req.GetOwner() != nil && req.GetOwner().GetId() != nil { - owner = req.GetOwner().GetId() + root.SetOwner(req.GetOwner().GetId()) } else { - owner = &userv1beta1.UserId{OpaqueId: spaceID, Type: userv1beta1.UserType_USER_TYPE_SPACE_OWNER} - } - if err := root.WriteOwner(owner); err != nil { - return nil, err + root.SetOwner(&userv1beta1.UserId{OpaqueId: spaceID, Type: userv1beta1.UserType_USER_TYPE_SPACE_OWNER}) } - err = fs.updateIndexes(ctx, req.GetOwner().GetId().GetOpaqueId(), req.Type, root.ID) - if err != nil { - return nil, err - } + metadata := node.Attributes{} + metadata.SetString(prefixes.OwnerIDAttr, root.Owner().GetOpaqueId()) + metadata.SetString(prefixes.OwnerIDPAttr, root.Owner().GetIdp()) + metadata.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(root.Owner().GetType())) - metadata := make(node.Attributes, 6) - - // always enable propagation on the storage space root - // mark the space root node as the end of propagation + // always mark the space root node as the end of propagation metadata.SetString(prefixes.PropagationAttr, "1") metadata.SetString(prefixes.NameAttr, req.Name) metadata.SetString(prefixes.SpaceNameAttr, req.Name) @@ -151,14 +141,20 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr metadata.SetString(prefixes.SpaceAliasAttr, alias) } + // Write node if err := root.SetXattrs(metadata, true); err != nil { return nil, err } + // Write index + err = fs.updateIndexes(ctx, req.GetOwner().GetId().GetOpaqueId(), req.Type, root.ID) + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, utils.SpaceGrant, struct{ SpaceType string }{SpaceType: req.Type}) if req.Type != _spaceTypePersonal { - u := ctxpkg.ContextMustGetUser(ctx) if err := fs.AddGrant(ctx, &provider.Reference{ ResourceId: &provider.ResourceId{ SpaceId: spaceID, @@ -633,12 +629,17 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return err } + // invalidate cache + if err := fs.lu.MetadataBackend().Purge(n.InternalPath()); err != nil { + return err + } + // remove space metadata if err := os.RemoveAll(fs.getSpaceRoot(spaceID)); err != nil { return err } - // FIXME remove space blobs + // TODO remove space blobs with s3 backend by adding a purge method to the Blobstore interface return nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go index 96ecf1e86..2cb35524c 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go @@ -151,15 +151,15 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool) return errors.Wrap(err, "Decomposedfs: error creating node") } - err = n.WriteAllNodeMetadata(ctx) + attributes := n.NodeMetadata() + if markprocessing { + attributes[prefixes.StatusPrefix] = []byte(node.ProcessingStatus) + } + err = n.SetXattrs(attributes, true) if err != nil { return err } - if markprocessing { - _ = n.SetXattr(prefixes.StatusPrefix, []byte(node.ProcessingStatus)) - } - // link child name to parent if it is new childNameLink := filepath.Join(n.ParentPath(), n.Name) var link string @@ -196,10 +196,6 @@ func (t *Tree) CreateDir(ctx context.Context, n *node.Node) (err error) { return } - if err := n.SetTreeSize(0); err != nil { - return err - } - // make child appear in listings relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) err = os.Symlink(relativeNodePath, filepath.Join(n.ParentPath(), n.Name)) @@ -907,7 +903,12 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { return errors.Wrap(err, "Decomposedfs: error creating node") } - return n.WriteAllNodeMetadata(ctx) + attributes := n.NodeMetadata() + attributes[prefixes.TreesizeAttr] = []byte("0") // initialize as empty, TODO why bother? if it is not set we could treat it as 0? + if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting { + attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation + } + return n.SetXattrs(attributes, true) } var nodeIDRegep = regexp.MustCompile(`.*/nodes/([^.]*).*`) @@ -938,19 +939,21 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( if err != nil { return } - recycleNode.SetType(t.lookup.TypeFromPath(recycleNode.InternalPath())) + recycleNode.SetType(t.lookup.TypeFromPath(deletedNodePath)) var attrBytes []byte - // lookup blobID in extended attributes - if attrBytes, err = backend.Get(deletedNodePath, prefixes.BlobIDAttr); err == nil { - recycleNode.BlobID = string(attrBytes) - } else { - return - } + if recycleNode.Type() == provider.ResourceType_RESOURCE_TYPE_FILE { + // lookup blobID in extended attributes + if attrBytes, err = backend.Get(deletedNodePath, prefixes.BlobIDAttr); err == nil { + recycleNode.BlobID = string(attrBytes) + } else { + return + } - // lookup blobSize in extended attributes - if recycleNode.Blobsize, err = backend.GetInt64(deletedNodePath, prefixes.BlobsizeAttr); err != nil { - return + // lookup blobSize in extended attributes + if recycleNode.Blobsize, err = backend.GetInt64(deletedNodePath, prefixes.BlobsizeAttr); err != nil { + return + } } // lookup parent id in extended attributes diff --git a/vendor/modules.txt b/vendor/modules.txt index 8466d4b30..bd92942e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -349,7 +349,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.12.1-0.20230425110812-8a3352fb1960 +# github.com/cs3org/reva/v2 v2.12.1-0.20230427075231-7842414d18e1 ## explicit; go 1.19 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime