[full-ci] Improve ini backend, support redis sentinel caches (#5737)
* Configure the file metadata cache for storage-users * Also configure the cache for the system storage * Update services/storage-system/pkg/config/config.go Co-authored-by: Martin <github@diemattels.at> * Improve documentation * Bump reva to pull in the latest ini backend changes * Fix missing comment * Update services/storage-users/README.md Co-authored-by: Martin <github@diemattels.at> * Update services/storage-system/README.md Co-authored-by: Martin <github@diemattels.at> * Tweak docs * Apply suggestions from code review Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de> Co-authored-by: Martin <github@diemattels.at> --------- Co-authored-by: Martin <github@diemattels.at> Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Andre Duffeck
co-authored by
Martin
Jörn Friedrich Dreyer
Martin
Jörn Friedrich Dreyer
parent
1e1b4379ae
commit
8b68830201
@@ -0,0 +1,18 @@
|
||||
# Storage-System Service
|
||||
|
||||
Purpose and description to be added
|
||||
|
||||
## Caching
|
||||
|
||||
The `storage-system` service caches file metadata via the configured store in `STORAGE_SYSTEM_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments.
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies.
|
||||
3. The `storage-system` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
@@ -23,6 +23,7 @@ type Config struct {
|
||||
|
||||
SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."`
|
||||
|
||||
Cache Cache `yaml:"cache"`
|
||||
Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service."`
|
||||
Drivers Drivers `yaml:"drivers"`
|
||||
DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service."`
|
||||
@@ -80,3 +81,10 @@ type OCISDriver struct {
|
||||
MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_SYSTEM_OCIS_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, ocis will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used."`
|
||||
LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_SYSTEM_OCIS_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, ocis will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used."`
|
||||
}
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_SYSTEM_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', 'etcd' and 'noop'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_SYSTEM_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_SYSTEM_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
}
|
||||
|
||||
@@ -47,6 +47,10 @@ func DefaultConfig() *config.Config {
|
||||
LockCycleDurationFactor: 30,
|
||||
},
|
||||
},
|
||||
Cache: config.Cache{
|
||||
Store: "memory",
|
||||
Database: "system",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -161,6 +161,11 @@ func metadataDrivers(cfg *config.Config) map[string]interface{} {
|
||||
"permissionssvc": cfg.GRPC.Addr,
|
||||
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles,
|
||||
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor,
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,3 +80,18 @@ The configuration for the `purge-expired` command is done by using the following
|
||||
* `STORAGE_USERS_PURGE_TRASH_BIN_PERSONAL_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `personal space` trash-bin files.
|
||||
|
||||
* `STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `project space` trash-bin files.
|
||||
|
||||
## Caching
|
||||
|
||||
The `storage-users` service caches file metadata via the configured store in `STORAGE_USERS_CACHE_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis`: Stores metadata in a configured Redis cluster.
|
||||
- `redis-sentinel`: Stores metadata in a configured Redis Sentinel cluster.
|
||||
- `etcd`: Stores metadata in a configured etcd cluster.
|
||||
- `nats-js`: Stores metadata using the key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in productive enviroments.
|
||||
|
||||
1. Note that in-memory stores are by nature not reboot persistent.
|
||||
2. Though usually not necessary, a database name can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`, `redis` and `redis-sentinel`. These settings are blank by default which means that the standard settings of the configured store applies.
|
||||
3. The `storage-users` service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances.
|
||||
4. When using `redis-sentinel`, the Redis master to use is configured via `STORAGE_SYSTEM_CACHE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
@@ -169,9 +169,9 @@ type Events struct {
|
||||
|
||||
// Cache holds cache config
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Valid values are \"memory\" (default), \"redis\", and \"etcd\"."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"Node addresses to use for the cache store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"Database name of the cache."`
|
||||
Store string `yaml:"store" env:"OCIS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE_TYPE;STORAGE_USERS_CACHE_STORE" desc:"Store implementation for the cache. Supported values are 'memory' (default), 'redis', 'redis-sentinel', 'nats-js', and 'etcd'. See the text description for details."`
|
||||
Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_STORE_ADDRESS;STORAGE_USERS_CACHE_NODES" desc:"A comma separated list of addresses to access the configured store. This has no effect when the 'memory' store is configured. Note that the behaviour how addresses are used is dependent on the library of the configured store."`
|
||||
Database string `yaml:"database" env:"STORAGE_USERS_CACHE_DATABASE" desc:"(optional) The database name the configured store should use. This has no effect when a 'memory', 'redis' or 'redis-sentinel' store is configured."`
|
||||
}
|
||||
|
||||
// S3Driver is the storage driver configuration when using 's3' storage driver
|
||||
|
||||
@@ -130,6 +130,11 @@ func Ocis(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
"natsclusterid": cfg.Events.ClusterID,
|
||||
@@ -167,6 +172,11 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,6 +219,11 @@ func S3NG(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"events": map[string]interface{}{
|
||||
"natsaddress": cfg.Events.Addr,
|
||||
"natsclusterid": cfg.Events.ClusterID,
|
||||
@@ -250,5 +265,10 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} {
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
"filemetadatacache": map[string]interface{}{
|
||||
"cache_store": cfg.Cache.Store,
|
||||
"cache_nodes": cfg.Cache.Nodes,
|
||||
"cache_database": cfg.Cache.Database,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user