[full-ci] Refactor stores (#6019)

* Streamline the store implementation with and into reva

* Adapt to the cache/store refactoring in reva

* Streamline config options and their env vars

* Apply suggestions from code review

Co-authored-by: Martin <github@diemattels.at>

* Use the same database for all stores

* Bump reva

* Configure stat and filemetadata cache separately

* Fix default config

---------

Co-authored-by: Martin <github@diemattels.at>
This commit is contained in:
Andre Duffeck
2023-04-24 15:13:35 +02:00
committed by GitHub
co-authored by Martin
parent 39c8a45984
commit 77bb3d8bcd
76 changed files with 665 additions and 2529 deletions
-1
View File
@@ -43,7 +43,6 @@ import (
_ "github.com/cs3org/reva/v2/pkg/preferences/loader"
_ "github.com/cs3org/reva/v2/pkg/publicshare/manager/loader"
_ "github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/loader"
_ "github.com/cs3org/reva/v2/pkg/share/cache/loader"
_ "github.com/cs3org/reva/v2/pkg/share/cache/warmup/loader"
_ "github.com/cs3org/reva/v2/pkg/share/manager/loader"
_ "github.com/cs3org/reva/v2/pkg/storage/favorite/loader"
@@ -72,8 +72,11 @@ type config struct {
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"`
CreateHomeCacheSize int `mapstructure:"create_home_cache_size"`
ProviderCacheTTL int `mapstructure:"provider_cache_ttl"`
ProviderCacheSize int `mapstructure:"provider_cache_size"`
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
StatCacheSize int `mapstructure:"stat_cache_size"`
UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"`
}
@@ -166,10 +169,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
c: c,
dataGatewayURL: *u,
tokenmgr: tokenManager,
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second),
providerCache: cache.GetProviderCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second),
createHomeCache: cache.GetCreateHomeCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second),
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second),
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize),
providerCache: cache.GetProviderCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize),
createHomeCache: cache.GetCreateHomeCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
}
return s, nil
@@ -87,19 +87,18 @@ func (s *svc) handlePathCopy(w http.ResponseWriter, r *http.Request, ns string)
return
}
for _, r := range nameRules {
if !r.Test(src) {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "source failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if !r.Test(dst) {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "destination failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if err := ValidateName(src, s.nameValidators); err != nil {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "source failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if err := ValidateName(dst, s.nameValidators); err != nil {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "destination failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
dst = path.Join(ns, dst)
@@ -40,10 +40,8 @@ func (s *svc) handlePathMkcol(w http.ResponseWriter, r *http.Request, ns string)
defer span.End()
fn := path.Join(ns, r.URL.Path)
for _, r := range nameRules {
if !r.Test(fn) {
return http.StatusBadRequest, fmt.Errorf("invalid name rule")
}
if err := ValidateName(fn, s.nameValidators); err != nil {
return http.StatusBadRequest, err
}
sublog := appctx.GetLogger(ctx).With().Str("path", fn).Logger()
@@ -60,19 +60,18 @@ func (s *svc) handlePathMove(w http.ResponseWriter, r *http.Request, ns string)
return
}
for _, r := range nameRules {
if !r.Test(srcPath) {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "source failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if !r.Test(dstPath) {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "destination naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if err := ValidateName(srcPath, s.nameValidators); err != nil {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "source failed naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
if err := ValidateName(dstPath, s.nameValidators); err != nil {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, "destination naming rules", "")
errors.HandleWebdavError(appctx.GetLogger(ctx), w, b, err)
return
}
dstPath = path.Join(ns, dstPath)
@@ -55,31 +55,6 @@ import (
// name is the Tracer name used to identify this instrumentation library.
const tracerName = "ocdav"
var (
nameRules = [...]nameRule{
nameNotEmpty{},
nameDoesNotContain{chars: "\f\r\n\\"},
}
)
type nameRule interface {
Test(name string) bool
}
type nameNotEmpty struct{}
func (r nameNotEmpty) Test(name string) bool {
return len(strings.TrimSpace(name)) > 0
}
type nameDoesNotContain struct {
chars string
}
func (r nameDoesNotContain) Test(name string) bool {
return !strings.ContainsAny(name, r.chars)
}
func init() {
global.Register("ocdav", New)
}
@@ -113,9 +88,17 @@ type Config struct {
ProductName string `mapstructure:"product_name"`
ProductVersion string `mapstructure:"product_version"`
NameValidation NameValidation `mapstructure:"validation"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
}
// NameValidation is the validation configuration for file and folder names
type NameValidation struct {
InvalidChars []string `mapstructure:"invalid_chars"`
MaxLength int `mapstructure:"max_length"`
}
func (c *Config) init() {
// note: default c.Prefix is an empty string
c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
@@ -147,6 +130,14 @@ func (c *Config) init() {
if c.Edition == "" {
c.Edition = "community"
}
if c.NameValidation.InvalidChars == nil {
c.NameValidation.InvalidChars = []string{"\f", "\r", "\n", "\\"}
}
if c.NameValidation.MaxLength == 0 {
c.NameValidation.MaxLength = 255
}
}
type svc struct {
@@ -160,6 +151,7 @@ type svc struct {
LockSystem LockSystem
userIdentifierCache *ttlcache.Cache
tracerProvider trace.TracerProvider
nameValidators []Validator
}
func (s *svc) Config() *Config {
@@ -204,6 +196,9 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error)
// NewWith returns a new ocdav service
func NewWith(conf *Config, fm favorite.Manager, ls LockSystem, _ *zerolog.Logger, tp trace.TracerProvider, gwc gateway.GatewayAPIClient) (global.Service, error) {
// be safe - init the conf again
conf.init()
s := &svc{
c: conf,
webDavHandler: new(WebDavHandler),
@@ -217,6 +212,7 @@ func NewWith(conf *Config, fm favorite.Manager, ls LockSystem, _ *zerolog.Logger
LockSystem: ls,
userIdentifierCache: ttlcache.NewCache(),
tracerProvider: tp,
nameValidators: ValidatorsFromConfig(conf),
}
_ = s.userIdentifierCache.SetTTL(60 * time.Second)
@@ -23,6 +23,7 @@ import (
"io"
"net/http"
"path"
"path/filepath"
"strconv"
"strings"
@@ -139,6 +140,14 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
return
}
fn := filepath.Base(ref.Path)
if err := ValidateName(fn, s.nameValidators); err != nil {
w.WriteHeader(http.StatusBadRequest)
b, err := errors.Marshal(http.StatusBadRequest, err.Error(), "")
errors.HandleWebdavError(&log, w, b, err)
return
}
if length == 0 {
tfRes, err := s.gwClient.TouchFile(ctx, &provider.TouchFileRequest{
Ref: ref,
@@ -50,11 +50,9 @@ func (s *svc) handlePathTusPost(w http.ResponseWriter, r *http.Request, ns strin
// read filename from metadata
meta := tusd.ParseMetadataHeader(r.Header.Get(net.HeaderUploadMetadata))
for _, r := range nameRules {
if !r.Test(meta["filename"]) {
w.WriteHeader(http.StatusPreconditionFailed)
return
}
if err := ValidateName(meta["filename"], s.nameValidators); err != nil {
w.WriteHeader(http.StatusPreconditionFailed)
return
}
// append filename to current dir
@@ -76,11 +74,9 @@ func (s *svc) handleSpacesTusPost(w http.ResponseWriter, r *http.Request, spaceI
// read filename from metadata
meta := tusd.ParseMetadataHeader(r.Header.Get(net.HeaderUploadMetadata))
for _, r := range nameRules {
if !r.Test(meta["filename"]) {
w.WriteHeader(http.StatusPreconditionFailed)
return
}
if err := ValidateName(meta["filename"], s.nameValidators); err != nil {
w.WriteHeader(http.StatusPreconditionFailed)
return
}
sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger()
@@ -0,0 +1,63 @@
package ocdav
import (
"errors"
"fmt"
"strings"
)
// Validator validates strings
type Validator func(string) error
// ValidatorsFromConfig returns the configured Validators
func ValidatorsFromConfig(c *Config) []Validator {
// we always want to exclude empty names
vals := []Validator{notEmpty()}
// forbidden characters
vals = append(vals, doesNotContain(c.NameValidation.InvalidChars))
// max length
vals = append(vals, isShorterThan(c.NameValidation.MaxLength))
return vals
}
// ValidateName will validate a file or folder name, returning an error when it is not accepted
func ValidateName(name string, validators []Validator) error {
for _, v := range validators {
if err := v(name); err != nil {
return fmt.Errorf("name validation failed: %w", err)
}
}
return nil
}
func notEmpty() Validator {
return func(s string) error {
if strings.TrimSpace(s) == "" {
return errors.New("must not be empty")
}
return nil
}
}
func doesNotContain(bad []string) Validator {
return func(s string) error {
for _, b := range bad {
if strings.Contains(s, b) {
return fmt.Errorf("must not contain %s", b)
}
}
return nil
}
}
func isShorterThan(maxLength int) Validator {
return func(s string) error {
if len(s) > maxLength {
return fmt.Errorf("must be shorter than %d", maxLength)
}
return nil
}
}
@@ -37,9 +37,12 @@ type Config struct {
AdditionalInfoAttribute string `mapstructure:"additional_info_attribute"`
CacheWarmupDriver string `mapstructure:"cache_warmup_driver"`
CacheWarmupDrivers map[string]map[string]interface{} `mapstructure:"cache_warmup_drivers"`
ResourceInfoCacheDriver string `mapstructure:"resource_info_cache_type"`
ResourceInfoCacheStore string `mapstructure:"resource_info_cache_store"`
ResourceInfoCacheNodes []string `mapstructure:"resource_info_cache_nodes"`
ResourceInfoCacheDatabase string `mapstructure:"resource_info_cache_database"`
ResourceInfoCacheTable string `mapstructure:"resource_info_cache_table"`
ResourceInfoCacheTTL int `mapstructure:"resource_info_cache_ttl"`
ResourceInfoCacheDrivers map[string]map[string]interface{} `mapstructure:"resource_info_caches"`
ResourceInfoCacheSize int `mapstructure:"resource_info_cache_size"`
UserIdentifierCacheTTL int `mapstructure:"user_identifier_cache_ttl"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
SkipUpdatingExistingSharesMountpoints bool `mapstructure:"skip_updating_existing_shares_mountpoint"`
@@ -52,9 +52,9 @@ import (
"github.com/cs3org/reva/v2/pkg/publicshare"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/share"
"github.com/cs3org/reva/v2/pkg/share/cache"
cachereg "github.com/cs3org/reva/v2/pkg/share/cache/registry"
sharecache "github.com/cs3org/reva/v2/pkg/share/cache"
warmupreg "github.com/cs3org/reva/v2/pkg/share/cache/warmup/registry"
"github.com/cs3org/reva/v2/pkg/storage/cache"
"github.com/cs3org/reva/v2/pkg/storage/utils/templates"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
@@ -83,8 +83,7 @@ type Handler struct {
skipUpdatingExistingSharesMountpoints bool
additionalInfoTemplate *template.Template
userIdentifierCache *ttlcache.Cache
resourceInfoCache cache.ResourceInfoCache
resourceInfoCacheTTL time.Duration
resourceInfoCache cache.StatCache
deniable bool
resharing bool
@@ -104,7 +103,7 @@ type ocsError struct {
Message string
}
func getCacheWarmupManager(c *config.Config) (cache.Warmup, error) {
func getCacheWarmupManager(c *config.Config) (sharecache.Warmup, error) {
if f, ok := warmupreg.NewFuncs[c.CacheWarmupDriver]; ok {
return f(c.CacheWarmupDrivers[c.CacheWarmupDriver])
}
@@ -114,13 +113,6 @@ func getCacheWarmupManager(c *config.Config) (cache.Warmup, error) {
// GatewayClientGetter is the function being used to retrieve a gateway client instance
type GatewayClientGetter func() (gateway.GatewayAPIClient, error)
func getCacheManager(c *config.Config) (cache.ResourceInfoCache, error) {
if f, ok := cachereg.NewFuncs[c.ResourceInfoCacheDriver]; ok {
return f(c.ResourceInfoCacheDrivers[c.ResourceInfoCacheDriver])
}
return nil, fmt.Errorf("driver not found: %s", c.ResourceInfoCacheDriver)
}
// Init initializes this and any contained handlers
func (h *Handler) Init(c *config.Config) {
h.gatewayAddr = c.GatewaySvc
@@ -132,19 +124,14 @@ func (h *Handler) Init(c *config.Config) {
h.skipUpdatingExistingSharesMountpoints = c.SkipUpdatingExistingSharesMountpoints
h.additionalInfoTemplate, _ = template.New("additionalInfo").Parse(c.AdditionalInfoAttribute)
h.resourceInfoCacheTTL = time.Second * time.Duration(c.ResourceInfoCacheTTL)
h.userIdentifierCache = ttlcache.NewCache()
_ = h.userIdentifierCache.SetTTL(time.Second * time.Duration(c.UserIdentifierCacheTTL))
h.deniable = c.EnableDenials
h.resharing = resharing(c)
cache, err := getCacheManager(c)
if err == nil {
h.resourceInfoCache = cache
}
if h.resourceInfoCacheTTL > 0 {
h.resourceInfoCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize)
if c.CacheWarmupDriver != "" {
cwm, err := getCacheWarmupManager(c)
if err == nil {
go h.startCacheWarmup(cwm)
@@ -159,15 +146,15 @@ func (h *Handler) InitWithGetter(c *config.Config, clientGetter GatewayClientGet
h.getClient = clientGetter
}
func (h *Handler) startCacheWarmup(c cache.Warmup) {
func (h *Handler) startCacheWarmup(c sharecache.Warmup) {
time.Sleep(2 * time.Second)
infos, err := c.GetResourceInfos()
if err != nil {
return
}
for _, r := range infos {
key := storagespace.FormatResourceID(*r.Id)
_ = h.resourceInfoCache.SetWithExpire(key, r, h.resourceInfoCacheTTL)
key := h.resourceInfoCache.GetKey(r.Owner, &provider.Reference{ResourceId: r.Id}, []string{}, []string{})
_ = h.resourceInfoCache.PushToCache(key, r)
}
}
@@ -781,6 +768,10 @@ func (h *Handler) updateShare(w http.ResponseWriter, r *http.Request, shareID st
return
}
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
h.resourceInfoCache.RemoveStat(currentUser.Id, shareR.Share.ResourceId)
}
share, err := conversions.CS3Share2ShareData(ctx, uRes.Share)
if err != nil {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error mapping share data", err)
@@ -1350,63 +1341,45 @@ func (h *Handler) getAdditionalInfoAttribute(ctx context.Context, u *userIdentif
}
func (h *Handler) getResourceInfoByReference(ctx context.Context, client gateway.GatewayAPIClient, ref *provider.Reference) (*provider.ResourceInfo, *rpc.Status, error) {
var key string
if ref.ResourceId == nil {
// This is a path based reference
key = ref.Path
} else {
var err error
key, err = storagespace.FormatReference(ref)
if err != nil {
return nil, nil, err
}
}
return h.getResourceInfo(ctx, client, key, ref)
return h.getResourceInfo(ctx, client, ref)
}
func (h *Handler) getResourceInfoByID(ctx context.Context, client gateway.GatewayAPIClient, id *provider.ResourceId) (*provider.ResourceInfo, *rpc.Status, error) {
return h.getResourceInfo(ctx, client, storagespace.FormatResourceID(*id), &provider.Reference{ResourceId: id})
return h.getResourceInfo(ctx, client, &provider.Reference{ResourceId: id})
}
// getResourceInfo retrieves the resource info to a target.
// This method utilizes caching if it is enabled.
func (h *Handler) getResourceInfo(ctx context.Context, client gateway.GatewayAPIClient, key string, ref *provider.Reference) (*provider.ResourceInfo, *rpc.Status, error) {
func (h *Handler) getResourceInfo(ctx context.Context, client gateway.GatewayAPIClient, ref *provider.Reference) (*provider.ResourceInfo, *rpc.Status, error) {
logger := appctx.GetLogger(ctx)
var pinfo *provider.ResourceInfo
var status *rpc.Status
var err error
var foundInCache bool
if h.resourceInfoCacheTTL > 0 && h.resourceInfoCache != nil {
if pinfo, err = h.resourceInfoCache.Get(key); err == nil {
logger.Debug().Msgf("cache hit for resource %+v", key)
status = &rpc.Status{Code: rpc.Code_CODE_OK}
foundInCache = true
}
}
if !foundInCache {
logger.Debug().Msgf("cache miss for resource %+v, statting", key)
statReq := &provider.StatRequest{
Ref: ref,
}
statRes, err := client.Stat(ctx, statReq)
if err != nil {
return nil, nil, err
}
if statRes.Status.Code != rpc.Code_CODE_OK {
return nil, statRes.Status, nil
}
pinfo = statRes.GetInfo()
status = statRes.Status
if h.resourceInfoCacheTTL > 0 {
_ = h.resourceInfoCache.SetWithExpire(key, pinfo, h.resourceInfoCacheTTL)
key := ""
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
key = h.resourceInfoCache.GetKey(currentUser.Id, ref, []string{}, []string{})
pinfo := &provider.ResourceInfo{}
if err := h.resourceInfoCache.PullFromCache(key, pinfo); err == nil {
return pinfo, &rpc.Status{Code: rpc.Code_CODE_OK}, nil
}
}
return pinfo, status, nil
logger.Debug().Msgf("cache miss for resource %+v, statting", ref)
statReq := &provider.StatRequest{
Ref: ref,
}
statRes, err := client.Stat(ctx, statReq)
if err != nil {
return nil, nil, err
}
if statRes.Status.Code != rpc.Code_CODE_OK {
return nil, statRes.Status, nil
}
if key != "" {
_ = h.resourceInfoCache.PushToCache(key, *statRes.Info)
}
return statRes.Info, statRes.Status, nil
}
func (h *Handler) createCs3Share(ctx context.Context, w http.ResponseWriter, r *http.Request, client gateway.GatewayAPIClient, req *collaboration.CreateShareRequest) (*collaboration.Share, *ocsError) {
@@ -206,6 +206,9 @@ func (h *Handler) removeUserShare(w http.ResponseWriter, r *http.Request, shareI
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc delete share request failed", err)
return
}
if currentUser, ok := ctxpkg.ContextGetUser(ctx); ok {
h.resourceInfoCache.RemoveStat(currentUser.Id, getShareResp.Share.ResourceId)
}
response.WriteOCSSuccess(w, r, data)
}
+14
View File
@@ -307,3 +307,17 @@ func AllowedHeaders(val []string) Option {
o.AllowedHeaders = val
}
}
// ItemNameInvalidChars provides a function to set forbidden characters in file or folder names
func ItemNameInvalidChars(chars []string) Option {
return func(o *Options) {
o.config.NameValidation.InvalidChars = chars
}
}
// ItemNameMaxLength provides a function to set the maximum length of a file or folder name
func ItemNameMaxLength(i int) Option {
return func(o *Options) {
o.config.NameValidation.MaxLength = i
}
}
+15 -11
View File
@@ -33,6 +33,7 @@ import (
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/utils/download"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/cache"
@@ -44,21 +45,14 @@ func init() {
registry.Register("simple", New)
}
type config struct {
CacheStore string `mapstructure:"cache_store"`
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
CacheTable string `mapstructure:"cache_table"`
}
type manager struct {
conf *config
conf *cache.Config
publisher events.Publisher
statCache cache.StatCache
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
c := &cache.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
@@ -76,7 +70,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
return &manager{
conf: c,
publisher: publisher,
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, c.CacheTable, 0),
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
}, nil
}
@@ -87,8 +81,18 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
switch r.Method {
case "GET", "HEAD":
if r.Method == "GET" {
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
}
download.GetOrHeadFile(w, r, fs, "")
case "PUT":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
fn := r.URL.Path
defer r.Body.Close()
+16 -11
View File
@@ -32,6 +32,7 @@ import (
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/utils/download"
"github.com/cs3org/reva/v2/pkg/rhttp/router"
"github.com/cs3org/reva/v2/pkg/storage"
@@ -46,21 +47,14 @@ func init() {
registry.Register("spaces", New)
}
type config struct {
CacheStore string `mapstructure:"cache_store"`
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
CacheTable string `mapstructure:"cache_table"`
}
type manager struct {
conf *config
conf *cache.Config
publisher events.Publisher
statCache cache.StatCache
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
c := &cache.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
@@ -78,7 +72,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
return &manager{
conf: c,
publisher: publisher,
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, c.CacheTable, 0),
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
}, nil
}
@@ -92,8 +86,19 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
switch r.Method {
case "GET", "HEAD":
if r.Method == "GET" {
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
}
download.GetOrHeadFile(w, r, fs, spaceID)
case "PUT":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// make a clean relative path
fn := path.Clean(strings.TrimLeft(r.URL.Path, "/"))
defer r.Body.Close()
+18 -11
View File
@@ -24,6 +24,7 @@ import (
"net/http"
"path"
"path/filepath"
"time"
"github.com/pkg/errors"
tusd "github.com/tus/tusd/pkg/handler"
@@ -36,6 +37,7 @@ import (
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/cs3org/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/cache"
"github.com/cs3org/reva/v2/pkg/utils"
@@ -46,21 +48,14 @@ func init() {
registry.Register("tus", New)
}
type config struct {
CacheStore string `mapstructure:"cache_store"`
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
CacheTable string `mapstructure:"cache_table"`
}
type manager struct {
conf *config
conf *cache.Config
publisher events.Publisher
statCache cache.StatCache
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
c := &cache.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
@@ -77,7 +72,7 @@ func New(m map[string]interface{}, publisher events.Publisher) (datatx.DataTX, e
return &manager{
conf: c,
publisher: publisher,
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, c.CacheTable, 0),
statCache: cache.GetStatCache(c.Store, c.Nodes, c.Database, c.Table, time.Duration(c.TTL)*time.Second, c.Size),
}, nil
}
@@ -144,17 +139,29 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
switch method {
case "POST":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// set etag, mtime and file id
handler.PostFile(w, r)
case "HEAD":
handler.HeadFile(w, r)
case "PATCH":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// set etag, mtime and file id
setExpiresHeader(fs, w, r)
handler.PatchFile(w, r)
case "DELETE":
handler.DelFile(w, r)
case "GET":
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
// NOTE: this is breaking change - allthought it does not seem to be used
// We can make a switch here depending on some header value if that is needed
// download.GetOrHeadFile(w, r, fs, "")
+20
View File
@@ -0,0 +1,20 @@
// Package metrics provides prometheus metrics for the data managers..
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
// DownloadsActive is the number of active downloads
DownloadsActive = promauto.NewGauge(prometheus.GaugeOpts{
Name: "reva_download_active",
Help: "Number of active downloads",
})
// UploadsActive is the number of active uploads
UploadsActive = promauto.NewGauge(prometheus.GaugeOpts{
Name: "reva_upload_active",
Help: "Number of active uploads",
})
)
@@ -213,7 +213,6 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI
sublog.Error().Int64("copied", c).Int64("size", sendSize).Msg("copied vs size mismatch")
}
}
}
func handleError(w http.ResponseWriter, log *zerolog.Logger, err error, action string) {
-10
View File
@@ -19,8 +19,6 @@
package cache
import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
@@ -28,11 +26,3 @@ import (
type Warmup interface {
GetResourceInfos() ([]*provider.ResourceInfo, error)
}
// ResourceInfoCache is the interface to implement caches for resource infos
type ResourceInfoCache interface {
Get(key string) (*provider.ResourceInfo, error)
GetKeys(keys []string) ([]*provider.ResourceInfo, error)
Set(key string, info *provider.ResourceInfo) error
SetWithExpire(key string, info *provider.ResourceInfo, expiration time.Duration) error
}
-26
View File
@@ -1,26 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load share cache drivers.
_ "github.com/cs3org/reva/v2/pkg/share/cache/memory"
_ "github.com/cs3org/reva/v2/pkg/share/cache/redis"
// Add your own here
)
-83
View File
@@ -1,83 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package memory
import (
"time"
"github.com/bluele/gcache"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/share/cache"
"github.com/cs3org/reva/v2/pkg/share/cache/registry"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("memory", New)
}
type config struct {
CacheSize int `mapstructure:"cache_size"`
}
type manager struct {
cache gcache.Cache
}
// New returns an implementation of a resource info cache that stores the objects in memory
func New(m map[string]interface{}) (cache.ResourceInfoCache, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, errors.Wrap(err, "error decoding conf")
}
if c.CacheSize == 0 {
c.CacheSize = 10000
}
return &manager{
cache: gcache.New(c.CacheSize).LFU().Build(),
}, nil
}
func (m *manager) Get(key string) (*provider.ResourceInfo, error) {
infoIf, err := m.cache.Get(key)
if err != nil {
return nil, err
}
return infoIf.(*provider.ResourceInfo), nil
}
func (m *manager) GetKeys(keys []string) ([]*provider.ResourceInfo, error) {
infos := make([]*provider.ResourceInfo, len(keys))
for i, key := range keys {
if infoIf, err := m.cache.Get(key); err == nil {
infos[i] = infoIf.(*provider.ResourceInfo)
}
}
return infos, nil
}
func (m *manager) Set(key string, info *provider.ResourceInfo) error {
return m.cache.Set(key, info)
}
func (m *manager) SetWithExpire(key string, info *provider.ResourceInfo, expiration time.Duration) error {
return m.cache.SetWithExpire(key, info, expiration)
}
-153
View File
@@ -1,153 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package redis
import (
"encoding/json"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/share/cache"
"github.com/cs3org/reva/v2/pkg/share/cache/registry"
"github.com/gomodule/redigo/redis"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("redis", New)
}
type config struct {
RedisAddress string `mapstructure:"redis_address"`
RedisUsername string `mapstructure:"redis_username"`
RedisPassword string `mapstructure:"redis_password"`
}
type manager struct {
redisPool *redis.Pool
}
// New returns an implementation of a resource info cache that stores the objects in a redis cluster
func New(m map[string]interface{}) (cache.ResourceInfoCache, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, errors.Wrap(err, "error decoding conf")
}
if c.RedisAddress == "" {
c.RedisAddress = "localhost:6379"
}
pool := &redis.Pool{
MaxIdle: 50,
MaxActive: 1000,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
var opts []redis.DialOption
if c.RedisUsername != "" {
opts = append(opts, redis.DialUsername(c.RedisUsername))
}
if c.RedisPassword != "" {
opts = append(opts, redis.DialPassword(c.RedisPassword))
}
c, err := redis.Dial("tcp", c.RedisAddress, opts...)
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return &manager{
redisPool: pool,
}, nil
}
func (m *manager) Get(key string) (*provider.ResourceInfo, error) {
infos, err := m.getVals([]string{key})
if err != nil {
return nil, err
}
return infos[0], nil
}
func (m *manager) GetKeys(keys []string) ([]*provider.ResourceInfo, error) {
return m.getVals(keys)
}
func (m *manager) Set(key string, info *provider.ResourceInfo) error {
return m.setVal(key, info, -1)
}
func (m *manager) SetWithExpire(key string, info *provider.ResourceInfo, expiration time.Duration) error {
return m.setVal(key, info, int(expiration.Seconds()))
}
func (m *manager) setVal(key string, info *provider.ResourceInfo, expiration int) error {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
encodedInfo, err := json.Marshal(&info)
if err != nil {
return err
}
args := []interface{}{key, encodedInfo}
if expiration != -1 {
args = append(args, "EX", expiration)
}
if _, err := conn.Do("SET", args); err != nil {
return err
}
return nil
}
return errors.New("cache: unable to get connection from redis pool")
}
func (m *manager) getVals(keys []string) ([]*provider.ResourceInfo, error) {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
vals, err := redis.Strings(conn.Do("MGET", keys))
if err != nil {
return nil, err
}
infos := make([]*provider.ResourceInfo, len(keys))
for i, v := range vals {
if v != "" {
if err = json.Unmarshal([]byte(v), &infos[i]); err != nil {
infos[i] = nil
}
}
}
return infos, nil
}
return nil, errors.New("cache: unable to get connection from redis pool")
}
-34
View File
@@ -1,34 +0,0 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/share/cache"
// NewFunc is the function that cache implementations
// should register at init time.
type NewFunc func(map[string]interface{}) (cache.ResourceInfoCache, error)
// NewFuncs is a map containing all the registered cache implementations.
var NewFuncs = map[string]NewFunc{}
// Register registers a new cache function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+43 -87
View File
@@ -26,11 +26,7 @@ import (
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
natsjs "github.com/go-micro/plugins/v4/store/nats-js"
"github.com/go-micro/plugins/v4/store/redis"
redisopts "github.com/go-redis/redis/v8"
"github.com/nats-io/nats.go"
microetcd "github.com/owncloud/ocis/v2/ocis-pkg/store/etcd"
"github.com/cs3org/reva/v2/pkg/store"
"github.com/shamaton/msgpack/v2"
microstore "go-micro.dev/v4/store"
)
@@ -45,7 +41,20 @@ var (
mutex sync.Mutex
)
// Config contains the configuring for a cache
type Config struct {
Store string `mapstructure:"cache_store"`
Nodes []string `mapstructure:"cache_nodes"`
Database string `mapstructure:"cache_database"`
Table string `mapstructure:"cache_table"`
TTL int `mapstructure:"cache_ttl"`
Size int `mapstructure:"cache_size"`
}
// Cache handles key value operations on caches
// It, and the interfaces derived from it, are currently being used
// for building caches around go-micro stores, encoding the data
// in the messsagepack format.
type Cache interface {
PullFromCache(key string, dest interface{}) error
PushToCache(key string, src interface{}) error
@@ -89,65 +98,65 @@ type FileMetadataCache interface {
// GetStatCache will return an existing StatCache for the given store, nodes, database and table
// If it does not exist yet it will be created, different TTLs are ignored
func GetStatCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration) StatCache {
func GetStatCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) StatCache {
mutex.Lock()
defer mutex.Unlock()
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
if statCaches[key] == nil {
statCaches[key] = NewStatCache(cacheStore, cacheNodes, database, table, ttl)
statCaches[key] = NewStatCache(cacheStore, cacheNodes, database, table, ttl, size)
}
return statCaches[key]
}
// GetProviderCache will return an existing ProviderCache for the given store, nodes, database and table
// If it does not exist yet it will be created, different TTLs are ignored
func GetProviderCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration) ProviderCache {
func GetProviderCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) ProviderCache {
mutex.Lock()
defer mutex.Unlock()
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
if providerCaches[key] == nil {
providerCaches[key] = NewProviderCache(cacheStore, cacheNodes, database, table, ttl)
providerCaches[key] = NewProviderCache(cacheStore, cacheNodes, database, table, ttl, size)
}
return providerCaches[key]
}
// GetCreateHomeCache will return an existing CreateHomeCache for the given store, nodes, database and table
// If it does not exist yet it will be created, different TTLs are ignored
func GetCreateHomeCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration) CreateHomeCache {
func GetCreateHomeCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) CreateHomeCache {
mutex.Lock()
defer mutex.Unlock()
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
if createHomeCaches[key] == nil {
createHomeCaches[key] = NewCreateHomeCache(cacheStore, cacheNodes, database, table, ttl)
createHomeCaches[key] = NewCreateHomeCache(cacheStore, cacheNodes, database, table, ttl, size)
}
return createHomeCaches[key]
}
// GetCreatePersonalSpaceCache will return an existing CreatePersonalSpaceCache for the given store, nodes, database and table
// If it does not exist yet it will be created, different TTLs are ignored
func GetCreatePersonalSpaceCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration) CreatePersonalSpaceCache {
func GetCreatePersonalSpaceCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) CreatePersonalSpaceCache {
mutex.Lock()
defer mutex.Unlock()
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
if createPersonalSpaceCaches[key] == nil {
createPersonalSpaceCaches[key] = NewCreatePersonalSpaceCache(cacheStore, cacheNodes, database, table, ttl)
createPersonalSpaceCaches[key] = NewCreatePersonalSpaceCache(cacheStore, cacheNodes, database, table, ttl, size)
}
return createPersonalSpaceCaches[key]
}
// GetFileMetadataCache will return an existing GetFileMetadataCache for the given store, nodes, database and table
// If it does not exist yet it will be created, different TTLs are ignored
func GetFileMetadataCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration) FileMetadataCache {
func GetFileMetadataCache(cacheStore string, cacheNodes []string, database, table string, ttl time.Duration, size int) FileMetadataCache {
mutex.Lock()
defer mutex.Unlock()
key := strings.Join(append(append([]string{cacheStore}, cacheNodes...), database, table), ":")
if fileMetadataCaches[key] == nil {
fileMetadataCaches[key] = NewFileMetadataCache(cacheStore, cacheNodes, database, table, ttl)
fileMetadataCaches[key] = NewFileMetadataCache(cacheStore, cacheNodes, database, table, ttl, size)
}
return fileMetadataCaches[key]
}
@@ -159,77 +168,6 @@ type cacheStore struct {
ttl time.Duration
}
// NewCache initializes a new CacheStore
func NewCache(store string, nodes []string, database, table string, ttl time.Duration) Cache {
return cacheStore{
s: getStore(store, nodes, database, table, ttl), // some stores use a default ttl so we pass it when initializing
database: database,
table: table,
ttl: ttl, // some stores use the ttl on every write, so we remember it here
}
}
func getStore(store string, nodes []string, database, table string, ttl time.Duration) microstore.Store {
switch store {
case "etcd":
return microetcd.NewEtcdStore(
microstore.Nodes(nodes...),
microstore.Database(database),
microstore.Table(table),
)
case "nats-js":
// TODO nats needs a DefaultTTL option as it does not support per Write TTL ...
// FIXME nats has restrictions on the key, we cannot use slashes AFAICT
// host, port, clusterid
return natsjs.NewStore(
microstore.Nodes(nodes...),
microstore.Database(database),
microstore.Table(table),
natsjs.NatsOptions(nats.Options{Name: "TODO"}),
natsjs.DefaultTTL(ttl),
) // TODO test with ocis nats
case "redis":
return redis.NewStore(
microstore.Database(database),
microstore.Table(table),
microstore.Nodes(nodes...),
) // only the first node is taken into account
case "redis-sentinel":
redisMaster := ""
redisNodes := []string{}
for _, node := range nodes {
parts := strings.SplitN(node, "/", 2)
if len(parts) != 2 {
return nil
}
// the first node is used to retrieve the redis master
redisNodes = append(redisNodes, parts[0])
if redisMaster == "" {
redisMaster = parts[1]
}
}
return redis.NewStore(
microstore.Database(database),
microstore.Table(table),
microstore.Nodes(redisNodes...),
redis.WithRedisOptions(redisopts.UniversalOptions{
MasterName: redisMaster,
}),
)
case "memory":
return microstore.NewStore(
microstore.Database(database),
microstore.Table(table),
)
default:
return microstore.NewNoopStore(
microstore.Database(database),
microstore.Table(table),
)
}
}
// PullFromCache pulls a value from the configured database and table of the underlying store using the given key
func (cache cacheStore) PullFromCache(key string, dest interface{}) error {
r, err := cache.s.Read(key, microstore.ReadFrom(cache.database, cache.table), microstore.ReadLimit(1))
@@ -249,8 +187,15 @@ func (cache cacheStore) PushToCache(key string, src interface{}) error {
if err != nil {
return err
}
record := &microstore.Record{
Key: key,
Value: b,
Expiry: cache.ttl,
}
return cache.s.Write(
&microstore.Record{Key: key, Value: b},
record,
microstore.WriteTo(cache.database, cache.table),
microstore.WriteTTL(cache.ttl),
)
@@ -285,3 +230,14 @@ func (cache cacheStore) Delete(key string, opts ...microstore.DeleteOption) erro
func (cache cacheStore) Close() error {
return cache.s.Close()
}
func getStore(storeType string, nodes []string, database, table string, ttl time.Duration, size int) microstore.Store {
return store.Create(
store.Store(storeType),
microstore.Nodes(nodes...),
microstore.Database(database),
microstore.Table(table),
store.TTL(ttl),
store.Size(size),
)
}
+2 -2
View File
@@ -32,9 +32,9 @@ type createHomeCache struct {
}
// NewCreateHomeCache creates a new CreateHomeCache
func NewCreateHomeCache(store string, nodes []string, database, table string, ttl time.Duration) CreateHomeCache {
func NewCreateHomeCache(store string, nodes []string, database, table string, ttl time.Duration, size int) CreateHomeCache {
c := &createHomeCache{}
c.s = getStore(store, nodes, database, table, ttl)
c.s = getStore(store, nodes, database, table, ttl, size)
c.database = database
c.table = table
c.ttl = ttl
+2 -2
View File
@@ -30,9 +30,9 @@ type createPersonalSpaceCache struct {
}
// NewCreatePersonalSpaceCache creates a new CreatePersonalSpaceCache
func NewCreatePersonalSpaceCache(store string, nodes []string, database, table string, ttl time.Duration) CreatePersonalSpaceCache {
func NewCreatePersonalSpaceCache(store string, nodes []string, database, table string, ttl time.Duration, size int) CreatePersonalSpaceCache {
c := &createPersonalSpaceCache{}
c.s = getStore(store, nodes, database, table, ttl)
c.s = getStore(store, nodes, database, table, ttl, size)
c.database = database
c.table = table
c.ttl = ttl
+2 -2
View File
@@ -28,9 +28,9 @@ type fileMetadataCache struct {
}
// NewFileMetadataCache creates a new FileMetadataCache
func NewFileMetadataCache(store string, nodes []string, database, table string, ttl time.Duration) FileMetadataCache {
func NewFileMetadataCache(store string, nodes []string, database, table string, ttl time.Duration, size int) FileMetadataCache {
c := &fileMetadataCache{}
c.s = getStore(store, nodes, database, table, ttl)
c.s = getStore(store, nodes, database, table, ttl, size)
c.database = database
c.table = table
c.ttl = ttl
+2 -2
View File
@@ -32,9 +32,9 @@ type providerCache struct {
}
// NewProviderCache creates a new ProviderCache
func NewProviderCache(store string, nodes []string, database, table string, ttl time.Duration) ProviderCache {
func NewProviderCache(store string, nodes []string, database, table string, ttl time.Duration, size int) ProviderCache {
c := &providerCache{}
c.s = getStore(store, nodes, database, table, ttl)
c.s = getStore(store, nodes, database, table, ttl, size)
c.database = database
c.table = table
c.ttl = ttl
+2 -2
View File
@@ -27,9 +27,9 @@ import (
)
// NewStatCache creates a new StatCache
func NewStatCache(store string, nodes []string, database, table string, ttl time.Duration) StatCache {
func NewStatCache(store string, nodes []string, database, table string, ttl time.Duration, size int) StatCache {
c := statCache{}
c.s = getStore(store, nodes, database, table, ttl)
c.s = getStore(store, nodes, database, table, ttl, size)
c.database = database
c.table = table
c.ttl = ttl
@@ -160,7 +160,7 @@ func New(o *options.Options, lu *lookup.Lookup, p Permissions, tp Tree, es event
p: p,
chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")),
stream: es,
cache: cache.GetStatCache(o.StatCache.CacheStore, o.StatCache.CacheNodes, o.StatCache.CacheDatabase, "stat", 0),
cache: cache.GetStatCache(o.StatCache.Store, o.StatCache.Nodes, o.StatCache.Database, "stat", time.Duration(o.StatCache.TTL)*time.Second, o.StatCache.Size),
}
if o.AsyncFileUploads {
@@ -27,7 +27,6 @@ import (
"time"
"github.com/cs3org/reva/v2/pkg/storage/cache"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
"github.com/pkg/xattr"
"github.com/rogpeppe/go-internal/lockedfile"
"github.com/shamaton/msgpack/v2"
@@ -46,10 +45,10 @@ type readWriteCloseSeekTruncater interface {
}
// NewMessagePackBackend returns a new MessagePackBackend instance
func NewMessagePackBackend(rootPath string, o options.CacheOptions) MessagePackBackend {
func NewMessagePackBackend(rootPath string, o cache.Config) MessagePackBackend {
return MessagePackBackend{
rootPath: filepath.Clean(rootPath),
metaCache: cache.GetFileMetadataCache(o.CacheStore, o.CacheNodes, o.CacheDatabase, "filemetadata", 24*time.Hour),
metaCache: cache.GetFileMetadataCache(o.Store, o.Nodes, o.Database, "filemetadata", time.Duration(o.TTL)*time.Second, o.Size),
}
}
@@ -25,9 +25,9 @@ import (
"path/filepath"
"strings"
"github.com/cs3org/reva/v2/pkg/storage/cache"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
)
// Migration0003 migrates the file metadata to the current backend.
@@ -44,7 +44,7 @@ func (m *Migrator) Migration0003() (Result, error) {
m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...")
xattrs := metadata.XattrsBackend{}
mpk := metadata.NewMessagePackBackend(m.lu.InternalRoot(), options.CacheOptions{})
mpk := metadata.NewMessagePackBackend(m.lu.InternalRoot(), cache.Config{})
spaces, _ := filepath.Glob(filepath.Join(m.lu.InternalRoot(), "spaces", "*", "*"))
for _, space := range spaces {
@@ -24,6 +24,7 @@ import (
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/sharedconf"
"github.com/cs3org/reva/v2/pkg/storage/cache"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
@@ -63,8 +64,8 @@ type Options struct {
Tokens TokenOptions `mapstructure:"tokens"`
StatCache CacheOptions `mapstructure:"statcache"`
FileMetadataCache CacheOptions `mapstructure:"filemetadatacache"`
StatCache cache.Config `mapstructure:"statcache"`
FileMetadataCache cache.Config `mapstructure:"filemetadatacache"`
MaxAcquireLockCycles int `mapstructure:"max_acquire_lock_cycles"`
LockCycleDurationFactor int `mapstructure:"lock_cycle_duration_factor"`
@@ -90,13 +91,6 @@ type TokenOptions struct {
TransferExpires int64 `mapstructure:"transfer_expires"`
}
// CacheOptions contains options of configuring a cache
type CacheOptions struct {
CacheStore string `mapstructure:"cache_store"`
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
}
// New returns a new Options instance for the given configuration
func New(m map[string]interface{}) (*Options, error) {
o := &Options{}
@@ -246,6 +246,9 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide
case provider.ListStorageSpacesRequest_Filter_TYPE_USER:
// TODO: refactor this to GetUserId() in cs3
requestedUserID = filter[i].GetUser().GetOpaqueId()
case provider.ListStorageSpacesRequest_Filter_TYPE_OWNER:
// TODO: improve further by not evaluating shares
requestedUserID = filter[i].GetOwner().GetOpaqueId()
}
}
if len(spaceTypes) == 0 {
+532
View File
@@ -0,0 +1,532 @@
package etcd
import (
"context"
"encoding/json"
"strings"
"time"
"go-micro.dev/v4/store"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
)
const (
prefixNS = ".prefix"
suffixNS = ".suffix"
)
// Store is a store implementation which uses etcd to store the data
type Store struct {
options store.Options
client *clientv3.Client
}
// NewStore creates a new go-micro store backed by etcd
func NewStore(opts ...store.Option) store.Store {
es := &Store{}
_ = es.Init(opts...)
return es
}
func (es *Store) getCtx() (context.Context, context.CancelFunc) {
currentCtx := es.options.Context
if currentCtx == nil {
currentCtx = context.TODO()
}
ctx, cancel := context.WithTimeout(currentCtx, 10*time.Second)
return ctx, cancel
}
// Setup the etcd client based on the current options. The old client (if any)
// will be closed.
// Currently, only the etcd nodes are configurable. If no node is provided,
// it will use the "127.0.0.1:2379" node.
// Context timeout is setup to 10 seconds, and dial timeout to 2 seconds
func (es *Store) setupClient() {
if es.client != nil {
es.client.Close()
}
endpoints := []string{"127.0.0.1:2379"}
if len(es.options.Nodes) > 0 {
endpoints = es.options.Nodes
}
cli, _ := clientv3.New(clientv3.Config{
DialTimeout: 2 * time.Second,
Endpoints: endpoints,
})
es.client = cli
}
// Init initializes the go-micro store implementation.
// Currently, only the nodes are configurable, the rest of the options
// will be ignored.
func (es *Store) Init(opts ...store.Option) error {
optList := store.Options{}
for _, opt := range opts {
opt(&optList)
}
es.options = optList
es.setupClient()
return nil
}
// Options returns the store options
func (es *Store) Options() store.Options {
return es.options
}
// Get the effective TTL, as int64 number of seconds. It will prioritize
// the TTL set in the options, then the expiry time in the options, and
// finally the one set as part of the record
func getEffectiveTTL(r *store.Record, opts store.WriteOptions) int64 {
// set base ttl duration and expiration time based on the record
duration := r.Expiry
// overwrite ttl duration and expiration time based on options
if !opts.Expiry.IsZero() {
// options.Expiry is a time.Time, newRecord.Expiry is a time.Duration
duration = time.Until(opts.Expiry)
}
// TTL option takes precedence over expiration time
if opts.TTL != 0 {
duration = opts.TTL
}
// use milliseconds because it returns an int64 instead of a float64
return duration.Milliseconds() / 1000
}
// Write the record into the etcd. The record will be duplicated in order to
// find it by prefix or by suffix. This means that it will take double space.
// Note that this is an implementation detail and it will be handled
// transparently.
//
// Database and Table options will be used to provide a different prefix to
// the key. Each service using this store should use a different database+table
// combination in order to prevent key collisions.
//
// Due to how TTLs are implemented in etcd, the minimum valid TTL seems to
// be 2 secs. Using lower values or even negative values will force the etcd
// server to use the minimum value instead.
// In addition, getting a lease for the TTL and attach it to the target key
// are 2 different operations that can't be sent as part of a transaction.
// This means that it's possible to get a lease and have that lease expire
// before attaching it to the key. Errors are expected to happen if this is
// the case, and no key will be inserted.
// According to etcd documentation, the key is guaranteed to be available
// AT LEAST the TTL duration. This means that the key might be available for
// a longer period of time in special circumstances.
//
// It's recommended to use a minimum TTL of 10 secs or higher (or not to use
// TTL) in order to prevent problematic scenarios.
func (es *Store) Write(r *store.Record, opts ...store.WriteOption) error {
wopts := store.WriteOptions{}
for _, opt := range opts {
opt(&wopts)
}
prefix := buildPrefix(wopts.Database, wopts.Table, prefixNS)
suffix := buildPrefix(wopts.Database, wopts.Table, suffixNS)
kv := es.client.KV
jsonRecord, err := json.Marshal(r)
if err != nil {
return err
}
jsonStringRecord := string(jsonRecord)
effectiveTTL := getEffectiveTTL(r, wopts)
var opOpts []clientv3.OpOption
if effectiveTTL != 0 {
lease := es.client.Lease
ctx, cancel := es.getCtx()
gResp, gErr := lease.Grant(ctx, getEffectiveTTL(r, wopts))
cancel()
if gErr != nil {
return gErr
}
opOpts = []clientv3.OpOption{clientv3.WithLease(gResp.ID)}
} else {
opOpts = []clientv3.OpOption{clientv3.WithLease(0)}
}
ctx, cancel := es.getCtx()
_, err = kv.Txn(ctx).Then(
clientv3.OpPut(prefix+r.Key, jsonStringRecord, opOpts...),
clientv3.OpPut(suffix+reverseString(r.Key), jsonStringRecord, opOpts...),
).Commit()
cancel()
return err
}
// Process a Get response taking into account the provided offset
func processGetResponse(resp *clientv3.GetResponse, offset int64) ([]*store.Record, error) {
result := make([]*store.Record, 0, len(resp.Kvs))
for index, kvs := range resp.Kvs {
if int64(index) < offset {
// skip entries before the offset
continue
}
value := &store.Record{}
err := json.Unmarshal(kvs.Value, value)
if err != nil {
return nil, err
}
result = append(result, value)
}
return result, nil
}
// Process a List response taking into account the provided offset.
// The reverse flag will be used to reverse the keys found. For example,
// "zyxw" will be reversed to "wxyz". This is used for suffix searches,
// where the keys are stored reversed and need to be changed
func processListResponse(resp *clientv3.GetResponse, offset int64, reverse bool) ([]string, error) {
result := make([]string, 0, len(resp.Kvs))
for index, kvs := range resp.Kvs {
if int64(index) < offset {
// skip entries before the offset
continue
}
targetKey := string(kvs.Key)
if reverse {
targetKey = reverseString(targetKey)
}
result = append(result, targetKey)
}
return result, nil
}
// Perform an exact key read and return the result
func (es *Store) directRead(kv clientv3.KV, key string) ([]*store.Record, error) {
ctx, cancel := es.getCtx()
resp, err := kv.Get(ctx, key)
cancel()
if err != nil {
return nil, err
}
if len(resp.Kvs) == 0 {
return nil, store.ErrNotFound
}
return processGetResponse(resp, 0)
}
// Perform a prefix read with limit and offset. A limit of 0 will return all
// results. Usage of offset isn't recommended because those results must still
// be fethed from the server in order to be discarded.
func (es *Store) prefixRead(kv clientv3.KV, key string, limit, offset int64) ([]*store.Record, error) {
getOptions := []clientv3.OpOption{
clientv3.WithPrefix(),
}
if limit > 0 {
getOptions = append(getOptions, clientv3.WithLimit(limit+offset))
}
ctx, cancel := es.getCtx()
resp, err := kv.Get(ctx, key, getOptions...)
cancel()
if err != nil {
return nil, err
}
return processGetResponse(resp, offset)
}
// Perform a prefix + suffix read with limit and offset. A limit of 0 will
// return all results found. Usage of this function is discouraged because
// we'll have to request a prefix search and match the suffix manually. This
// means that even with a limit = 3 and offset = 0, there is no guarantee
// we'll find all the results we need within that range, and we'll likely
// need to request more data from the server. The number of requests we need
// to perform is unknown and might cause load.
func (es *Store) prefixSuffixRead(kv clientv3.KV, prefix, suffix string, limit, offset int64) ([]*store.Record, error) {
firstKeyOut := firstKeyOutOfPrefixString(prefix)
getOptions := []clientv3.OpOption{
clientv3.WithRange(firstKeyOut),
}
if limit > 0 {
// unlikely to find all the entries we need within offset + limit
getOptions = append(getOptions, clientv3.WithLimit((limit+offset)*2))
}
var currentRecordOffset int64
result := []*store.Record{}
initialKey := prefix
keepGoing := true
for keepGoing {
ctx, cancel := es.getCtx()
resp, respErr := kv.Get(ctx, initialKey, getOptions...)
cancel()
if respErr != nil {
return nil, respErr
}
records, err := processGetResponse(resp, 0)
if err != nil {
return nil, err
}
for _, record := range records {
if !strings.HasSuffix(record.Key, suffix) {
continue
}
if currentRecordOffset < offset {
currentRecordOffset++
continue
}
if !shouldFinish(int64(len(result)), limit) {
result = append(result, record)
if shouldFinish(int64(len(result)), limit) {
break
}
}
}
if !resp.More || shouldFinish(int64(len(result)), limit) {
keepGoing = false
} else {
initialKey = string(append(resp.Kvs[len(resp.Kvs)-1].Key, 0)) // append byte 0 (nul char) to the last key
}
}
return result, nil
}
// Read records from the etcd server based in the key. Database and Table
// options are highly recommended, otherwise we'll use a default one (which
// might not have the requested keys)
//
// If no prefix or suffix option is provided, we'll read the record matching
// the provided key. Note that a list of records will be provided anyway,
// likely with only one record (the one requested)
//
// Prefix and suffix options are supported and should perform fine even with
// a large amount of data. Note that the limit option should also be included
// in order to limit the amount of records we need to fetch.
//
// Note that using both prefix and suffix options at the same time is possible
// but discouraged. A prefix search will be send to the etcd server, and from
// there we'll manually pick the records matching the suffix. This might become
// very inefficient since we might need to request more data to the etcd
// multiple times in order to provide the results asked.
// Usage of the offset option is also discouraged because we'll have to request
// records that we'll have to skip manually on our side.
//
// Don't rely on any particular order of the keys. The records are expected to
// be sorted by key except if the suffix option (suffix without prefix) is
// used. In this case, the keys will be sorted based on the reversed key
func (es *Store) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
ropts := store.ReadOptions{}
for _, opt := range opts {
opt(&ropts)
}
prefix := buildPrefix(ropts.Database, ropts.Table, prefixNS)
suffix := buildPrefix(ropts.Database, ropts.Table, suffixNS)
kv := es.client.KV
preKv := namespace.NewKV(kv, prefix)
sufKv := namespace.NewKV(kv, suffix)
if ropts.Prefix && ropts.Suffix {
return es.prefixSuffixRead(preKv, key, key, int64(ropts.Limit), int64(ropts.Offset))
}
if ropts.Prefix {
return es.prefixRead(preKv, key, int64(ropts.Limit), int64(ropts.Offset))
}
if ropts.Suffix {
return es.prefixRead(sufKv, reverseString(key), int64(ropts.Limit), int64(ropts.Offset))
}
return es.directRead(preKv, key)
}
// Delete the record containing the key provided. Database and Table
// options are highly recommended, otherwise we'll use a default one (which
// might not have the requested keys)
//
// Since the Write method inserts 2 entries for a given key, those both
// entries will also be removed using the same key. This is handled
// transparently.
func (es *Store) Delete(key string, opts ...store.DeleteOption) error {
dopts := store.DeleteOptions{}
for _, opt := range opts {
opt(&dopts)
}
prefix := buildPrefix(dopts.Database, dopts.Table, prefixNS)
suffix := buildPrefix(dopts.Database, dopts.Table, suffixNS)
kv := es.client.KV
ctx, cancel := es.getCtx()
_, err := kv.Txn(ctx).Then(
clientv3.OpDelete(prefix+key),
clientv3.OpDelete(suffix+reverseString(key)),
).Commit()
cancel()
return err
}
// List the keys based on the provided prefix. Use the empty string (and no
// limit nor offset) to list all keys available.
// Limit and offset options are available to limit the keys we need to return.
// The reverse option will reverse the keys before returning them. Use it when
// listing the keys from the suffix KV.
//
// Note that values for the keys won't be requested to the etcd server, that's
// why the reverse option is important
func (es *Store) listKeys(kv clientv3.KV, prefixKey string, limit, offset int64, reverse bool) ([]string, error) {
getOptions := []clientv3.OpOption{
clientv3.WithKeysOnly(),
clientv3.WithPrefix(),
}
if limit > 0 {
getOptions = append(getOptions, clientv3.WithLimit(limit+offset))
}
ctx, cancel := es.getCtx()
resp, err := kv.Get(ctx, prefixKey, getOptions...)
cancel()
if err != nil {
return nil, err
}
return processListResponse(resp, offset, reverse)
}
// List the keys matching both prefix and suffix, with the provided limit and
// offset. Usage of this function is discouraged because we'll have to match
// the suffix manually on our side, which means we'll likely need to perform
// additional requests to the etcd server to get more results matching all the
// requirements.
func (es *Store) prefixSuffixList(kv clientv3.KV, prefix, suffix string, limit, offset int64) ([]string, error) {
firstKeyOut := firstKeyOutOfPrefixString(prefix)
getOptions := []clientv3.OpOption{
clientv3.WithKeysOnly(),
clientv3.WithRange(firstKeyOut),
}
if firstKeyOut == "" {
// could happen of all bytes are "\xff"
getOptions = getOptions[:1] // remove the WithRange option
}
if limit > 0 {
// unlikely to find all the entries we need within offset + limit
getOptions = append(getOptions, clientv3.WithLimit((limit+offset)*2))
}
var currentRecordOffset int64
result := []string{}
initialKey := prefix
keepGoing := true
for keepGoing {
ctx, cancel := es.getCtx()
resp, respErr := kv.Get(ctx, initialKey, getOptions...)
cancel()
if respErr != nil {
return nil, respErr
}
keys, err := processListResponse(resp, 0, false)
if err != nil {
return nil, err
}
for _, key := range keys {
if !strings.HasSuffix(key, suffix) {
continue
}
if currentRecordOffset < offset {
currentRecordOffset++
continue
}
if !shouldFinish(int64(len(result)), limit) {
result = append(result, key)
if shouldFinish(int64(len(result)), limit) {
break
}
}
}
if !resp.More || shouldFinish(int64(len(result)), limit) {
keepGoing = false
} else {
initialKey = string(append(resp.Kvs[len(resp.Kvs)-1].Key, 0)) // append byte 0 (nul char) to the last key
}
}
return result, nil
}
// List the keys available in the etcd server. Database and Table
// options are highly recommended, otherwise we'll use a default one (which
// might not have the requested keys)
//
// With the Database and Table options, all the keys returned will be within
// that database and table. Each service is expected to use a different
// database + table, so using those options will list only the keys used by
// that particular service.
//
// Prefix and suffix options are available along with the limit and offset
// ones.
//
// Using prefix and suffix options at the same time is discourage because
// the suffix matching will be done on our side, and we'll likely need to
// perform multiple requests to get the requested results. Note that using
// just the suffix option is fine.
// In addition, using the offset option is also discouraged because we'll
// need to request additional keys that will be skipped on our side.
func (es *Store) List(opts ...store.ListOption) ([]string, error) {
lopts := store.ListOptions{}
for _, opt := range opts {
opt(&lopts)
}
prefix := buildPrefix(lopts.Database, lopts.Table, prefixNS)
suffix := buildPrefix(lopts.Database, lopts.Table, suffixNS)
kv := es.client.KV
preKv := namespace.NewKV(kv, prefix)
sufKv := namespace.NewKV(kv, suffix)
if lopts.Prefix != "" && lopts.Suffix != "" {
return es.prefixSuffixList(preKv, lopts.Prefix, lopts.Suffix, int64(lopts.Limit), int64(lopts.Offset))
}
if lopts.Prefix != "" {
return es.listKeys(preKv, lopts.Prefix, int64(lopts.Limit), int64(lopts.Offset), false)
}
if lopts.Suffix != "" {
return es.listKeys(sufKv, reverseString(lopts.Suffix), int64(lopts.Limit), int64(lopts.Offset), true)
}
return es.listKeys(preKv, "", int64(lopts.Limit), int64(lopts.Offset), false)
}
// Close the client
func (es *Store) Close() error {
return es.client.Close()
}
// Return the service name
func (es *Store) String() string {
return "Etcd"
}
+65
View File
@@ -0,0 +1,65 @@
package etcd
import (
"strings"
)
// Returns true if the limit isn't 0 AND is greater or equal to the number
// of results.
// If the limit is 0 or the number of items is less than the number of items,
// it will return false
func shouldFinish(numberOfResults, limit int64) bool {
if limit == 0 || numberOfResults < limit {
return false
}
return true
}
// Return the first key out of the prefix represented by the parameter,
// as a byte sequence. Note that it applies to byte sequences and not
// rune sequences, so it might be ill-suited for multi-byte chars
func firstKeyOutOfPrefix(src []byte) []byte {
dst := make([]byte, len(src))
copy(dst, src)
var i int
for i = len(dst) - 1; i >= 0; i-- {
if dst[i] < 255 {
dst[i]++
break
}
}
return dst[:i+1]
}
// Return the first key out of the prefix represented by the parameter.
// This function relies on the firstKeyOutOfPrefix one, which uses a byte
// sequence, so it might be ill-suited if the string contains multi-byte chars.
func firstKeyOutOfPrefixString(src string) string {
srcBytes := []byte(src)
dstBytes := firstKeyOutOfPrefix(srcBytes)
return string(dstBytes)
}
// Reverse the string based on the containing runes
func reverseString(s string) string {
r := []rune(s)
for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
r[i], r[j] = r[j], r[i]
}
return string(r)
}
// Build a string based on the parts, to be used as a prefix. Empty string is
// expected if no part is passed as parameter.
// The string will contain all the parts separated by '/'. The last char will
// also be '/'
//
// For example `buildPrefix(P1, P2, P3)` will return "P1/P2/P3/"
func buildPrefix(parts ...string) string {
var b strings.Builder
for _, part := range parts {
b.WriteString(part)
b.WriteRune('/')
}
return b.String()
}
+518
View File
@@ -0,0 +1,518 @@
package memory
import (
"container/list"
"context"
"strings"
"sync"
"time"
"github.com/armon/go-radix"
"go-micro.dev/v4/store"
)
// MemStore is a in-memory store implementation using radix tree for fast
// prefix and suffix searches.
// Insertions are expected to be a bit slow due to the data structures, but
// searches are expected to be fast, including exact key search, as well as
// prefix and suffix searches (based on the number of elements to be returned).
// Prefix+suffix search isn't optimized and will depend on how many items we
// need to skip.
// It's also recommended to use reasonable limits when using prefix or suffix
// searches because we'll need to traverse the data structures to provide the
// results. The traversal will stop a soon as we have the required number of
// results, so it will be faster if we use a short limit.
//
// The overall performance will depend on how the radix trees are built.
// The number of elements won't directly affect the performance but how the
// keys are dispersed. The more dispersed the keys are, the faster the search
// will be, regardless of the number of keys. This happens due to the number
// of hops we need to do to reach the target element.
// This also mean that if the keys are too similar, the performance might be
// slower than expected even if the number of elements isn't too big.
type MemStore struct {
preRadix *radix.Tree
sufRadix *radix.Tree
evictionList *list.List
options store.Options
lockGlob sync.RWMutex
lockEvicList sync.RWMutex // Read operation will modify the eviction list
}
type storeRecord struct {
Key string
Value []byte
Metadata map[string]interface{}
Expiry time.Duration
ExpiresAt time.Time
}
type contextKey string
var targetContextKey contextKey
// NewContext prepares a context to be used with the memory implementation.
// The context is used to set up custom parameters to the specific implementation.
// In this case, you can configure the maximum capacity for the MemStore
// implementation as shown below.
// ```
// cache := NewMemStore(
//
// store.WithContext(
// NewContext(
// ctx,
// map[string]interface{}{
// "maxCap": 50,
// },
// ),
// ),
//
// )
// ```
//
// Available options for the MemStore are:
// * "maxCap" -> 512 (int) The maximum number of elements the cache will hold.
// Adding additional elements will remove old elements to ensure we aren't over
// the maximum capacity.
//
// For convenience, this can also be used for the MultiMemStore.
func NewContext(ctx context.Context, storeParams map[string]interface{}) context.Context {
return context.WithValue(ctx, targetContextKey, storeParams)
}
// NewMemStore creates a new MemStore instance
func NewMemStore(opts ...store.Option) store.Store {
m := &MemStore{}
_ = m.Init(opts...)
return m
}
// Get the maximum capacity configured. If no maxCap has been configured
// (via `NewContext`), 512 will be used as maxCap.
func (m *MemStore) getMaxCap() int {
maxCap := 512
ctx := m.options.Context
if ctx == nil {
return maxCap
}
ctxValue := ctx.Value(targetContextKey)
if ctxValue == nil {
return maxCap
}
additionalOpts := ctxValue.(map[string]interface{})
confCap, exists := additionalOpts["maxCap"]
if exists {
maxCap = confCap.(int)
}
return maxCap
}
// Init initializes the MemStore. If the MemStore was used, this will reset
// all the internal structures and the new options (passed as parameters)
// will be used.
func (m *MemStore) Init(opts ...store.Option) error {
optList := store.Options{}
for _, opt := range opts {
opt(&optList)
}
m.lockGlob.Lock()
defer m.lockGlob.Unlock()
m.preRadix = radix.New()
m.sufRadix = radix.New()
m.evictionList = list.New()
m.options = optList
return nil
}
// Options returns the options being used
func (m *MemStore) Options() store.Options {
m.lockGlob.RLock()
defer m.lockGlob.RUnlock()
return m.options
}
// Write the record in the MemStore.
// Note that Database and Table options will be ignored.
// Expiration options will take the following precedence:
// TTL option > expiration option > TTL record
//
// New elements will take the last position in the eviction list. Updating
// an element will also move the element to the last position.
//
// Although not recommended, new elements might be inserted with an
// already-expired date
func (m *MemStore) Write(r *store.Record, opts ...store.WriteOption) error {
var element *list.Element
wopts := store.WriteOptions{}
for _, opt := range opts {
opt(&wopts)
}
cRecord := toStoreRecord(r, wopts)
m.lockGlob.Lock()
defer m.lockGlob.Unlock()
ele, exists := m.preRadix.Get(cRecord.Key)
if exists {
element = ele.(*list.Element)
element.Value = cRecord
m.evictionList.MoveToBack(element)
} else {
if m.evictionList.Len() >= m.getMaxCap() {
elementToDelete := m.evictionList.Front()
if elementToDelete != nil {
recordToDelete := elementToDelete.Value.(*storeRecord)
_, _ = m.preRadix.Delete(recordToDelete.Key)
_, _ = m.sufRadix.Delete(recordToDelete.Key)
m.evictionList.Remove(elementToDelete)
}
}
element = m.evictionList.PushBack(cRecord)
_, _ = m.preRadix.Insert(cRecord.Key, element)
_, _ = m.sufRadix.Insert(reverseString(cRecord.Key), element)
}
return nil
}
// Read the key from the MemStore. A list of records will be returned even if
// you're asking for the exact key (only one record is expected in that case).
//
// Reading the exact element will move such element to the last position of
// the eviction list. This WON'T apply for prefix and / or suffix reads.
//
// This method guarantees that no expired element will be returned. For the
// case of exact read, the element will be removed and a "not found" error
// will be returned.
// For prefix and suffix reads, all the elements that we traverse through
// will be removed. This includes the elements we need to skip as well as
// the elements that might have gotten into the the result. Note that the
// elements that are over the limit won't be touched
//
// All read options are supported except Database and Table.
//
// For prefix and prefix+suffix options, the records will be returned in
// alphabetical order on the keys.
// For the suffix option (just suffix, no prefix), the records will be
// returned in alphabetical order after reversing the keys. This means,
// reverse all the keys and then sort them alphabetically. This just affects
// the sorting order; the keys will be returned as expected.
// This means that ["aboz", "caaz", "ziuz"] will be sorted as ["caaz", "aboz", "ziuz"]
// for the key "z" as suffix.
//
// Note that offset are supported but not recommended. There is no direct access
// to the record X. We'd need to skip all the records until we reach the specified
// offset, which could be problematic.
// Performance for prefix and suffix searches should be good assuming we limit
// the number of results we need to return.
func (m *MemStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
var element *list.Element
ropts := store.ReadOptions{}
for _, opt := range opts {
opt(&ropts)
}
if !ropts.Prefix && !ropts.Suffix {
m.lockGlob.RLock()
ele, exists := m.preRadix.Get(key)
if !exists {
m.lockGlob.RUnlock()
return nil, store.ErrNotFound
}
element = ele.(*list.Element)
record := element.Value.(*storeRecord)
if record.Expiry != 0 && record.ExpiresAt.Before(time.Now()) {
// record expired -> need to delete
m.lockGlob.RUnlock()
m.lockGlob.Lock()
defer m.lockGlob.Unlock()
m.evictionList.Remove(element)
_, _ = m.preRadix.Delete(key)
_, _ = m.sufRadix.Delete(reverseString(key))
return nil, store.ErrNotFound
}
m.lockEvicList.Lock()
m.evictionList.MoveToBack(element)
m.lockEvicList.Unlock()
foundRecords := []*store.Record{
fromStoreRecord(record),
}
m.lockGlob.RUnlock()
return foundRecords, nil
}
records := []*store.Record{}
expiredElements := make(map[string]*list.Element)
m.lockGlob.RLock()
if ropts.Prefix && ropts.Suffix {
// if we need to check both prefix and suffix, go through the
// prefix tree and skip elements without the right suffix. We
// don't need to check the suffix tree because the elements
// must be in both trees
m.preRadix.WalkPrefix(key, m.radixTreeCallBackCheckSuffix(ropts.Offset, ropts.Limit, key, &records, expiredElements))
} else {
if ropts.Prefix {
m.preRadix.WalkPrefix(key, m.radixTreeCallBack(ropts.Offset, ropts.Limit, &records, expiredElements))
}
if ropts.Suffix {
m.sufRadix.WalkPrefix(reverseString(key), m.radixTreeCallBack(ropts.Offset, ropts.Limit, &records, expiredElements))
}
}
m.lockGlob.RUnlock()
// if there are expired elements, get a write lock and delete the expired elements
if len(expiredElements) > 0 {
m.lockGlob.Lock()
for key, element := range expiredElements {
m.evictionList.Remove(element)
_, _ = m.preRadix.Delete(key)
_, _ = m.sufRadix.Delete(reverseString(key))
}
m.lockGlob.Unlock()
}
return records, nil
}
// Delete removes the record based on the key. It won't return any error if it's missing
//
// Database and Table options aren't supported
func (m *MemStore) Delete(key string, opts ...store.DeleteOption) error {
m.lockGlob.Lock()
defer m.lockGlob.Unlock()
ele, exists := m.preRadix.Get(key)
if exists {
element := ele.(*list.Element)
m.evictionList.Remove(element)
_, _ = m.preRadix.Delete(key)
_, _ = m.sufRadix.Delete(reverseString(key))
}
return nil
}
// List the keys currently used in the MemStore
//
// # All options are supported except Database and Table
//
// For prefix and prefix+suffix options, the keys will be returned in
// alphabetical order.
// For the suffix option (just suffix, no prefix), the keys will be
// returned in alphabetical order after reversing the keys. This means,
// reverse all the keys and then sort them alphabetically. This just affects
// the sorting order; the keys will be returned as expected.
// This means that ["aboz", "caaz", "ziuz"] will be sorted as ["caaz", "aboz", "ziuz"]
func (m *MemStore) List(opts ...store.ListOption) ([]string, error) {
records := []string{}
expiredElements := make(map[string]*list.Element)
lopts := store.ListOptions{}
for _, opt := range opts {
opt(&lopts)
}
if lopts.Prefix == "" && lopts.Suffix == "" {
m.lockGlob.RLock()
m.preRadix.Walk(m.radixTreeCallBackKeysOnly(lopts.Offset, lopts.Limit, &records, expiredElements))
m.lockGlob.RUnlock()
// if there are expired elements, get a write lock and delete the expired elements
if len(expiredElements) > 0 {
m.lockGlob.Lock()
for key, element := range expiredElements {
m.evictionList.Remove(element)
_, _ = m.preRadix.Delete(key)
_, _ = m.sufRadix.Delete(reverseString(key))
}
m.lockGlob.Unlock()
}
return records, nil
}
m.lockGlob.RLock()
if lopts.Prefix != "" && lopts.Suffix != "" {
// if we need to check both prefix and suffix, go through the
// prefix tree and skip elements without the right suffix. We
// don't need to check the suffix tree because the elements
// must be in both trees
m.preRadix.WalkPrefix(lopts.Prefix, m.radixTreeCallBackKeysOnlyWithSuffix(lopts.Offset, lopts.Limit, lopts.Suffix, &records, expiredElements))
} else {
if lopts.Prefix != "" {
m.preRadix.WalkPrefix(lopts.Prefix, m.radixTreeCallBackKeysOnly(lopts.Offset, lopts.Limit, &records, expiredElements))
}
if lopts.Suffix != "" {
m.sufRadix.WalkPrefix(reverseString(lopts.Suffix), m.radixTreeCallBackKeysOnly(lopts.Offset, lopts.Limit, &records, expiredElements))
}
}
m.lockGlob.RUnlock()
// if there are expired elements, get a write lock and delete the expired elements
if len(expiredElements) > 0 {
m.lockGlob.Lock()
for key, element := range expiredElements {
m.evictionList.Remove(element)
_, _ = m.preRadix.Delete(key)
_, _ = m.sufRadix.Delete(reverseString(key))
}
m.lockGlob.Unlock()
}
return records, nil
}
// Close closes the store
func (m *MemStore) Close() error {
return nil
}
// String returns the name of the store implementation
func (m *MemStore) String() string {
return "RadixMemStore"
}
// Len returns the number of items in the store
func (m *MemStore) Len() (int, bool) {
eLen := m.evictionList.Len()
pLen := m.preRadix.Len()
sLen := m.sufRadix.Len()
if eLen == pLen && pLen == sLen {
return eLen, true
}
return 0, false
}
func (m *MemStore) radixTreeCallBack(offset, limit uint, result *[]*store.Record, expiredElements map[string]*list.Element) radix.WalkFn {
currentIndex := new(uint) // needs to be a pointer so the value persist across callback calls
maxIndex := new(uint) // needs to be a pointer so the value persist across callback calls
*maxIndex = offset + limit
return func(key string, value interface{}) bool {
element := value.(*list.Element)
record := element.Value.(*storeRecord)
if record.Expiry != 0 && record.ExpiresAt.Before(time.Now()) {
// record has expired -> add element to the expiredElements map
// and jump directly to the next element without increasing the index
expiredElements[record.Key] = element
return false
}
if *currentIndex >= offset && (*currentIndex < *maxIndex || *maxIndex == offset) {
// if it's within expected range, add a copy to the results
*result = append(*result, fromStoreRecord(record))
}
*currentIndex++
if *currentIndex < *maxIndex || *maxIndex == offset {
return false
}
return true
}
}
func (m *MemStore) radixTreeCallBackCheckSuffix(offset, limit uint, presuf string, result *[]*store.Record, expiredElements map[string]*list.Element) radix.WalkFn {
currentIndex := new(uint) // needs to be a pointer so the value persist across callback calls
maxIndex := new(uint) // needs to be a pointer so the value persist across callback calls
*maxIndex = offset + limit
return func(key string, value interface{}) bool {
if !strings.HasSuffix(key, presuf) {
return false
}
element := value.(*list.Element)
record := element.Value.(*storeRecord)
if record.Expiry != 0 && record.ExpiresAt.Before(time.Now()) {
// record has expired -> add element to the expiredElements map
// and jump directly to the next element without increasing the index
expiredElements[record.Key] = element
return false
}
if *currentIndex >= offset && (*currentIndex < *maxIndex || *maxIndex == offset) {
*result = append(*result, fromStoreRecord(record))
}
*currentIndex++
if *currentIndex < *maxIndex || *maxIndex == offset {
return false
}
return true
}
}
func (m *MemStore) radixTreeCallBackKeysOnly(offset, limit uint, result *[]string, expiredElements map[string]*list.Element) radix.WalkFn {
currentIndex := new(uint) // needs to be a pointer so the value persist across callback calls
maxIndex := new(uint) // needs to be a pointer so the value persist across callback calls
*maxIndex = offset + limit
return func(key string, value interface{}) bool {
element := value.(*list.Element)
record := element.Value.(*storeRecord)
if record.Expiry != 0 && record.ExpiresAt.Before(time.Now()) {
// record has expired -> add element to the expiredElements map
// and jump directly to the next element without increasing the index
expiredElements[record.Key] = element
return false
}
if *currentIndex >= offset && (*currentIndex < *maxIndex || *maxIndex == offset) {
*result = append(*result, record.Key)
}
*currentIndex++
if *currentIndex < *maxIndex || *maxIndex == offset {
return false
}
return true
}
}
func (m *MemStore) radixTreeCallBackKeysOnlyWithSuffix(offset, limit uint, presuf string, result *[]string, expiredElements map[string]*list.Element) radix.WalkFn {
currentIndex := new(uint) // needs to be a pointer so the value persist across callback calls
maxIndex := new(uint) // needs to be a pointer so the value persist across callback calls
*maxIndex = offset + limit
return func(key string, value interface{}) bool {
if !strings.HasSuffix(key, presuf) {
return false
}
element := value.(*list.Element)
record := element.Value.(*storeRecord)
if record.Expiry != 0 && record.ExpiresAt.Before(time.Now()) {
// record has expired -> add element to the expiredElements map
// and jump directly to the next element without increasing the index
expiredElements[record.Key] = element
return false
}
if *currentIndex >= offset && (*currentIndex < *maxIndex || *maxIndex == offset) {
*result = append(*result, record.Key)
}
*currentIndex++
if *currentIndex < *maxIndex || *maxIndex == offset {
return false
}
return true
}
}
+160
View File
@@ -0,0 +1,160 @@
package memory
import (
"sync"
"go-micro.dev/v4/store"
)
// MultiMemStore is a in-memory store implementation using multiple MemStore
// to provide support for multiple databases and tables.
// Each table will be mapped to its own MemStore, which will be completely
// isolated from the rest. In particular, each MemStore will have its own
// capacity, so it's possible to have 10 MemStores with full capacity (512
// by default)
//
// The options will be the same for all MemStores unless they're explicitly
// initialized otherwise.
//
// Since each MemStore is isolated, the required synchronization caused by
// concurrency will be minimal if the threads use different tables
type MultiMemStore struct {
storeMap map[string]*MemStore
storeMapLock sync.RWMutex
genOpts []store.Option
}
// NewMultiMemStore creates a new MultiMemStore. A new MemStore will be mapped based on the options.
// A default MemStore will be mapped if no Database and Table aren't used.
func NewMultiMemStore(opts ...store.Option) store.Store {
m := &MultiMemStore{
storeMap: make(map[string]*MemStore),
genOpts: opts,
}
_ = m.Init(opts...)
return m
}
func (m *MultiMemStore) getMemStore(prefix string) *MemStore {
m.storeMapLock.RLock()
mStore, exists := m.storeMap[prefix]
if exists {
m.storeMapLock.RUnlock()
return mStore
}
m.storeMapLock.RUnlock()
// if not exists
newStore := NewMemStore(m.genOpts...).(*MemStore)
m.storeMapLock.Lock()
m.storeMap[prefix] = newStore
m.storeMapLock.Unlock()
return newStore
}
// Init initializes the mapped MemStore based on the Database and Table values
// from the options with the same options. The target MemStore will be
// reinitialized if needed.
func (m *MultiMemStore) Init(opts ...store.Option) error {
optList := store.Options{}
for _, opt := range opts {
opt(&optList)
}
prefix := optList.Database + "/" + optList.Table
mStore := m.getMemStore(prefix)
return mStore.Init(opts...)
}
// Options returns the options used to create the MultiMemStore.
// Specific options for each MemStore aren't available
func (m *MultiMemStore) Options() store.Options {
optList := store.Options{}
for _, opt := range m.genOpts {
opt(&optList)
}
return optList
}
// Write the record in the target MemStore based on the Database and Table
// values from the options. A default MemStore will be used if no Database
// and Table options are provided.
// The write options will be forwarded to the target MemStore
func (m *MultiMemStore) Write(r *store.Record, opts ...store.WriteOption) error {
wopts := store.WriteOptions{}
for _, opt := range opts {
opt(&wopts)
}
prefix := wopts.Database + "/" + wopts.Table
mStore := m.getMemStore(prefix)
return mStore.Write(r, opts...)
}
// Read the matching records in the target MemStore based on the Database and Table
// values from the options. A default MemStore will be used if no Database
// and Table options are provided.
// The read options will be forwarded to the target MemStore.
//
// The expectations regarding the results (sort order, eviction policies, etc)
// will be the same as the target MemStore
func (m *MultiMemStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
ropts := store.ReadOptions{}
for _, opt := range opts {
opt(&ropts)
}
prefix := ropts.Database + "/" + ropts.Table
mStore := m.getMemStore(prefix)
return mStore.Read(key, opts...)
}
// Delete the matching records in the target MemStore based on the Database and Table
// values from the options. A default MemStore will be used if no Database
// and Table options are provided.
//
// Matching records from other Tables won't be affected. In fact, we won't
// access to other Tables
func (m *MultiMemStore) Delete(key string, opts ...store.DeleteOption) error {
dopts := store.DeleteOptions{}
for _, opt := range opts {
opt(&dopts)
}
prefix := dopts.Database + "/" + dopts.Table
mStore := m.getMemStore(prefix)
return mStore.Delete(key, opts...)
}
// List the keys in the target MemStore based on the Database and Table
// values from the options. A default MemStore will be used if no Database
// and Table options are provided.
// The list options will be forwarded to the target MemStore.
func (m *MultiMemStore) List(opts ...store.ListOption) ([]string, error) {
lopts := store.ListOptions{}
for _, opt := range opts {
opt(&lopts)
}
prefix := lopts.Database + "/" + lopts.Table
mStore := m.getMemStore(prefix)
return mStore.List(opts...)
}
// Close closes the store
func (m *MultiMemStore) Close() error {
return nil
}
// String returns the name of the store implementation
func (m *MultiMemStore) String() string {
return "MultiRadixMemStore"
}
+63
View File
@@ -0,0 +1,63 @@
package memory
import (
"time"
"go-micro.dev/v4/store"
)
func toStoreRecord(src *store.Record, options store.WriteOptions) *storeRecord {
newRecord := &storeRecord{}
newRecord.Key = src.Key
newRecord.Value = make([]byte, len(src.Value))
copy(newRecord.Value, src.Value)
// set base ttl duration and expiration time based on the record
newRecord.Expiry = src.Expiry
if src.Expiry != 0 {
newRecord.ExpiresAt = time.Now().Add(src.Expiry)
}
// overwrite ttl duration and expiration time based on options
if !options.Expiry.IsZero() {
// options.Expiry is a time.Time, newRecord.Expiry is a time.Duration
newRecord.Expiry = time.Until(options.Expiry)
newRecord.ExpiresAt = options.Expiry
}
// TTL option takes precedence over expiration time
if options.TTL != 0 {
newRecord.Expiry = options.TTL
newRecord.ExpiresAt = time.Now().Add(options.TTL)
}
newRecord.Metadata = make(map[string]interface{})
for k, v := range src.Metadata {
newRecord.Metadata[k] = v
}
return newRecord
}
func fromStoreRecord(src *storeRecord) *store.Record {
newRecord := &store.Record{}
newRecord.Key = src.Key
newRecord.Value = make([]byte, len(src.Value))
copy(newRecord.Value, src.Value)
if src.Expiry != 0 {
newRecord.Expiry = time.Until(src.ExpiresAt)
}
newRecord.Metadata = make(map[string]interface{})
for k, v := range src.Metadata {
newRecord.Metadata[k] = v
}
return newRecord
}
func reverseString(s string) string {
r := []rune(s)
for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
r[i], r[j] = r[j], r[i]
}
return string(r)
}
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2018-2023 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package store
import (
"context"
"time"
"go-micro.dev/v4/store"
)
type typeContextKey struct{}
// Store determines the implementation:
// - "memory", for a in-memory implementation, which is also the default if noone matches
// - "noop", for a noop store (it does nothing)
// - "etcd", for etcd
// - "nats-js" for nats-js, needs to have TTL configured at creation
// - "redis", for redis
// - "redis-sentinel", for redis-sentinel
// - "ocmem", custom in-memory implementation, with fixed size and optimized prefix
// and suffix search
func Store(val string) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, typeContextKey{}, val)
}
}
type sizeContextKey struct{}
// Size configures the maximum capacity of the cache for the "ocmem" implementation,
// in number of items that the cache can hold per table.
// You can use 5000 to make the cache hold up to 5000 elements.
// The parameter only affects to the "ocmem" implementation, the rest will ignore it.
// If an invalid value is used, the default of 512 will be used instead.
func Size(val int) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, sizeContextKey{}, val)
}
}
type ttlContextKey struct{}
// TTL is the time to live for documents stored in the store
func TTL(val time.Duration) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, ttlContextKey{}, val)
}
}
+137
View File
@@ -0,0 +1,137 @@
// Copyright 2018-2023 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package store
import (
"context"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/store/etcd"
"github.com/cs3org/reva/v2/pkg/store/memory"
natsjs "github.com/go-micro/plugins/v4/store/nats-js"
"github.com/go-micro/plugins/v4/store/redis"
redisopts "github.com/go-redis/redis/v8"
"github.com/nats-io/nats.go"
"go-micro.dev/v4/logger"
microstore "go-micro.dev/v4/store"
)
var ocMemStore *microstore.Store
const (
// TypeMemory represents memory stores
TypeMemory = "memory"
// TypeNoop represents noop stores
TypeNoop = "noop"
// TypeEtcd represents etcd stores
TypeEtcd = "etcd"
// TypeRedis represents redis stores
TypeRedis = "redis"
// TypeRedisSentinel represents redis-sentinel stores
TypeRedisSentinel = "redis-sentinel"
// TypeOCMem represents ocmem stores
TypeOCMem = "ocmem"
// TypeNatsJS represents nats-js stores
TypeNatsJS = "nats-js"
)
// Create initializes a new store
func Create(opts ...microstore.Option) microstore.Store {
options := &microstore.Options{
Context: context.Background(),
}
for _, o := range opts {
o(options)
}
storeType, _ := options.Context.Value(typeContextKey{}).(string)
switch storeType {
case TypeNoop:
return microstore.NewNoopStore(opts...)
case TypeEtcd:
return etcd.NewStore(opts...)
case TypeRedis:
// FIXME redis plugin does not support redis cluster or ring -> needs upstream patch or our implementation
return redis.NewStore(opts...)
case TypeRedisSentinel:
redisMaster := ""
redisNodes := []string{}
for _, node := range options.Nodes {
parts := strings.SplitN(node, "/", 2)
if len(parts) != 2 {
return nil
}
// the first node is used to retrieve the redis master
redisNodes = append(redisNodes, parts[0])
if redisMaster == "" {
redisMaster = parts[1]
}
}
return redis.NewStore(
microstore.Database(options.Database),
microstore.Table(options.Table),
microstore.Nodes(redisNodes...),
redis.WithRedisOptions(redisopts.UniversalOptions{
MasterName: redisMaster,
}),
)
case TypeOCMem:
if ocMemStore == nil {
var memStore microstore.Store
sizeNum, _ := options.Context.Value(sizeContextKey{}).(int)
if sizeNum <= 0 {
memStore = memory.NewMultiMemStore()
} else {
memStore = memory.NewMultiMemStore(
microstore.WithContext(
memory.NewContext(
context.Background(),
map[string]interface{}{
"maxCap": sizeNum,
},
)),
)
}
ocMemStore = &memStore
}
return *ocMemStore
case TypeNatsJS:
ttl, _ := options.Context.Value(ttlContextKey{}).(time.Duration)
// TODO nats needs a DefaultTTL option as it does not support per Write TTL ...
// FIXME nats has restrictions on the key, we cannot use slashes AFAICT
// host, port, clusterid
return natsjs.NewStore(
append(opts,
natsjs.NatsOptions(nats.Options{Name: "TODO"}),
natsjs.DefaultTTL(ttl))...,
) // TODO test with ocis nats
case TypeMemory, "mem", "": // allow existing short form and use as default
return microstore.NewMemoryStore(opts...)
default:
// try to log an error
if options.Logger == nil {
options.Logger = logger.DefaultLogger
}
options.Logger.Logf(logger.ErrorLevel, "unknown store type: '%s', falling back to memory", storeType)
return microstore.NewMemoryStore(opts...)
}
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"go-micro.dev/v4/store"
)
// setStoreOption returns a function to setup a context with given value.
// setStoreOption returns a function to setup a context with given value
func setStoreOption(k, v interface{}) store.Option {
return func(o *store.Options) {
if o.Context == nil {
+3 -5
View File
@@ -41,7 +41,7 @@ func init() {
cmd.DefaultStores["natsjs"] = NewStore
}
// NewStore will create a new NATS JetStream Object Store.
// NewStore will create a new NATS JetStream Object Store
func NewStore(opts ...store.Option) store.Store {
options := store.Options{
Nodes: []string{},
@@ -64,9 +64,7 @@ func NewStore(opts ...store.Option) store.Store {
return n
}
// Init initializes the store. It must perform any required setup on the
// backing storage implementation and check that it is ready for use,
// returning any errors.
// Init initialises the store. It must perform any required setup on the backing storage implementation and check that it is ready for use, returning any errors.
func (n *natsStore) Init(opts ...store.Option) error {
n.setOption(opts...)
@@ -401,7 +399,7 @@ func (n *natsStore) List(opts ...store.ListOption) ([]string, error) {
return keys, nil
}
// Close the store.
// Close the store
func (n *natsStore) Close() error {
n.conn.Close()
return nil
+11 -19
View File
@@ -7,7 +7,7 @@ import (
"go-micro.dev/v4/store"
)
// store.Option.
// store.Option
type natsOptionsKey struct{}
type jsOptionsKey struct{}
type objOptionsKey struct{}
@@ -15,15 +15,15 @@ type ttlOptionsKey struct{}
type memoryOptionsKey struct{}
type descriptionOptionsKey struct{}
// store.DeleteOption.
// store.DeleteOption
type delBucketOptionsKey struct{}
// NatsOptions accepts nats.Options.
// NatsOptions accepts nats.Options
func NatsOptions(opts nats.Options) store.Option {
return setStoreOption(natsOptionsKey{}, opts)
}
// JetStreamOptions accepts multiple nats.JSOpt.
// JetStreamOptions accepts multiple nats.JSOpt
func JetStreamOptions(opts ...nats.JSOpt) store.Option {
return setStoreOption(jsOptionsKey{}, opts)
}
@@ -35,42 +35,34 @@ func ObjectStoreOptions(cfg ...*nats.ObjectStoreConfig) store.Option {
}
// DefaultTTL sets the default TTL to use for new buckets
//
// By default no TTL is set.
// By default no TTL is set.
//
// TTL ON INDIVIDUAL WRITE CALLS IS NOT SUPPORTED, only bucket wide TTL.
// Either set a default TTL with this option or provide bucket specific options
//
// with ObjectStoreOptions
// with ObjectStoreOptions
func DefaultTTL(ttl time.Duration) store.Option {
return setStoreOption(ttlOptionsKey{}, ttl)
}
// DefaultMemory sets the default storage type to memory only.
//
// The default is file storage, persisting storage between service restarts.
//
// The default is file storage, persisting storage between service restarts.
// Be aware that the default storage location of NATS the /tmp dir is, and thus
//
// won't persist reboots.
// won't persist reboots.
func DefaultMemory() store.Option {
return setStoreOption(memoryOptionsKey{}, nats.MemoryStorage)
}
// DefaultDescription sets the default description to use when creating new
//
// buckets. The default is "Store managed by go-micro"
// buckets. The default is "Store managed by go-micro"
func DefaultDescription(text string) store.Option {
return setStoreOption(descriptionOptionsKey{}, text)
}
// DeleteBucket will use the key passed to Delete as a bucket (database) name,
//
// and delete the bucket.
//
// and delete the bucket.
// This option should not be combined with the store.DeleteFrom option, as
//
// that will overwrite the delete action.
// that will overwrite the delete action.
func DeleteBucket() store.DeleteOption {
return func(d *store.DeleteOptions) {
d.Table = "DELETE_BUCKET"
+6 -6
View File
@@ -349,7 +349,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
# github.com/cs3org/reva/v2 v2.12.1-0.20230420073005-11edad1f09fe
# github.com/cs3org/reva/v2 v2.12.1-0.20230424091007-8d8b567179b1
## explicit; go 1.19
github.com/cs3org/reva/v2/cmd/revad/internal/grace
github.com/cs3org/reva/v2/cmd/revad/runtime
@@ -576,16 +576,13 @@ github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/registry
github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple
github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/spaces
github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/tus
github.com/cs3org/reva/v2/pkg/rhttp/datatx/metrics
github.com/cs3org/reva/v2/pkg/rhttp/datatx/utils/download
github.com/cs3org/reva/v2/pkg/rhttp/global
github.com/cs3org/reva/v2/pkg/rhttp/router
github.com/cs3org/reva/v2/pkg/sdk/common
github.com/cs3org/reva/v2/pkg/share
github.com/cs3org/reva/v2/pkg/share/cache
github.com/cs3org/reva/v2/pkg/share/cache/loader
github.com/cs3org/reva/v2/pkg/share/cache/memory
github.com/cs3org/reva/v2/pkg/share/cache/redis
github.com/cs3org/reva/v2/pkg/share/cache/registry
github.com/cs3org/reva/v2/pkg/share/cache/warmup/cbox
github.com/cs3org/reva/v2/pkg/share/cache/warmup/loader
github.com/cs3org/reva/v2/pkg/share/cache/warmup/registry
@@ -674,6 +671,9 @@ github.com/cs3org/reva/v2/pkg/storage/utils/sync
github.com/cs3org/reva/v2/pkg/storage/utils/templates
github.com/cs3org/reva/v2/pkg/storage/utils/walker
github.com/cs3org/reva/v2/pkg/storagespace
github.com/cs3org/reva/v2/pkg/store
github.com/cs3org/reva/v2/pkg/store/etcd
github.com/cs3org/reva/v2/pkg/store/memory
github.com/cs3org/reva/v2/pkg/sysinfo
github.com/cs3org/reva/v2/pkg/tags
github.com/cs3org/reva/v2/pkg/token
@@ -906,7 +906,7 @@ github.com/go-micro/plugins/v4/server/grpc
# github.com/go-micro/plugins/v4/server/http v1.2.1
## explicit; go 1.17
github.com/go-micro/plugins/v4/server/http
# github.com/go-micro/plugins/v4/store/nats-js v1.2.0
# github.com/go-micro/plugins/v4/store/nats-js v1.1.0
## explicit; go 1.17
github.com/go-micro/plugins/v4/store/nats-js
# github.com/go-micro/plugins/v4/store/redis v1.2.0