From 1a01d21f6071b0c41669f075ca139d2cd851709d Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 27 Oct 2021 07:55:41 +0200 Subject: [PATCH 1/4] prepare oCIS 1.14.0 changelog --- CHANGELOG.md | 6 +++--- .../add-user-setting-capability.md | 0 changelog/{unreleased => 1.14.0_2021-10-27}/buf-tooling.md | 0 .../{unreleased => 1.14.0_2021-10-27}/create-space-quota.md | 0 changelog/{unreleased => 1.14.0_2021-10-27}/data-path.md | 0 .../dont-expose-services-by-default.md | 0 .../fix-spaces-ocis-url.md | 0 .../go-embed-instead-fileb0x.md | 0 .../{unreleased => 1.14.0_2021-10-27}/go-micro-v4-1-0.md | 0 changelog/{unreleased => 1.14.0_2021-10-27}/http-header.md | 0 .../{unreleased => 1.14.0_2021-10-27}/idp-parent-dir.md | 0 .../lower-tus-chunk-size.md | 0 .../ocs-additional-info.md | 0 .../public-share-auth-middleware.md | 0 .../report-quota-states.md | 0 .../reva-tokens-skip-groups-config.md | 0 .../update-enforce-permission.md | 0 changelog/{unreleased => 1.14.0_2021-10-27}/update-lico.md | 0 changelog/{unreleased => 1.14.0_2021-10-27}/update-reva.md | 0 .../{unreleased => 1.14.0_2021-10-27}/update-web-4.4.0.md | 0 20 files changed, 3 insertions(+), 3 deletions(-) rename changelog/{unreleased => 1.14.0_2021-10-27}/add-user-setting-capability.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/buf-tooling.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/create-space-quota.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/data-path.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/dont-expose-services-by-default.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/fix-spaces-ocis-url.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/go-embed-instead-fileb0x.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/go-micro-v4-1-0.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/http-header.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/idp-parent-dir.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/lower-tus-chunk-size.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/ocs-additional-info.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/public-share-auth-middleware.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/report-quota-states.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/reva-tokens-skip-groups-config.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/update-enforce-permission.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/update-lico.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/update-reva.md (100%) rename changelog/{unreleased => 1.14.0_2021-10-27}/update-web-4.4.0.md (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 686a5eb0a..b30092a4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ -# Changelog for [unreleased] (UNRELEASED) +# Changelog for [1.14.0] (2021-10-27) -The following sections list the changes for unreleased. +The following sections list the changes for 1.14.0. -[unreleased]: https://github.com/owncloud/ocis/compare/v1.13.0...master +[1.14.0]: https://github.com/owncloud/ocis/compare/v1.13.0...v1.14.0 ## Summary diff --git a/changelog/unreleased/add-user-setting-capability.md b/changelog/1.14.0_2021-10-27/add-user-setting-capability.md similarity index 100% rename from changelog/unreleased/add-user-setting-capability.md rename to changelog/1.14.0_2021-10-27/add-user-setting-capability.md diff --git a/changelog/unreleased/buf-tooling.md b/changelog/1.14.0_2021-10-27/buf-tooling.md similarity index 100% rename from changelog/unreleased/buf-tooling.md rename to changelog/1.14.0_2021-10-27/buf-tooling.md diff --git a/changelog/unreleased/create-space-quota.md b/changelog/1.14.0_2021-10-27/create-space-quota.md similarity index 100% rename from changelog/unreleased/create-space-quota.md rename to changelog/1.14.0_2021-10-27/create-space-quota.md diff --git a/changelog/unreleased/data-path.md b/changelog/1.14.0_2021-10-27/data-path.md similarity index 100% rename from changelog/unreleased/data-path.md rename to changelog/1.14.0_2021-10-27/data-path.md diff --git a/changelog/unreleased/dont-expose-services-by-default.md b/changelog/1.14.0_2021-10-27/dont-expose-services-by-default.md similarity index 100% rename from changelog/unreleased/dont-expose-services-by-default.md rename to changelog/1.14.0_2021-10-27/dont-expose-services-by-default.md diff --git a/changelog/unreleased/fix-spaces-ocis-url.md b/changelog/1.14.0_2021-10-27/fix-spaces-ocis-url.md similarity index 100% rename from changelog/unreleased/fix-spaces-ocis-url.md rename to changelog/1.14.0_2021-10-27/fix-spaces-ocis-url.md diff --git a/changelog/unreleased/go-embed-instead-fileb0x.md b/changelog/1.14.0_2021-10-27/go-embed-instead-fileb0x.md similarity index 100% rename from changelog/unreleased/go-embed-instead-fileb0x.md rename to changelog/1.14.0_2021-10-27/go-embed-instead-fileb0x.md diff --git a/changelog/unreleased/go-micro-v4-1-0.md b/changelog/1.14.0_2021-10-27/go-micro-v4-1-0.md similarity index 100% rename from changelog/unreleased/go-micro-v4-1-0.md rename to changelog/1.14.0_2021-10-27/go-micro-v4-1-0.md diff --git a/changelog/unreleased/http-header.md b/changelog/1.14.0_2021-10-27/http-header.md similarity index 100% rename from changelog/unreleased/http-header.md rename to changelog/1.14.0_2021-10-27/http-header.md diff --git a/changelog/unreleased/idp-parent-dir.md b/changelog/1.14.0_2021-10-27/idp-parent-dir.md similarity index 100% rename from changelog/unreleased/idp-parent-dir.md rename to changelog/1.14.0_2021-10-27/idp-parent-dir.md diff --git a/changelog/unreleased/lower-tus-chunk-size.md b/changelog/1.14.0_2021-10-27/lower-tus-chunk-size.md similarity index 100% rename from changelog/unreleased/lower-tus-chunk-size.md rename to changelog/1.14.0_2021-10-27/lower-tus-chunk-size.md diff --git a/changelog/unreleased/ocs-additional-info.md b/changelog/1.14.0_2021-10-27/ocs-additional-info.md similarity index 100% rename from changelog/unreleased/ocs-additional-info.md rename to changelog/1.14.0_2021-10-27/ocs-additional-info.md diff --git a/changelog/unreleased/public-share-auth-middleware.md b/changelog/1.14.0_2021-10-27/public-share-auth-middleware.md similarity index 100% rename from changelog/unreleased/public-share-auth-middleware.md rename to changelog/1.14.0_2021-10-27/public-share-auth-middleware.md diff --git a/changelog/unreleased/report-quota-states.md b/changelog/1.14.0_2021-10-27/report-quota-states.md similarity index 100% rename from changelog/unreleased/report-quota-states.md rename to changelog/1.14.0_2021-10-27/report-quota-states.md diff --git a/changelog/unreleased/reva-tokens-skip-groups-config.md b/changelog/1.14.0_2021-10-27/reva-tokens-skip-groups-config.md similarity index 100% rename from changelog/unreleased/reva-tokens-skip-groups-config.md rename to changelog/1.14.0_2021-10-27/reva-tokens-skip-groups-config.md diff --git a/changelog/unreleased/update-enforce-permission.md b/changelog/1.14.0_2021-10-27/update-enforce-permission.md similarity index 100% rename from changelog/unreleased/update-enforce-permission.md rename to changelog/1.14.0_2021-10-27/update-enforce-permission.md diff --git a/changelog/unreleased/update-lico.md b/changelog/1.14.0_2021-10-27/update-lico.md similarity index 100% rename from changelog/unreleased/update-lico.md rename to changelog/1.14.0_2021-10-27/update-lico.md diff --git a/changelog/unreleased/update-reva.md b/changelog/1.14.0_2021-10-27/update-reva.md similarity index 100% rename from changelog/unreleased/update-reva.md rename to changelog/1.14.0_2021-10-27/update-reva.md diff --git a/changelog/unreleased/update-web-4.4.0.md b/changelog/1.14.0_2021-10-27/update-web-4.4.0.md similarity index 100% rename from changelog/unreleased/update-web-4.4.0.md rename to changelog/1.14.0_2021-10-27/update-web-4.4.0.md From 405809f13a7fac410a6f9fe59731015c26f674bd Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 27 Oct 2021 12:27:25 +0200 Subject: [PATCH 2/4] fix accounts backend regression from #2590 --- accounts/pkg/flagset/flagset.go | 2 +- accounts/pkg/service/v0/service.go | 95 ++++++++++++++++-------------- ocis-pkg/indexer/config/config.go | 5 +- ocis-pkg/indexer/indexer.go | 16 ++--- 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index fc29ca6d1..abc32ec57 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -184,7 +184,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "storage-backend", - Value: flags.OverrideDefaultString(cfg.Repo.Disk.Path, "CS3"), + Value: flags.OverrideDefaultString(cfg.Repo.Backend, "CS3"), Usage: "Which backend to use to store accounts data (CS3 or disk)", EnvVars: []string{"ACCOUNTS_STORAGE_BACKEND"}, Destination: &cfg.Repo.Backend, diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index ffedcb5f6..25f1be757 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -2,13 +2,14 @@ package service import ( "context" - "errors" "path" "path/filepath" "strconv" "strings" "time" + "github.com/pkg/errors" + "github.com/owncloud/ocis/ocis-pkg/service/grpc" "github.com/owncloud/ocis/accounts/pkg/storage" @@ -48,17 +49,22 @@ func New(opts ...Option) (s *Service, err error) { roleManager = &m } + storage, err := createMetadataStorage(cfg, logger) + if err != nil { + return nil, errors.Wrap(err, "could not create metadata storage") + } + s = &Service{ id: cfg.GRPC.Namespace + "." + cfg.Server.Name, log: logger, Config: cfg, RoleService: roleService, RoleManager: roleManager, - repo: createMetadataStorage(cfg, logger), + repo: storage, } r := oreg.GetRegistry() - if strings.ToLower(cfg.Repo.Backend) != "disk" { + if strings.ToLower(cfg.Repo.Backend) == "cs3" { if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil { logger.Error().Err(err).Msg("index: storage-metadata service not present") return nil, err @@ -113,40 +119,41 @@ func configFromSvc(cfg *config.Config) (*idxcfg.Config, error) { } }(cfg) - if (config.Repo{}) != cfg.Repo { - if (config.Disk{}) != cfg.Repo.Disk { - c.Repo = idxcfg.Repo{ - Disk: idxcfg.Disk{ - Path: cfg.Repo.Disk.Path, - }, - } + switch backend := strings.ToLower(cfg.Repo.Backend); backend { + case "disk": + c.Repo = idxcfg.Repo{ + Backend: backend, + Disk: idxcfg.Disk{ + Path: cfg.Repo.Disk.Path, + }, } + case "cs3": + c.Repo = idxcfg.Repo{ + Backend: backend, + CS3: idxcfg.CS3{ + ProviderAddr: cfg.Repo.CS3.ProviderAddr, + DataURL: cfg.Repo.CS3.DataURL, + DataPrefix: cfg.Repo.CS3.DataPrefix, + JWTSecret: cfg.Repo.CS3.JWTSecret, + }, + } + default: + return nil, errors.New("index backend " + cfg.Repo.Backend + " is not supported") + } - if (config.CS3{}) != cfg.Repo.CS3 { - c.Repo = idxcfg.Repo{ - CS3: idxcfg.CS3{ - ProviderAddr: cfg.Repo.CS3.ProviderAddr, - DataURL: cfg.Repo.CS3.DataURL, - DataPrefix: cfg.Repo.CS3.DataPrefix, - JWTSecret: cfg.Repo.CS3.JWTSecret, - }, - } + if (config.Index{}) != cfg.Index { + c.Index = idxcfg.Index{ + UID: idxcfg.Bound{ + Lower: cfg.Index.UID.Lower, + }, + GID: idxcfg.Bound{ + Lower: cfg.Index.GID.Lower, + }, } + } - if (config.Index{}) != cfg.Index { - c.Index = idxcfg.Index{ - UID: idxcfg.Bound{ - Lower: cfg.Index.UID.Lower, - }, - GID: idxcfg.Bound{ - Lower: cfg.Index.GID.Lower, - }, - } - } - - if (config.ServiceUser{}) != cfg.ServiceUser { - c.ServiceUser = cfg.ServiceUser - } + if (config.ServiceUser{}) != cfg.ServiceUser { + c.ServiceUser = cfg.ServiceUser } return c, nil @@ -417,17 +424,19 @@ func (s Service) createDefaultGroups(withDemoGroups bool) (err error) { return nil } -func createMetadataStorage(cfg *config.Config, logger log.Logger) storage.Repo { - // for now we detect the used storage implementation based on which storage is configured - // the config with defaults needs to be checked last - if cfg.Repo.Disk.Path != "" { - return storage.NewDiskRepo(cfg, logger) +func createMetadataStorage(cfg *config.Config, logger log.Logger) (storage.Repo, error) { + switch strings.ToLower(cfg.Repo.Backend) { + case "disk": + return storage.NewDiskRepo(cfg, logger), nil + case "cs3": + repo, err := storage.NewCS3Repo(cfg) + if err != nil { + return nil, errors.Wrap(err, "cs3 backend was configured but failed to start") + } + return repo, nil + default: + return nil, errors.New("backend type " + cfg.Repo.Backend + " is not supported") } - repo, err := storage.NewCS3Repo(cfg) - if err != nil { - logger.Fatal().Err(err).Msg("cs3 storage was configured but failed to start") - } - return repo } // Service implements the AccountsServiceHandler interface diff --git a/ocis-pkg/indexer/config/config.go b/ocis-pkg/indexer/config/config.go index b718f560c..5d2b23f71 100644 --- a/ocis-pkg/indexer/config/config.go +++ b/ocis-pkg/indexer/config/config.go @@ -7,8 +7,9 @@ import ( // Repo defines which storage implementation is to be used. type Repo struct { - Disk Disk - CS3 CS3 + Backend string + Disk Disk + CS3 CS3 } // Disk is the local disk implementation of the storage. diff --git a/ocis-pkg/indexer/indexer.go b/ocis-pkg/indexer/indexer.go index 51a9967d4..151773442 100644 --- a/ocis-pkg/indexer/indexer.go +++ b/ocis-pkg/indexer/indexer.go @@ -3,10 +3,11 @@ package indexer import ( "fmt" - "github.com/owncloud/ocis/ocis-pkg/sync" "path" "strings" + "github.com/owncloud/ocis/ocis-pkg/sync" + "github.com/CiscoM31/godata" "github.com/iancoleman/strcase" "github.com/owncloud/ocis/ocis-pkg/indexer/config" @@ -39,14 +40,6 @@ func CreateIndexer(cfg *config.Config) *Indexer { } } -func getRegistryStrategy(cfg *config.Config) string { - if cfg.Repo.Disk.Path != "" { - return "disk" - } - - return "cs3" -} - // Reset takes care of deleting all indices from storage and from the internal map of indices func (i *Indexer) Reset() error { for j := range i.indices { @@ -66,11 +59,10 @@ func (i *Indexer) Reset() error { // AddIndex adds a new index to the indexer receiver. func (i *Indexer) AddIndex(t interface{}, indexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error { - strategy := getRegistryStrategy(i.config) - f := registry.IndexConstructorRegistry[strategy][indexType] + f := registry.IndexConstructorRegistry[i.config.Repo.Backend][indexType] var idx index.Index - if strategy == "cs3" { + if i.config.Repo.Backend == "cs3" { idx = f( option.CaseInsensitive(caseInsensitive), option.WithEntity(t), From ef406c8baa7e64c6a204fbe9e8181088c17f4c32 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 27 Oct 2021 12:53:25 +0200 Subject: [PATCH 3/4] ensure lowercase backend in the command --- accounts/pkg/command/server.go | 2 ++ accounts/pkg/service/v0/service.go | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/accounts/pkg/command/server.go b/accounts/pkg/command/server.go index 0378c39cd..6dbabfaff 100644 --- a/accounts/pkg/command/server.go +++ b/accounts/pkg/command/server.go @@ -30,6 +30,8 @@ func Server(cfg *config.Config) *cli.Command { cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") } + cfg.Repo.Backend = strings.ToLower(cfg.Repo.Backend) + // When running on single binary mode the before hook from the root command won't get called. We manually // call this before hook from ocis command, so the configuration can be loaded. if !cfg.Supervised { diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index 25f1be757..b95455204 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -64,7 +64,7 @@ func New(opts ...Option) (s *Service, err error) { } r := oreg.GetRegistry() - if strings.ToLower(cfg.Repo.Backend) == "cs3" { + if cfg.Repo.Backend == "cs3" { if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil { logger.Error().Err(err).Msg("index: storage-metadata service not present") return nil, err @@ -119,17 +119,17 @@ func configFromSvc(cfg *config.Config) (*idxcfg.Config, error) { } }(cfg) - switch backend := strings.ToLower(cfg.Repo.Backend); backend { + switch cfg.Repo.Backend { case "disk": c.Repo = idxcfg.Repo{ - Backend: backend, + Backend: cfg.Repo.Backend, Disk: idxcfg.Disk{ Path: cfg.Repo.Disk.Path, }, } case "cs3": c.Repo = idxcfg.Repo{ - Backend: backend, + Backend: cfg.Repo.Backend, CS3: idxcfg.CS3{ ProviderAddr: cfg.Repo.CS3.ProviderAddr, DataURL: cfg.Repo.CS3.DataURL, @@ -425,7 +425,7 @@ func (s Service) createDefaultGroups(withDemoGroups bool) (err error) { } func createMetadataStorage(cfg *config.Config, logger log.Logger) (storage.Repo, error) { - switch strings.ToLower(cfg.Repo.Backend) { + switch cfg.Repo.Backend { case "disk": return storage.NewDiskRepo(cfg, logger), nil case "cs3": From f45254bb5dd413b11a75087636c0b6c59ad5fec6 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 27 Oct 2021 13:56:22 +0200 Subject: [PATCH 4/4] set backend to disk in tests --- accounts/pkg/proto/v0/accounts.pb.micro_test.go | 1 + accounts/pkg/service/v0/accounts_permission_test.go | 1 + ocis-pkg/indexer/index/disk/non_unique_test.go | 1 + ocis-pkg/indexer/index/disk/unique_test.go | 1 + ocis-pkg/indexer/indexer_test.go | 1 + ocs/pkg/server/http/svc_test.go | 1 + 6 files changed, 6 insertions(+) diff --git a/accounts/pkg/proto/v0/accounts.pb.micro_test.go b/accounts/pkg/proto/v0/accounts.pb.micro_test.go index fa2ec3758..bcd53c974 100644 --- a/accounts/pkg/proto/v0/accounts.pb.micro_test.go +++ b/accounts/pkg/proto/v0/accounts.pb.micro_test.go @@ -80,6 +80,7 @@ func init() { ) cfg := config.New() + cfg.Repo.Backend = "disk" cfg.Repo.Disk.Path = dataPath cfg.Server.DemoUsersAndGroups = true var hdlr *svc.Service diff --git a/accounts/pkg/service/v0/accounts_permission_test.go b/accounts/pkg/service/v0/accounts_permission_test.go index 6ebe75140..477369b66 100644 --- a/accounts/pkg/service/v0/accounts_permission_test.go +++ b/accounts/pkg/service/v0/accounts_permission_test.go @@ -33,6 +33,7 @@ var ( func init() { cfg := config.New() cfg.Server.Name = "accounts" + cfg.Repo.Backend = "disk" cfg.Repo.Disk.Path = dataPath logger := olog.NewLogger(olog.Color(true), olog.Pretty(true)) roleServiceMock = buildRoleServiceMock() diff --git a/ocis-pkg/indexer/index/disk/non_unique_test.go b/ocis-pkg/indexer/index/disk/non_unique_test.go index b9f592d61..5d0a25ca0 100644 --- a/ocis-pkg/indexer/index/disk/non_unique_test.go +++ b/ocis-pkg/indexer/index/disk/non_unique_test.go @@ -83,6 +83,7 @@ func getNonUniqueIdxSut(t *testing.T, entity interface{}, indexBy string) (index dataPath, _ := WriteIndexTestData(Data, "ID", "") cfg := config.Config{ Repo: config.Repo{ + Backend: "disk", Disk: config.Disk{ Path: dataPath, }, diff --git a/ocis-pkg/indexer/index/disk/unique_test.go b/ocis-pkg/indexer/index/disk/unique_test.go index 2ecbfbe17..fca28895e 100644 --- a/ocis-pkg/indexer/index/disk/unique_test.go +++ b/ocis-pkg/indexer/index/disk/unique_test.go @@ -103,6 +103,7 @@ func getUniqueIdxSut(t *testing.T, indexBy string, entityType interface{}) (inde dataPath, _ := WriteIndexTestData(Data, "ID", "") cfg := config.Config{ Repo: config.Repo{ + Backend: "disk", Disk: config.Disk{ Path: dataPath, }, diff --git a/ocis-pkg/indexer/indexer_test.go b/ocis-pkg/indexer/indexer_test.go index 47440f1a2..baf27a511 100644 --- a/ocis-pkg/indexer/indexer_test.go +++ b/ocis-pkg/indexer/indexer_test.go @@ -300,6 +300,7 @@ func TestQueryDiskImpl(t *testing.T) { func createDiskIndexer(dataDir string) *Indexer { return CreateIndexer(&config.Config{ Repo: config.Repo{ + Backend: "disk", Disk: config.Disk{ Path: dataDir, }, diff --git a/ocs/pkg/server/http/svc_test.go b/ocs/pkg/server/http/svc_test.go index 28d4f4d1f..ee896414f 100644 --- a/ocs/pkg/server/http/svc_test.go +++ b/ocs/pkg/server/http/svc_test.go @@ -542,6 +542,7 @@ func init() { DemoUsersAndGroups: true, }, Repo: accountsCfg.Repo{ + Backend: "disk", Disk: accountsCfg.Disk{ Path: dataPath, },