sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #118 from opencloud-eu/switch-to-decomposed-decomposed-s3

Browse Source 
Switch from ocis/s3ng to decomposed/decomposed_s3 storage drivers
This commit is contained in:
Jörn Friedrich Dreyer
2025-01-27 16:43:40 +01:00
committed by GitHub
parent 1bd1c38cf7 f00537dfc6
commit 3900a08049
26 changed files with 334 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.star
+2 -2
View File
@@ -1039,7 +1039,7 @@ def localApiTests(ctx, name, suites, storage = "opencloud", extra_environment =
"SEND_SCENARIO_LINE_REFERENCES": "true", "SEND_SCENARIO_LINE_REFERENCES": "true",
"STORAGE_DRIVER": storage, "STORAGE_DRIVER": storage,
"BEHAT_SUITES": ",".join(suites), "BEHAT_SUITES": ",".join(suites),
"BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"), "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnDecomposed-%s-Storage" % ("OC" if storage == "owncloud" else "DECOMPOSED"),
"EXPECTED_FAILURES_FILE": expected_failures_file, "EXPECTED_FAILURES_FILE": expected_failures_file,
"UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
"OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME,
@@ -1212,7 +1212,7 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty =
} }
def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "opencloud", accounts_hash_difficulty = 4): def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "opencloud", accounts_hash_difficulty = 4):
filterTags = "~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS") filterTags = "~@skipOnGraph&&~@skipOnDecomposed-%s-Storage" % ("OC" if storage == "owncloud" else "DECOMPOSED")
test_dir = "%s/tests/acceptance" % dirs["base"] test_dir = "%s/tests/acceptance" % dirs["base"]
expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage.upper()) expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage.upper())
deployments/examples/opencloud_full/decomposed_s3.yml
+14
View File
@@ -0,0 +1,14 @@
---
services:
opencloud:
environment:
# activate decomposed_s3 storage driver
STORAGE_USERS_DRIVER: decomposed_s3
# keep system data on opencloud storage since this are only small files atm
STORAGE_SYSTEM_DRIVER: decomposed
# s3ng specific settings
STORAGE_USERS_DECOMPOSED_S3_ENDPOINT: ${DECOMPOSED_S3_ENDPOINT:-http://minio:9000}
STORAGE_USERS_DECOMPOSED_S3_REGION: ${DECOMPOSED_S3_REGION:-default}
STORAGE_USERS_DECOMPOSED_S3_ACCESS_KEY: ${DECOMPOSED_S3_ACCESS_KEY:-opencloud}
STORAGE_USERS_DECOMPOSED_S3_SECRET_KEY: ${DECOMPOSED_S3_SECRET_KEY:-opencloud-secret-key}
STORAGE_USERS_DECOMPOSED_S3_BUCKET: ${DECOMPOSED_S3_BUCKET:-opencloud-bucket}
deployments/examples/opencloud_full/s3ng.yml
-14
View File
@@ -1,14 +0,0 @@
---
services:
opencloud:
environment:
# activate s3ng storage driver
STORAGE_USERS_DRIVER: s3ng
# keep system data on opencloud storage since this are only small files atm
STORAGE_SYSTEM_DRIVER: ocis
# s3ng specific settings
STORAGE_USERS_S3NG_ENDPOINT: ${S3NG_ENDPOINT:-http://minio:9000}
STORAGE_USERS_S3NG_REGION: ${S3NG_REGION:-default}
STORAGE_USERS_S3NG_ACCESS_KEY: ${S3NG_ACCESS_KEY:-opencloud}
STORAGE_USERS_S3NG_SECRET_KEY: ${S3NG_SECRET_KEY:-opencloud-secret-key}
STORAGE_USERS_S3NG_BUCKET: ${S3NG_BUCKET:-opencloud-bucket}
go.mod
+1 -1
View File
@@ -65,7 +65,7 @@ require (
github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/ginkgo/v2 v2.22.2
github.com/onsi/gomega v1.36.2 github.com/onsi/gomega v1.36.2
github.com/open-policy-agent/opa v0.70.0 github.com/open-policy-agent/opa v0.70.0
github.com/opencloud-eu/reva/v2 v2.27.3-0.20250121094357-24f23b6a27ed github.com/opencloud-eu/reva/v2 v2.27.3-0.20250127153848-a84e6c39c206
github.com/orcaman/concurrent-map v1.0.0 github.com/orcaman/concurrent-map v1.0.0
github.com/owncloud/libre-graph-api-go v1.0.5-0.20240829135935-80dc00d6f5ea github.com/owncloud/libre-graph-api-go v1.0.5-0.20240829135935-80dc00d6f5ea
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
go.sum
+2
View File
@@ -873,6 +873,8 @@ github.com/open-policy-agent/opa v0.70.0 h1:B3cqCN2iQAyKxK6+GI+N40uqkin+wzIrM7YA
github.com/open-policy-agent/opa v0.70.0/go.mod h1:Y/nm5NY0BX0BqjBriKUiV81sCl8XOjjvqQG7dXrggtI= github.com/open-policy-agent/opa v0.70.0/go.mod h1:Y/nm5NY0BX0BqjBriKUiV81sCl8XOjjvqQG7dXrggtI=
github.com/opencloud-eu/reva/v2 v2.27.3-0.20250121094357-24f23b6a27ed h1:0rVMOlcGXgFRNrNgjeCGTyuGKXwD+Y+wQGzm/uVe3CU= github.com/opencloud-eu/reva/v2 v2.27.3-0.20250121094357-24f23b6a27ed h1:0rVMOlcGXgFRNrNgjeCGTyuGKXwD+Y+wQGzm/uVe3CU=
github.com/opencloud-eu/reva/v2 v2.27.3-0.20250121094357-24f23b6a27ed/go.mod h1:lk0GfBt0cLaOcc1nWJikinTK5ibFtKRxp10ATxtCalU= github.com/opencloud-eu/reva/v2 v2.27.3-0.20250121094357-24f23b6a27ed/go.mod h1:lk0GfBt0cLaOcc1nWJikinTK5ibFtKRxp10ATxtCalU=
github.com/opencloud-eu/reva/v2 v2.27.3-0.20250127153848-a84e6c39c206 h1:sTbtA2hU40r6eh24aswG0oP7NiJrVyEiqM1nn72TrHA=
github.com/opencloud-eu/reva/v2 v2.27.3-0.20250127153848-a84e6c39c206/go.mod h1:lk0GfBt0cLaOcc1nWJikinTK5ibFtKRxp10ATxtCalU=
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
opencloud/README.md
+2 -2
View File
@@ -55,7 +55,7 @@ A node that doesn't have any (or malformed) metadata.
This command provides additional options: This command provides additional options:
* `-b` / `--blobstore`\ * `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details). Allows specifying the blobstore to use. Defaults to `decomposed`. Empty blobs will not be checked. Can also be switched to `decomposed_s3`, but needs addtional envvar configuration (see the `storage-users` service for more details).
* `--fail`\ * `--fail`\
Exits with non-zero exit code if inconsistencies are found. Useful for automation. Exits with non-zero exit code if inconsistencies are found. Useful for automation.
@@ -86,7 +86,7 @@ This command provides additional options:
* `--dry-run` (default: `true`)\ * `--dry-run` (default: `true`)\
Do not remove any revisions but print the revisions that would be removed. Do not remove any revisions but print the revisions that would be removed.
* `-b` / `--blobstore`\ * `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details). Allows specifying the blobstore to use. Defaults to `decomposed`. Can be switched to `decomposed_s3` but needs addtional envvar configuration (see the `storage-users` service for more details).
* `-v` / `--verbose`\ * `-v` / `--verbose`\
Prints additional information about the revisions that are removed. Prints additional information about the revisions that are removed.
* `--glob-mechanism` (default: `glob`\ * `--glob-mechanism` (default: `glob`\
opencloud/pkg/backup/provider.go
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/shamaton/msgpack/v2" "github.com/shamaton/msgpack/v2"
) )
@@ -132,7 +132,7 @@ func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
// FIXME: how to check if metadata is complete? // FIXME: how to check if metadata is complete?
if bid := m["user.ocis.blobid"]; string(bid) != "" { if bid := m["user.oc.blobid"]; string(bid) != "" {
spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path)) spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), "" return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), ""
} }
opencloud/pkg/command/backup.go
+11 -11
View File
@@ -9,8 +9,8 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config"
"github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/config/parser" "github.com/opencloud-eu/opencloud/pkg/config/parser"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore" ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore" s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed_s3/blobstore"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@@ -47,8 +47,8 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command {
&cli.StringFlag{ &cli.StringFlag{
Name: "blobstore", Name: "blobstore",
Aliases: []string{"b"}, Aliases: []string{"b"},
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", Usage: "the blobstore type. Can be (none, decomposed, decomposed_s3). Default decomposed",
Value: "ocis", Value: "decomposed",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "fail", Name: "fail",
@@ -67,16 +67,16 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command {
err error err error
) )
switch c.String("blobstore") { switch c.String("blobstore") {
case "s3ng": case "decomposed_s3":
bs, err = s3bs.New( bs, err = s3bs.New(
cfg.StorageUsers.Drivers.S3NG.Endpoint, cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.S3NG.Region, cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.S3NG.Bucket, cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.S3NG.AccessKey, cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.S3NG.SecretKey, cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
s3bs.Options{}, s3bs.Options{},
) )
case "ocis": case "decomposed":
bs, err = ocbs.New(basePath) bs, err = ocbs.New(basePath)
case "none": case "none":
bs = nil bs = nil
opencloud/pkg/command/decomposedfs.go
+6 -6
View File
@@ -15,13 +15,13 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx" revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/storage/cache" "github.com/opencloud-eu/reva/v2/pkg/storage/cache"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree"
"github.com/opencloud-eu/reva/v2/pkg/storagespace" "github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/store" "github.com/opencloud-eu/reva/v2/pkg/store"
"github.com/rs/zerolog" "github.com/rs/zerolog"
opencloud/pkg/command/migrate.go
+3 -3
View File
@@ -23,9 +23,9 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid" "github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid"
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry" "github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/migrator"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog" "github.com/rs/zerolog"
opencloud/pkg/command/revisions.go
+11 -11
View File
@@ -11,9 +11,9 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config"
"github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/config/parser" "github.com/opencloud-eu/opencloud/pkg/config/parser"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore" ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed_s3/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storagespace" "github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@@ -56,8 +56,8 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
&cli.StringFlag{ &cli.StringFlag{
Name: "blobstore", Name: "blobstore",
Aliases: []string{"b"}, Aliases: []string{"b"},
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service", Usage: "the blobstore type. Can be (none, decomposed, decomposed_s3). Default decomposed. Note: When using s3ng this needs same configuration as the storage-users service",
Value: "ocis", Value: "decomposed",
}, },
&cli.BoolFlag{ &cli.BoolFlag{
Name: "dry-run", Name: "dry-run",
@@ -93,16 +93,16 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
err error err error
) )
switch c.String("blobstore") { switch c.String("blobstore") {
case "s3ng": case "decomposed_s3":
bs, err = s3bs.New( bs, err = s3bs.New(
cfg.StorageUsers.Drivers.S3NG.Endpoint, cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.S3NG.Region, cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.S3NG.Bucket, cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.S3NG.AccessKey, cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.S3NG.SecretKey, cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
s3bs.Options{}, s3bs.Options{},
) )
case "ocis": case "decomposed":
bs, err = ocbs.New(basePath) bs, err = ocbs.New(basePath)
case "none": case "none":
bs = nil bs = nil
opencloud/pkg/revisions/revisions.go
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/shamaton/msgpack/v2" "github.com/shamaton/msgpack/v2"
) )
@@ -247,7 +247,7 @@ func getBlobID(path string) (string, error) {
return "", err return "", err
} }
if bid := m["user.ocis.blobid"]; string(bid) != "" { if bid := m["user.oc.blobid"]; string(bid) != "" {
return string(bid), nil return string(bid), nil
} }
opencloud/pkg/revisions/revisions_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/test-go/testify/require" "github.com/test-go/testify/require"
) )
services/storage-system/pkg/config/config.go
+4 -4
View File
@@ -27,7 +27,7 @@ type Config struct {
SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token." introductionVersion:"pre5.0"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_SYSTEM_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token." introductionVersion:"pre5.0"`
FileMetadataCache Cache `yaml:"cache"` FileMetadataCache Cache `yaml:"cache"`
Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service." introductionVersion:"pre5.0"` Driver string `yaml:"driver" env:"STORAGE_SYSTEM_DRIVER" desc:"The driver which should be used by the service. The only supported driver is 'decomposed'. For backwards compatibility reasons it's also possible to use the 'ocis' driver and configure it using the 'decomposed' options. " introductionVersion:"pre5.0"`
Drivers Drivers `yaml:"drivers"` Drivers Drivers `yaml:"drivers"`
DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service." introductionVersion:"pre5.0"` DataServerURL string `yaml:"data_server_url" env:"STORAGE_SYSTEM_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by other services using this service." introductionVersion:"pre5.0"`
@@ -72,11 +72,11 @@ type HTTPConfig struct {
// Drivers holds Drivers config // Drivers holds Drivers config
type Drivers struct { type Drivers struct {
OCIS OCISDriver `yaml:"ocis"` Decomposed DecomposedDriver `yaml:"decomposed"`
} }
// OCISDriver holds ocis Driver config // DecomposedDriver holds the decomposed Driver config
type OCISDriver struct { type DecomposedDriver struct {
// Root is the absolute path to the location of the data // Root is the absolute path to the location of the data
Root string `yaml:"root" env:"STORAGE_SYSTEM_OC_ROOT" desc:"Path for the directory where the STORAGE-SYSTEM service stores it's persistent data. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage." introductionVersion:"pre5.0"` Root string `yaml:"root" env:"STORAGE_SYSTEM_OC_ROOT" desc:"Path for the directory where the STORAGE-SYSTEM service stores it's persistent data. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage." introductionVersion:"pre5.0"`
services/storage-system/pkg/config/defaults/defaultconfig.go
+2 -2
View File
@@ -42,9 +42,9 @@ func DefaultConfig() *config.Config {
}, },
Reva: shared.DefaultRevaConfig(), Reva: shared.DefaultRevaConfig(),
DataServerURL: "http://localhost:9216/data", DataServerURL: "http://localhost:9216/data",
Driver: "ocis", Driver: "decomposed",
Drivers: config.Drivers{ Drivers: config.Drivers{
OCIS: config.OCISDriver{ Decomposed: config.DecomposedDriver{
Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"), Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
MaxAcquireLockCycles: 20, MaxAcquireLockCycles: 20,
LockCycleDurationFactor: 30, LockCycleDurationFactor: 30,
services/storage-system/pkg/revaconfig/config.go
+27 -24
View File
@@ -148,30 +148,33 @@ func StorageSystemFromStruct(cfg *config.Config) map[string]interface{} {
} }
func metadataDrivers(localEndpoint string, cfg *config.Config) map[string]interface{} { func metadataDrivers(localEndpoint string, cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ m := map[string]interface{}{
"ocis": map[string]interface{}{ "metadata_backend": "messagepack",
"metadata_backend": "messagepack", "root": cfg.Drivers.Decomposed.Root,
"root": cfg.Drivers.OCIS.Root, "user_layout": "{{.Id.OpaqueId}}",
"user_layout": "{{.Id.OpaqueId}}", "treetime_accounting": false,
"treetime_accounting": false, "treesize_accounting": false,
"treesize_accounting": false, "permissionssvc": localEndpoint,
"permissionssvc": localEndpoint, "max_acquire_lock_cycles": cfg.Drivers.Decomposed.MaxAcquireLockCycles,
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles, "lock_cycle_duration_factor": cfg.Drivers.Decomposed.LockCycleDurationFactor,
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor, "disable_versioning": true,
"disable_versioning": true, "statcache": map[string]interface{}{
"statcache": map[string]interface{}{ "cache_store": "noop",
"cache_store": "noop", "cache_database": "system",
"cache_database": "system", },
}, "filemetadatacache": map[string]interface{}{
"filemetadatacache": map[string]interface{}{ "cache_store": cfg.FileMetadataCache.Store,
"cache_store": cfg.FileMetadataCache.Store, "cache_nodes": cfg.FileMetadataCache.Nodes,
"cache_nodes": cfg.FileMetadataCache.Nodes, "cache_database": cfg.FileMetadataCache.Database,
"cache_database": cfg.FileMetadataCache.Database, "cache_ttl": cfg.FileMetadataCache.TTL,
"cache_ttl": cfg.FileMetadataCache.TTL, "cache_disable_persistence": cfg.FileMetadataCache.DisablePersistence,
"cache_disable_persistence": cfg.FileMetadataCache.DisablePersistence, "cache_auth_username": cfg.FileMetadataCache.AuthUsername,
"cache_auth_username": cfg.FileMetadataCache.AuthUsername, "cache_auth_password": cfg.FileMetadataCache.AuthPassword,
"cache_auth_password": cfg.FileMetadataCache.AuthPassword,
},
}, },
} }
return map[string]interface{}{
"ocis": m, // deprecated: use decomposed
"decomposed": m,
}
} }
services/storage-users/pkg/config/config.go
+47 -46
View File
@@ -24,7 +24,7 @@ type Config struct {
SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token." introductionVersion:"pre5.0"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token." introductionVersion:"pre5.0"`
GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details." introductionVersion:"pre5.0"` GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details." introductionVersion:"pre5.0"`
Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket." introductionVersion:"pre5.0"` Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'decomposed', Supported values are: 'decomposed', 'decomposed_s3' and 'owncloudsql'. For backwards compatibility reasons it's also possible to use the 'ocis' and 's3ng' driver and configure them using the 'decomposed'/'decomposed_s3' options. The 'decomposed' driver stores all data (blob and meta data) in an POSIX compliant volume. The 'decomposed_s3' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket." introductionVersion:"pre5.0"`
Drivers Drivers `yaml:"drivers"` Drivers Drivers `yaml:"drivers"`
DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed." introductionVersion:"pre5.0"` DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed." introductionVersion:"pre5.0"`
DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server" introductionVersion:"pre5.0"` DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server" introductionVersion:"pre5.0"`
@@ -98,10 +98,10 @@ type CORS struct {
// Drivers combine all storage driver configurations // Drivers combine all storage driver configurations
type Drivers struct { type Drivers struct {
OCIS OCISDriver `yaml:"ocis"` Decomposed DecomposedDriver `yaml:"decomposed"`
S3NG S3NGDriver `yaml:"s3ng"` DecomposedS3 DecomposedS3Driver `yaml:"decomposed_s3"`
OwnCloudSQL OwnCloudSQLDriver `yaml:"owncloudsql"` OwnCloudSQL OwnCloudSQLDriver `yaml:"owncloudsql"`
Posix PosixDriver `yaml:"posix"` Posix PosixDriver `yaml:"posix"`
S3 S3Driver `yaml:",omitempty"` // not supported by the OpenCloud product, therefore not part of docs S3 S3Driver `yaml:",omitempty"` // not supported by the OpenCloud product, therefore not part of docs
EOS EOSDriver `yaml:",omitempty"` // not supported by the OpenCloud product, therefore not part of docs EOS EOSDriver `yaml:",omitempty"` // not supported by the OpenCloud product, therefore not part of docs
@@ -113,64 +113,65 @@ type AsyncPropagatorOptions struct {
PropagationDelay time.Duration `yaml:"propagation_delay" env:"STORAGE_USERS_ASYNC_PROPAGATOR_PROPAGATION_DELAY" desc:"The delay between a change made to a tree and the propagation start on treesize and treetime. Multiple propagations are computed to a single one. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"` PropagationDelay time.Duration `yaml:"propagation_delay" env:"STORAGE_USERS_ASYNC_PROPAGATOR_PROPAGATION_DELAY" desc:"The delay between a change made to a tree and the propagation start on treesize and treetime. Multiple propagations are computed to a single one. See the Environment Variable Types description for more details." introductionVersion:"pre5.0"`
} }
// OCISDriver is the storage driver configuration when using 'ocis' storage driver // DecomposedDriver is the storage driver configuration when using 'decomposed' storage driver
type OCISDriver struct { type DecomposedDriver struct {
Propagator string `yaml:"propagator" env:"OC_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_OCIS_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option." introductionVersion:"pre5.0"` Propagator string `yaml:"propagator" env:"OC_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_DECOMPOSED_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option." introductionVersion:"pre5.0"`
AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"` AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"`
// Root is the absolute path to the location of the data // Root is the absolute path to the location of the data
Root string `yaml:"root" env:"STORAGE_USERS_OCIS_ROOT" desc:"The directory where the filesystem storage will store blobs and metadata. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"pre5.0"` Root string `yaml:"root" env:"STORAGE_USERS_DECOMPOSED_ROOT" desc:"The directory where the filesystem storage will store blobs and metadata. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"pre5.0"`
UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_OCIS_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory." introductionVersion:"pre5.0"` UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_DECOMPOSED_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory." introductionVersion:"pre5.0"`
PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_OCIS_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'." introductionVersion:"pre5.0"` PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_DECOMPOSED_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'decomposed' and 'decomposed_s3'." introductionVersion:"pre5.0"`
// PersonalSpaceAliasTemplate contains the template used to construct // PersonalSpaceAliasTemplate contains the template used to construct
// the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"` // the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"`
PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_OCIS_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases." introductionVersion:"pre5.0"` PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_DECOMPOSED_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases." introductionVersion:"pre5.0"`
PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_OCIS_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"` PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_DECOMPOSED_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"`
// GeneralSpaceAliasTemplate contains the template used to construct // GeneralSpaceAliasTemplate contains the template used to construct
// the general space alias, eg: `{{.SpaceType}}/{{.SpaceName | replace " " "-" | lower}}` // the general space alias, eg: `{{.SpaceType}}/{{.SpaceName | replace " " "-" | lower}}`
GeneralSpaceAliasTemplate string `yaml:"generalspacealias_template" env:"STORAGE_USERS_OCIS_GENERAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct general space aliases." introductionVersion:"pre5.0"` GeneralSpaceAliasTemplate string `yaml:"generalspacealias_template" env:"STORAGE_USERS_DECOMPOSED_GENERAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct general space aliases." introductionVersion:"pre5.0"`
GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_OCIS_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"` GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_DECOMPOSED_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"`
// ShareFolder defines the name of the folder jailing all shares // ShareFolder defines the name of the folder jailing all shares
ShareFolder string `yaml:"share_folder" env:"STORAGE_USERS_OCIS_SHARE_FOLDER" desc:"Name of the folder jailing all shares." introductionVersion:"pre5.0"` ShareFolder string `yaml:"share_folder" env:"STORAGE_USERS_DECOMPOSED_SHARE_FOLDER" desc:"Name of the folder jailing all shares." introductionVersion:"pre5.0"`
MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_USERS_OCIS_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, OpenCloud will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"` MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_USERS_DECOMPOSED_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, OpenCloud will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"`
LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_USERS_OCIS_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, OpenCloud will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"` LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_USERS_DECOMPOSED_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, OpenCloud will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"`
MaxConcurrency int `yaml:"max_concurrency" env:"OC_MAX_CONCURRENCY;STORAGE_USERS_OCIS_MAX_CONCURRENCY" desc:"Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"` MaxConcurrency int `yaml:"max_concurrency" env:"OC_MAX_CONCURRENCY;STORAGE_USERS_DECOMPOSED_MAX_CONCURRENCY" desc:"Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value will be used." introductionVersion:"pre5.0"`
AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"pre5.0"` AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"pre5.0"`
MaxQuota uint64 `yaml:"max_quota" env:"OC_SPACES_MAX_QUOTA;STORAGE_USERS_OCIS_MAX_QUOTA" desc:"Set a global max quota for spaces in bytes. A value of 0 equals unlimited. If not using the global OC_SPACES_MAX_QUOTA, you must define the FRONTEND_MAX_QUOTA in the frontend service." introductionVersion:"pre5.0"` MaxQuota uint64 `yaml:"max_quota" env:"OC_SPACES_MAX_QUOTA;STORAGE_USERS_DECOMPOSED_MAX_QUOTA" desc:"Set a global max quota for spaces in bytes. A value of 0 equals unlimited. If not using the global OC_SPACES_MAX_QUOTA, you must define the FRONTEND_MAX_QUOTA in the frontend service." introductionVersion:"pre5.0"`
DisableVersioning bool `yaml:"disable_versioning" env:"OC_DISABLE_VERSIONING" desc:"Disables versioning of files. When set to true, new uploads with the same filename will overwrite existing files instead of creating a new version." introductionVersion:"7.0.0"` DisableVersioning bool `yaml:"disable_versioning" env:"OC_DISABLE_VERSIONING" desc:"Disables versioning of files. When set to true, new uploads with the same filename will overwrite existing files instead of creating a new version." introductionVersion:"7.0.0"`
} }
// S3NGDriver is the storage driver configuration when using 's3ng' storage driver // DecomposedS3Driver is the storage driver configuration when using 'decomposed_s3' storage driver
type S3NGDriver struct { type DecomposedS3Driver struct {
Propagator string `yaml:"propagator" env:"OC_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_S3NG_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option." introductionVersion:"pre5.0"` Propagator string `yaml:"propagator" env:"OC_DECOMPOSEDFS_PROPAGATOR;STORAGE_USERS_DECOMPOSEDS3_PROPAGATOR" desc:"The propagator used for decomposedfs. At the moment, only 'sync' is fully supported, 'async' is available as an experimental option." introductionVersion:"pre5.0"`
AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"` AsyncPropagatorOptions AsyncPropagatorOptions `yaml:"async_propagator_options"`
// Root is the absolute path to the location of the data // Root is the absolute path to the location of the data
Root string `yaml:"root" env:"STORAGE_USERS_S3NG_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"pre5.0"` Root string `yaml:"root" env:"STORAGE_USERS_DECOMPOSEDS3_ROOT" desc:"The directory where the filesystem storage will store metadata for blobs. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"pre5.0"`
UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_S3NG_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory." introductionVersion:"pre5.0"` UserLayout string `yaml:"user_layout" env:"STORAGE_USERS_DECOMPOSEDS3_USER_LAYOUT" desc:"Template string for the user storage layout in the user directory." introductionVersion:"pre5.0"`
PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_S3NG_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis' and 's3ng'." introductionVersion:"pre5.0"` PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_DECOMPOSEDS3_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'decomposed' and 'decomposed_s3'." introductionVersion:"pre5.0"`
Region string `yaml:"region" env:"STORAGE_USERS_S3NG_REGION" desc:"Region of the S3 bucket." introductionVersion:"pre5.0"` Region string `yaml:"region" env:"STORAGE_USERS_DECOMPOSEDS3_REGION" desc:"Region of the S3 bucket." introductionVersion:"pre5.0"`
AccessKey string `yaml:"access_key" env:"STORAGE_USERS_S3NG_ACCESS_KEY" desc:"Access key for the S3 bucket." introductionVersion:"pre5.0"` AccessKey string `yaml:"access_key" env:"STORAGE_USERS_DECOMPOSEDS3_ACCESS_KEY" desc:"Access key for the S3 bucket." introductionVersion:"pre5.0"`
SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_S3NG_SECRET_KEY" desc:"Secret key for the S3 bucket." introductionVersion:"pre5.0"` SecretKey string `yaml:"secret_key" env:"STORAGE_USERS_DECOMPOSEDS3_SECRET_KEY" desc:"Secret key for the S3 bucket." introductionVersion:"pre5.0"`
Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_S3NG_ENDPOINT" desc:"Endpoint for the S3 bucket." introductionVersion:"pre5.0"` Endpoint string `yaml:"endpoint" env:"STORAGE_USERS_DECOMPOSEDS3_ENDPOINT" desc:"Endpoint for the S3 bucket." introductionVersion:"pre5.0"`
Bucket string `yaml:"bucket" env:"STORAGE_USERS_S3NG_BUCKET" desc:"Name of the S3 bucket." introductionVersion:"pre5.0"` Bucket string `yaml:"bucket" env:"STORAGE_USERS_DECOMPOSEDS3_BUCKET" desc:"Name of the S3 bucket." introductionVersion:"pre5.0"`
DisableContentSha256 bool `yaml:"put_object_disable_content_sha254" env:"STORAGE_USERS_S3NG_PUT_OBJECT_DISABLE_CONTENT_SHA256" desc:"Disable sending content sha256 when copying objects to S3." introductionVersion:"5.0"` DisableContentSha256 bool `yaml:"put_object_disable_content_sha254" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_DISABLE_CONTENT_SHA256" desc:"Disable sending content sha256 when copying objects to S3." introductionVersion:"5.0"`
DisableMultipart bool `yaml:"put_object_disable_multipart" env:"STORAGE_USERS_S3NG_PUT_OBJECT_DISABLE_MULTIPART" desc:"Disable multipart uploads when copying objects to S3" introductionVersion:"5.0"` DisableMultipart bool `yaml:"put_object_disable_multipart" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_DISABLE_MULTIPART" desc:"Disable multipart uploads when copying objects to S3" introductionVersion:"5.0"`
SendContentMd5 bool `yaml:"put_object_send_content_md5" env:"STORAGE_USERS_S3NG_PUT_OBJECT_SEND_CONTENT_MD5" desc:"Send a Content-MD5 header when copying objects to S3." introductionVersion:"5.0"` SendContentMd5 bool `yaml:"put_object_send_content_md5" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_SEND_CONTENT_MD5" desc:"Send a Content-MD5 header when copying objects to S3." introductionVersion:"5.0"`
ConcurrentStreamParts bool `yaml:"put_object_concurrent_stream_parts" env:"STORAGE_USERS_S3NG_PUT_OBJECT_CONCURRENT_STREAM_PARTS" desc:"Always precreate parts when copying objects to S3." introductionVersion:"5.0"` ConcurrentStreamParts bool `yaml:"put_object_concurrent_stream_parts" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_CONCURRENT_STREAM_PARTS" desc:"Always precreate parts when copying objects to S3." introductionVersion:"5.0"`
NumThreads uint `yaml:"put_object_num_threads" env:"STORAGE_USERS_S3NG_PUT_OBJECT_NUM_THREADS" desc:"Number of concurrent uploads to use when copying objects to S3." introductionVersion:"5.0"` NumThreads uint `yaml:"put_object_num_threads" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_NUM_THREADS" desc:"Number of concurrent uploads to use when copying objects to S3." introductionVersion:"5.0"`
PartSize uint64 `yaml:"put_object_part_size" env:"STORAGE_USERS_S3NG_PUT_OBJECT_PART_SIZE" desc:"Part size for concurrent uploads to S3. If no value or 0 is set, the library's default value of 16MB is used. The value range is min 5MB and max 5GB." introductionVersion:"5.0"` PartSize uint64 `yaml:"put_object_part_size" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_PART_SIZE" desc:"Part size for concurrent uploads to S3. If no value or 0 is set, the library's default value of 16MB is used. The value range is min 5MB and max 5GB." introductionVersion:"5.0"`
// PersonalSpaceAliasTemplate contains the template used to construct // PersonalSpaceAliasTemplate contains the template used to construct
// the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"` // the personal space alias, eg: `"{{.SpaceType}}/{{.User.Username | lower}}"`
PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_S3NG_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases." introductionVersion:"pre5.0"` PersonalSpaceAliasTemplate string `yaml:"personalspacealias_template" env:"STORAGE_USERS_DECOMPOSEDS3_PERSONAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct personal space aliases." introductionVersion:"pre5.0"`
PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_S3NG_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"` PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_DECOMPOSEDS3_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"`
// GeneralSpaceAliasTemplate contains the template used to construct // GeneralSpaceAliasTemplate contains the template used to construct
// the general space alias, eg: `{{.SpaceType}}/{{.SpaceName | replace " " "-" | lower}}` // the general space alias, eg: `{{.SpaceType}}/{{.SpaceName | replace " " "-" | lower}}`
GeneralSpaceAliasTemplate string `yaml:"generalspacealias_template" env:"STORAGE_USERS_S3NG_GENERAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct general space aliases." introductionVersion:"pre5.0"` GeneralSpaceAliasTemplate string `yaml:"generalspacealias_template" env:"STORAGE_USERS_DECOMPOSEDS3_GENERAL_SPACE_ALIAS_TEMPLATE" desc:"Template string to construct general space aliases." introductionVersion:"pre5.0"`
GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_S3NG_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"` GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_DECOMPOSEDS3_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"`
// ShareFolder defines the name of the folder jailing all shares // ShareFolder defines the name of the folder jailing all shares
ShareFolder string `yaml:"share_folder" env:"STORAGE_USERS_S3NG_SHARE_FOLDER" desc:"Name of the folder jailing all shares." introductionVersion:"pre5.0"` ShareFolder string `yaml:"share_folder" env:"STORAGE_USERS_DECOMPOSEDS3_SHARE_FOLDER" desc:"Name of the folder jailing all shares." introductionVersion:"pre5.0"`
MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_USERS_S3NG_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, OpenCloud will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used." introductionVersion:"pre5.0"` MaxAcquireLockCycles int `yaml:"max_acquire_lock_cycles" env:"STORAGE_USERS_DECOMPOSEDS3_MAX_ACQUIRE_LOCK_CYCLES" desc:"When trying to lock files, OpenCloud will try this amount of times to acquire the lock before failing. After each try it will wait for an increasing amount of time. Values of 0 or below will be ignored and the default value of 20 will be used." introductionVersion:"pre5.0"`
LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_USERS_S3NG_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, OpenCloud will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used." introductionVersion:"pre5.0"` LockCycleDurationFactor int `yaml:"lock_cycle_duration_factor" env:"STORAGE_USERS_DECOMPOSEDS3_LOCK_CYCLE_DURATION_FACTOR" desc:"When trying to lock files, OpenCloud will multiply the cycle with this factor and use it as a millisecond timeout. Values of 0 or below will be ignored and the default value of 30 will be used." introductionVersion:"pre5.0"`
MaxConcurrency int `yaml:"max_concurrency" env:"OC_MAX_CONCURRENCY;STORAGE_USERS_S3NG_MAX_CONCURRENCY" desc:"Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used." introductionVersion:"pre5.0"` MaxConcurrency int `yaml:"max_concurrency" env:"OC_MAX_CONCURRENCY;STORAGE_USERS_DECOMPOSEDS3_MAX_CONCURRENCY" desc:"Maximum number of concurrent go-routines. Higher values can potentially get work done faster but will also cause more load on the system. Values of 0 or below will be ignored and the default value of 100 will be used." introductionVersion:"pre5.0"`
AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"pre5.0"`
DisableVersioning bool `yaml:"disable_versioning" env:"OC_DISABLE_VERSIONING" desc:"Disables versioning of files. When set to true, new uploads with the same filename will overwrite existing files instead of creating a new version." introductionVersion:"7.0.0"` DisableVersioning bool `yaml:"disable_versioning" env:"OC_DISABLE_VERSIONING" desc:"Disables versioning of files. When set to true, new uploads with the same filename will overwrite existing files instead of creating a new version." introductionVersion:"7.0.0"`
} }
@@ -196,7 +197,7 @@ type PosixDriver struct {
Root string `yaml:"root" env:"STORAGE_USERS_POSIX_ROOT" desc:"The directory where the filesystem storage will store its data. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"6.0.0"` Root string `yaml:"root" env:"STORAGE_USERS_POSIX_ROOT" desc:"The directory where the filesystem storage will store its data. If not defined, the root directory derives from $OC_BASE_DATA_PATH/storage/users." introductionVersion:"6.0.0"`
PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_POSIX_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"` PersonalSpacePathTemplate string `yaml:"personalspacepath_template" env:"STORAGE_USERS_POSIX_PERSONAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the personal space roots." introductionVersion:"6.0.0"`
GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_POSIX_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"` GeneralSpacePathTemplate string `yaml:"generalspacepath_template" env:"STORAGE_USERS_POSIX_GENERAL_SPACE_PATH_TEMPLATE" desc:"Template string to construct the paths of the projects space roots." introductionVersion:"6.0.0"`
PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_POSIX_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'ocis', 'posix' and 's3ng'." introductionVersion:"6.0.0"` PermissionsEndpoint string `yaml:"permissions_endpoint" env:"STORAGE_USERS_PERMISSION_ENDPOINT;STORAGE_USERS_POSIX_PERMISSIONS_ENDPOINT" desc:"Endpoint of the permissions service. The endpoints can differ for 'decomposed', 'posix' and 'decomposed_s3'." introductionVersion:"6.0.0"`
AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"pre5.0"` AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"pre5.0"`
ScanDebounceDelay time.Duration `yaml:"scan_debounce_delay" env:"STORAGE_USERS_POSIX_SCAN_DEBOUNCE_DELAY" desc:"The time in milliseconds to wait before scanning the filesystem for changes after a change has been detected." introductionVersion:"6.0.0"` ScanDebounceDelay time.Duration `yaml:"scan_debounce_delay" env:"STORAGE_USERS_POSIX_SCAN_DEBOUNCE_DELAY" desc:"The time in milliseconds to wait before scanning the filesystem for changes after a change has been detected." introductionVersion:"6.0.0"`
services/storage-users/pkg/config/defaults/defaultconfig.go
+3 -3
View File
@@ -91,7 +91,7 @@ func DefaultConfig() *config.Config {
TransferExpires: 86400, TransferExpires: 86400,
UploadExpiration: 24 * 60 * 60, UploadExpiration: 24 * 60 * 60,
GracefulShutdownTimeout: 30, GracefulShutdownTimeout: 30,
Driver: "ocis", Driver: "decomposed",
Drivers: config.Drivers{ Drivers: config.Drivers{
OwnCloudSQL: config.OwnCloudSQLDriver{ OwnCloudSQL: config.OwnCloudSQLDriver{
Root: filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"), Root: filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"),
@@ -105,7 +105,7 @@ func DefaultConfig() *config.Config {
DBName: "owncloud", DBName: "owncloud",
UsersProviderEndpoint: "eu.opencloud.api.users", UsersProviderEndpoint: "eu.opencloud.api.users",
}, },
S3NG: config.S3NGDriver{ DecomposedS3: config.DecomposedS3Driver{
Propagator: "sync", Propagator: "sync",
Root: filepath.Join(defaults.BaseDataPath(), "storage", "users"), Root: filepath.Join(defaults.BaseDataPath(), "storage", "users"),
ShareFolder: "/Shares", ShareFolder: "/Shares",
@@ -124,7 +124,7 @@ func DefaultConfig() *config.Config {
LockCycleDurationFactor: 30, LockCycleDurationFactor: 30,
DisableMultipart: true, DisableMultipart: true,
}, },
OCIS: config.OCISDriver{ Decomposed: config.DecomposedDriver{
Propagator: "sync", Propagator: "sync",
Root: filepath.Join(defaults.BaseDataPath(), "storage", "users"), Root: filepath.Join(defaults.BaseDataPath(), "storage", "users"),
ShareFolder: "/Shares", ShareFolder: "/Shares",
services/storage-users/pkg/revaconfig/drivers.go
+84 -84
View File
@@ -154,31 +154,31 @@ func OwnCloudSQL(cfg *config.Config) map[string]interface{} {
} }
} }
// Ocis is the config mapping for the Ocis storage driver // Decomposed is the config mapping for the Decomposed storage driver
func Ocis(cfg *config.Config) map[string]interface{} { func Decomposed(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"metadata_backend": "messagepack", "metadata_backend": "messagepack",
"propagator": cfg.Drivers.OCIS.Propagator, "propagator": cfg.Drivers.Decomposed.Propagator,
"async_propagator_options": map[string]interface{}{ "async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.OCIS.AsyncPropagatorOptions.PropagationDelay, "propagation_delay": cfg.Drivers.Decomposed.AsyncPropagatorOptions.PropagationDelay,
}, },
"root": cfg.Drivers.OCIS.Root, "root": cfg.Drivers.Decomposed.Root,
"user_layout": cfg.Drivers.OCIS.UserLayout, "user_layout": cfg.Drivers.Decomposed.UserLayout,
"share_folder": cfg.Drivers.OCIS.ShareFolder, "share_folder": cfg.Drivers.Decomposed.ShareFolder,
"personalspacealias_template": cfg.Drivers.OCIS.PersonalSpaceAliasTemplate, "personalspacealias_template": cfg.Drivers.Decomposed.PersonalSpaceAliasTemplate,
"personalspacepath_template": cfg.Drivers.OCIS.PersonalSpacePathTemplate, "personalspacepath_template": cfg.Drivers.Decomposed.PersonalSpacePathTemplate,
"generalspacealias_template": cfg.Drivers.OCIS.GeneralSpaceAliasTemplate, "generalspacealias_template": cfg.Drivers.Decomposed.GeneralSpaceAliasTemplate,
"generalspacepath_template": cfg.Drivers.OCIS.GeneralSpacePathTemplate, "generalspacepath_template": cfg.Drivers.Decomposed.GeneralSpacePathTemplate,
"treetime_accounting": true, "treetime_accounting": true,
"treesize_accounting": true, "treesize_accounting": true,
"permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint, "permissionssvc": cfg.Drivers.Decomposed.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles, "max_acquire_lock_cycles": cfg.Drivers.Decomposed.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor, "lock_cycle_duration_factor": cfg.Drivers.Decomposed.LockCycleDurationFactor,
"max_concurrency": cfg.Drivers.OCIS.MaxConcurrency, "max_concurrency": cfg.Drivers.Decomposed.MaxConcurrency,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, "asyncfileuploads": cfg.Drivers.Decomposed.AsyncUploads,
"max_quota": cfg.Drivers.OCIS.MaxQuota, "max_quota": cfg.Drivers.Decomposed.MaxQuota,
"disable_versioning": cfg.Drivers.OCIS.DisableVersioning, "disable_versioning": cfg.Drivers.Decomposed.DisableVersioning,
"filemetadatacache": map[string]interface{}{ "filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store, "cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes, "cache_nodes": cfg.FilemetadataCache.Nodes,
@@ -209,30 +209,30 @@ func Ocis(cfg *config.Config) map[string]interface{} {
} }
} }
// OcisNoEvents is the config mapping for the ocis storage driver emitting no events // DecomposedsNoEvents is the config mapping for the Decomposed storage driver emitting no events
func OcisNoEvents(cfg *config.Config) map[string]interface{} { func DecomposedNoEvents(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"metadata_backend": "messagepack", "metadata_backend": "messagepack",
"propagator": cfg.Drivers.OCIS.Propagator, "propagator": cfg.Drivers.Decomposed.Propagator,
"async_propagator_options": map[string]interface{}{ "async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.OCIS.AsyncPropagatorOptions.PropagationDelay, "propagation_delay": cfg.Drivers.Decomposed.AsyncPropagatorOptions.PropagationDelay,
}, },
"root": cfg.Drivers.OCIS.Root, "root": cfg.Drivers.Decomposed.Root,
"user_layout": cfg.Drivers.OCIS.UserLayout, "user_layout": cfg.Drivers.Decomposed.UserLayout,
"share_folder": cfg.Drivers.OCIS.ShareFolder, "share_folder": cfg.Drivers.Decomposed.ShareFolder,
"personalspacealias_template": cfg.Drivers.OCIS.PersonalSpaceAliasTemplate, "personalspacealias_template": cfg.Drivers.Decomposed.PersonalSpaceAliasTemplate,
"personalspacepath_template": cfg.Drivers.OCIS.PersonalSpacePathTemplate, "personalspacepath_template": cfg.Drivers.Decomposed.PersonalSpacePathTemplate,
"generalspacealias_template": cfg.Drivers.OCIS.GeneralSpaceAliasTemplate, "generalspacealias_template": cfg.Drivers.Decomposed.GeneralSpaceAliasTemplate,
"generalspacepath_template": cfg.Drivers.OCIS.GeneralSpacePathTemplate, "generalspacepath_template": cfg.Drivers.Decomposed.GeneralSpacePathTemplate,
"treetime_accounting": true, "treetime_accounting": true,
"treesize_accounting": true, "treesize_accounting": true,
"permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint, "permissionssvc": cfg.Drivers.Decomposed.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles, "max_acquire_lock_cycles": cfg.Drivers.Decomposed.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor, "lock_cycle_duration_factor": cfg.Drivers.Decomposed.LockCycleDurationFactor,
"max_concurrency": cfg.Drivers.OCIS.MaxConcurrency, "max_concurrency": cfg.Drivers.Decomposed.MaxConcurrency,
"max_quota": cfg.Drivers.OCIS.MaxQuota, "max_quota": cfg.Drivers.Decomposed.MaxQuota,
"disable_versioning": cfg.Drivers.OCIS.DisableVersioning, "disable_versioning": cfg.Drivers.Decomposed.DisableVersioning,
"filemetadatacache": map[string]interface{}{ "filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store, "cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes, "cache_nodes": cfg.FilemetadataCache.Nodes,
@@ -267,41 +267,41 @@ func S3(cfg *config.Config) map[string]interface{} {
} }
} }
// S3NG is the config mapping for the s3ng storage driver // DecomposedS3 is the config mapping for the Decomposed-S3 storage driver
func S3NG(cfg *config.Config) map[string]interface{} { func DecomposedS3(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"metadata_backend": "messagepack", "metadata_backend": "messagepack",
"propagator": cfg.Drivers.S3NG.Propagator, "propagator": cfg.Drivers.DecomposedS3.Propagator,
"async_propagator_options": map[string]interface{}{ "async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay, "propagation_delay": cfg.Drivers.DecomposedS3.AsyncPropagatorOptions.PropagationDelay,
}, },
"root": cfg.Drivers.S3NG.Root, "root": cfg.Drivers.DecomposedS3.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout, "user_layout": cfg.Drivers.DecomposedS3.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder, "share_folder": cfg.Drivers.DecomposedS3.ShareFolder,
"personalspacealias_template": cfg.Drivers.OCIS.PersonalSpaceAliasTemplate, "personalspacealias_template": cfg.Drivers.DecomposedS3.PersonalSpaceAliasTemplate,
"personalspacepath_template": cfg.Drivers.OCIS.PersonalSpacePathTemplate, "personalspacepath_template": cfg.Drivers.DecomposedS3.PersonalSpacePathTemplate,
"generalspacealias_template": cfg.Drivers.OCIS.GeneralSpaceAliasTemplate, "generalspacealias_template": cfg.Drivers.DecomposedS3.GeneralSpaceAliasTemplate,
"generalspacepath_template": cfg.Drivers.OCIS.GeneralSpacePathTemplate, "generalspacepath_template": cfg.Drivers.DecomposedS3.GeneralSpacePathTemplate,
"treetime_accounting": true, "treetime_accounting": true,
"treesize_accounting": true, "treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, "permissionssvc": cfg.Drivers.DecomposedS3.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region, "s3.region": cfg.Drivers.DecomposedS3.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey, "s3.access_key": cfg.Drivers.DecomposedS3.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey, "s3.secret_key": cfg.Drivers.DecomposedS3.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint, "s3.endpoint": cfg.Drivers.DecomposedS3.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket, "s3.bucket": cfg.Drivers.DecomposedS3.Bucket,
"s3.disable_content_sha254": cfg.Drivers.S3NG.DisableContentSha256, "s3.disable_content_sha254": cfg.Drivers.DecomposedS3.DisableContentSha256,
"s3.disable_multipart": cfg.Drivers.S3NG.DisableMultipart, "s3.disable_multipart": cfg.Drivers.DecomposedS3.DisableMultipart,
"s3.send_content_md5": cfg.Drivers.S3NG.SendContentMd5, "s3.send_content_md5": cfg.Drivers.DecomposedS3.SendContentMd5,
"s3.concurrent_stream_parts": cfg.Drivers.S3NG.ConcurrentStreamParts, "s3.concurrent_stream_parts": cfg.Drivers.DecomposedS3.ConcurrentStreamParts,
"s3.num_threads": cfg.Drivers.S3NG.NumThreads, "s3.num_threads": cfg.Drivers.DecomposedS3.NumThreads,
"s3.part_size": cfg.Drivers.S3NG.PartSize, "s3.part_size": cfg.Drivers.DecomposedS3.PartSize,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, "max_acquire_lock_cycles": cfg.Drivers.DecomposedS3.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, "lock_cycle_duration_factor": cfg.Drivers.DecomposedS3.LockCycleDurationFactor,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, "max_concurrency": cfg.Drivers.DecomposedS3.MaxConcurrency,
"disable_versioning": cfg.Drivers.S3NG.DisableVersioning, "disable_versioning": cfg.Drivers.DecomposedS3.DisableVersioning,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads, "asyncfileuploads": cfg.Drivers.DecomposedS3.AsyncUploads,
"filemetadatacache": map[string]interface{}{ "filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store, "cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes, "cache_nodes": cfg.FilemetadataCache.Nodes,
@@ -332,34 +332,34 @@ func S3NG(cfg *config.Config) map[string]interface{} {
} }
} }
// S3NGNoEvents is the config mapping for the s3ng storage driver emitting no events // DecomposedS3NoEvents is the config mapping for the Decomposed-S3 storage driver emitting no events
func S3NGNoEvents(cfg *config.Config) map[string]interface{} { func DecomposedS3NoEvents(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"metadata_backend": "messagepack", "metadata_backend": "messagepack",
"propagator": cfg.Drivers.S3NG.Propagator, "propagator": cfg.Drivers.DecomposedS3.Propagator,
"async_propagator_options": map[string]interface{}{ "async_propagator_options": map[string]interface{}{
"propagation_delay": cfg.Drivers.S3NG.AsyncPropagatorOptions.PropagationDelay, "propagation_delay": cfg.Drivers.DecomposedS3.AsyncPropagatorOptions.PropagationDelay,
}, },
"root": cfg.Drivers.S3NG.Root, "root": cfg.Drivers.DecomposedS3.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout, "user_layout": cfg.Drivers.DecomposedS3.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder, "share_folder": cfg.Drivers.DecomposedS3.ShareFolder,
"personalspacealias_template": cfg.Drivers.OCIS.PersonalSpaceAliasTemplate, "personalspacealias_template": cfg.Drivers.Decomposed.PersonalSpaceAliasTemplate,
"personalspacepath_template": cfg.Drivers.OCIS.PersonalSpacePathTemplate, "personalspacepath_template": cfg.Drivers.Decomposed.PersonalSpacePathTemplate,
"generalspacealias_template": cfg.Drivers.OCIS.GeneralSpaceAliasTemplate, "generalspacealias_template": cfg.Drivers.Decomposed.GeneralSpaceAliasTemplate,
"generalspacepath_template": cfg.Drivers.OCIS.GeneralSpacePathTemplate, "generalspacepath_template": cfg.Drivers.Decomposed.GeneralSpacePathTemplate,
"treetime_accounting": true, "treetime_accounting": true,
"treesize_accounting": true, "treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint, "permissionssvc": cfg.Drivers.DecomposedS3.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode, "permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region, "s3.region": cfg.Drivers.DecomposedS3.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey, "s3.access_key": cfg.Drivers.DecomposedS3.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey, "s3.secret_key": cfg.Drivers.DecomposedS3.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint, "s3.endpoint": cfg.Drivers.DecomposedS3.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket, "s3.bucket": cfg.Drivers.DecomposedS3.Bucket,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles, "max_acquire_lock_cycles": cfg.Drivers.DecomposedS3.MaxAcquireLockCycles,
"max_concurrency": cfg.Drivers.S3NG.MaxConcurrency, "max_concurrency": cfg.Drivers.DecomposedS3.MaxConcurrency,
"disable_versioning": cfg.Drivers.S3NG.DisableVersioning, "disable_versioning": cfg.Drivers.DecomposedS3.DisableVersioning,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor, "lock_cycle_duration_factor": cfg.Drivers.DecomposedS3.LockCycleDurationFactor,
"filemetadatacache": map[string]interface{}{ "filemetadatacache": map[string]interface{}{
"cache_store": cfg.FilemetadataCache.Store, "cache_store": cfg.FilemetadataCache.Store,
"cache_nodes": cfg.FilemetadataCache.Nodes, "cache_nodes": cfg.FilemetadataCache.Nodes,
services/storage-users/pkg/revaconfig/user.go
+30 -21
View File
@@ -1,35 +1,44 @@
package revaconfig package revaconfig
import "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" import (
"github.com/opencloud-eu/opencloud/services/storage-users/pkg/config"
)
// StorageProviderDrivers are the drivers for the storage provider // StorageProviderDrivers are the drivers for the storage provider
func StorageProviderDrivers(cfg *config.Config) map[string]interface{} { func StorageProviderDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"eos": EOS(cfg), "eos": EOS(cfg),
"eoshome": EOSHome(cfg), "eoshome": EOSHome(cfg),
"eosgrpc": EOSGRPC(cfg), "eosgrpc": EOSGRPC(cfg),
"local": Local(cfg), "local": Local(cfg),
"localhome": LocalHome(cfg), "localhome": LocalHome(cfg),
"owncloudsql": OwnCloudSQL(cfg), "owncloudsql": OwnCloudSQL(cfg),
"ocis": OcisNoEvents(cfg), "decomposed": DecomposedNoEvents(cfg),
"s3": S3(cfg), "s3": S3(cfg),
"s3ng": S3NGNoEvents(cfg), "decomposed_s3": DecomposedS3NoEvents(cfg),
"posix": Posix(cfg, true), "posix": Posix(cfg, true),
"ocis": Decomposed(cfg), // deprecated: use decomposed
"s3ng": DecomposedS3NoEvents(cfg), // deprecated: use decomposed_s3
} }
} }
// DataProviderDrivers are the drivers for the storage provider // DataProviderDrivers are the drivers for the storage provider
func DataProviderDrivers(cfg *config.Config) map[string]interface{} { func DataProviderDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"eos": EOS(cfg), "eos": EOS(cfg),
"eoshome": EOSHome(cfg), "eoshome": EOSHome(cfg),
"eosgrpc": EOSGRPC(cfg), "eosgrpc": EOSGRPC(cfg),
"local": Local(cfg), "local": Local(cfg),
"localhome": LocalHome(cfg), "localhome": LocalHome(cfg),
"owncloudsql": OwnCloudSQL(cfg), "owncloudsql": OwnCloudSQL(cfg),
"ocis": Ocis(cfg), "decomposed": Decomposed(cfg),
"s3": S3(cfg), "s3": S3(cfg),
"s3ng": S3NG(cfg), "decomposed_s3": DecomposedS3(cfg),
"posix": Posix(cfg, false), "posix": Posix(cfg, false),
"ocis": Decomposed(cfg), // deprecated: use decomposed
"s3ng": DecomposedS3NoEvents(cfg), // deprecated: use decomposed_s3
} }
} }
services/thumbnails/README.md
+1 -1
View File
@@ -7,7 +7,7 @@ The thumbnails service provides methods to generate thumbnails for various files
The relevant environment variables defining file locations are: The relevant environment variables defining file locations are:
- (1) `OC_BASE_DATA_PATH` - (1) `OC_BASE_DATA_PATH`
- (2) `STORAGE_USERS_OCIS_ROOT` - (2) `STORAGE_USERS_DECOMPOSED_ROOT`
- (3) `THUMBNAILS_FILESYSTEMSTORAGE_ROOT` - (3) `THUMBNAILS_FILESYSTEMSTORAGE_ROOT`
(1) ... Having a default set by the OpenCloud code, but if defined, used as base path for other services. (1) ... Having a default set by the OpenCloud code, but if defined, used as base path for other services.
tests/README.md
+22 -22
View File
@@ -19,7 +19,7 @@ Basically we have two sources for feature tests and test suites:
At the moment, both can be applied to OpenCloud. At the moment, both can be applied to OpenCloud.
As a storage backend, we support the OpenCloud native storage, also called `ocis`. This stores files directly on disk. Along with that we also provide `s3ng` storage driver. As a storage backend, we support the OpenCloud native storage, also called `decomposed`. This stores files directly on disk. Along with that we also provide `decomposed_s3` storage driver.
You can invoke two types of test suite runs: You can invoke two types of test suite runs:
@@ -30,23 +30,23 @@ You can invoke two types of test suite runs:
#### Local OpenCloud Tests (prefix `api`) #### Local OpenCloud Tests (prefix `api`)
The names of the full test suite make targets have the same naming as in the CI pipeline. See the available local OpenCloud specific test suites [here](https://github.com/opencloud-eu/opencloud/tree/master/tests/acceptance/features). They can be run with `ocis` storage and `s3ng` storage. The names of the full test suite make targets have the same naming as in the CI pipeline. See the available local OpenCloud specific test suites [here](https://github.com/opencloud-eu/opencloud/tree/master/tests/acceptance/features). They can be run with `decomposed` storage and `decomposed_s3` storage.
For example, command: For example, command:
```bash ```bash
make -C tests/acceptance/docker localApiTests-apiGraph-ocis make -C tests/acceptance/docker localApiTests-apiGraph-decomposed
``` ```
runs the same tests as the `localApiTests-apiGraph-ocis` CI pipeline, which runs the OpenCloud test suite "apiGraph" against the OpenCloud server with `ocis` storage. runs the same tests as the `localApiTests-apiGraph-decomposed` CI pipeline, which runs the OpenCloud test suite "apiGraph" against the OpenCloud server with `decomposed` storage.
And command: And command:
```bash ```bash
make -C tests/acceptance/docker localApiTests-apiGraph-s3ng make -C tests/acceptance/docker localApiTests-apiGraph-decomposed_s3
``` ```
runs the OpenCloud test suite `apiGraph` against the OpenCloud server with `s3ng` storage. runs the OpenCloud test suite `apiGraph` against the OpenCloud server with `decomposed_s3` storage.
Note: Note:
While running the tests, OpenCloud server is started with [ocwrapper](https://github.com/opencloud-eu/opencloud/blob/master/tests/ocwrapper/README.md) (i.e. `WITH_WRAPPER=true`) by default. In order to run the tests without ocwrapper, provide `WITH_WRAPPER=false` when running the tests. For example: While running the tests, OpenCloud server is started with [ocwrapper](https://github.com/opencloud-eu/opencloud/blob/master/tests/ocwrapper/README.md) (i.e. `WITH_WRAPPER=true`) by default. In order to run the tests without ocwrapper, provide `WITH_WRAPPER=false` when running the tests. For example:
@@ -54,7 +54,7 @@ While running the tests, OpenCloud server is started with [ocwrapper](https://gi
```bash ```bash
WITH_WRAPPER=false \ WITH_WRAPPER=false \
BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \ BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
But some test suites that are tagged with `@env-config` require the OpenCloud server to be run with ocwrapper. So, running those tests require `WITH_WRAPPER=true` (default setting). But some test suites that are tagged with `@env-config` require the OpenCloud server to be run with ocwrapper. So, running those tests require `WITH_WRAPPER=true` (default setting).
@@ -65,7 +65,7 @@ To run the tests that require an email server (tests tagged with `@email`), you
```bash ```bash
START_EMAIL=true \ START_EMAIL=true \
BEHAT_FEATURE='tests/acceptance/features/apiNotification/emailNotification.feature' \ BEHAT_FEATURE='tests/acceptance/features/apiNotification/emailNotification.feature' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
Note: Note:
@@ -74,7 +74,7 @@ To run the tests that require tika service (tests tagged with `@tikaServiceNeede
```bash ```bash
START_TIKA=true \ START_TIKA=true \
BEHAT_FEATURE='tests/acceptance/features/apiSearchContent/contentSearch.feature' \ BEHAT_FEATURE='tests/acceptance/features/apiSearchContent/contentSearch.feature' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
Note: Note:
@@ -86,14 +86,14 @@ OC_ASYNC_UPLOADS=true \
OC_ADD_RUN_SERVICES=antivirus \ OC_ADD_RUN_SERVICES=antivirus \
POSTPROCESSING_STEPS=virusscan \ POSTPROCESSING_STEPS=virusscan \
BEHAT_FEATURE='tests/acceptance/features/apiAntivirus/antivirus.feature' \ BEHAT_FEATURE='tests/acceptance/features/apiAntivirus/antivirus.feature' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
#### Tests Transferred From Core (prefix `coreApi`) #### Tests Transferred From Core (prefix `coreApi`)
Command `make -C tests/acceptance/docker Core-API-Tests-ocis-storage-3` runs the same tests as the `Core-API-Tests-ocis-storage-3` CI pipeline, which runs the third (out of ten) test suite groups transferred from core against the OpenCloud server with `ocis` storage. Command `make -C tests/acceptance/docker Core-API-Tests-decomposed-storage-3` runs the same tests as the `Core-API-Tests-decomposed-storage-3` CI pipeline, which runs the third (out of ten) test suite groups transferred from core against the OpenCloud server with `decomposed` storage.
And `make -C tests/acceptance/docker Core-API-Tests-s3ng-storage-3` runs the third (out of ten) test suite groups transferred from core against the OpenCloud server with `s3ng` storage. And `make -C tests/acceptance/docker Core-API-Tests-decomposed_s3-storage-3` runs the third (out of ten) test suite groups transferred from core against the OpenCloud server with `decomposed_s3` storage.
### Run Single Feature Test ### Run Single Feature Test
@@ -103,7 +103,7 @@ For example;
```bash ```bash
BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature' \ BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
Note: Note:
@@ -116,19 +116,19 @@ A specific scenario from a feature can be run by adding `:<line-number>` at the
```bash ```bash
BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \ BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \
make -C tests/acceptance/docker test-opencloud-feature-ocis-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed-storage
``` ```
Similarly, with `s3ng` storage; Similarly, with `decomposed_s3` storage;
```bash ```bash
# run a whole feature # run a whole feature
BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature' \ BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature' \
make -C tests/acceptance/docker test-opencloud-feature-s3ng-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed_s3-storage
# run a single scenario # run a single scenario
BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \ BEHAT_FEATURE='tests/acceptance/features/apiGraphUserGroup/createUser.feature:26' \
make -C tests/acceptance/docker test-opencloud-feature-s3ng-storage make -C tests/acceptance/docker test-opencloud-feature-decomposed_s3-storage
``` ```
In the same way, tests transferred from core can be run as: In the same way, tests transferred from core can be run as:
@@ -136,11 +136,11 @@ In the same way, tests transferred from core can be run as:
```bash ```bash
# run a whole feature # run a whole feature
BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature' \ BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature' \
make -C tests/acceptance/docker test-core-feature-ocis-storage make -C tests/acceptance/docker test-core-feature-decomposed-storage
# run a single scenario # run a single scenario
BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature:15' \ BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature:15' \
make -C tests/acceptance/docker test-core-feature-ocis-storage make -C tests/acceptance/docker test-core-feature-decomposed-storage
``` ```
Note: Note:
@@ -222,7 +222,7 @@ A specific scenario from a feature can be run by adding `:<line-number>` at the
> >
> BEHAT_SUITE=apiGraph > BEHAT_SUITE=apiGraph
`STORAGE_DRIVER`: to run tests with a different user storage driver. Available options are `ocis` (default), `owncloudsql` and `s3ng` `STORAGE_DRIVER`: to run tests with a different user storage driver. Available options are `decomposed` (default), `owncloudsql` and `decomposed_s3`
> Example: > Example:
> >
@@ -237,7 +237,7 @@ A specific scenario from a feature can be run by adding `:<line-number>` at the
### Use Existing Tests for BDD ### Use Existing Tests for BDD
As a lot of scenarios are written for core, we can use those tests for Behaviour driven development in OpenCloud. As a lot of scenarios are written for core, we can use those tests for Behaviour driven development in OpenCloud.
Every scenario that does not work in OpenCloud with `ocis` storage, is listed in `tests/acceptance/expected-failures-API-on-OCIS-storage.md` with a link to the related issue. Every scenario that does not work in OpenCloud with `decomposed` storage, is listed in `tests/acceptance/expected-failures-API-on-decomposed-storage.md` with a link to the related issue.
Those scenarios are run in the ordinary acceptance test pipeline in CI. The scenarios that fail are checked against the Those scenarios are run in the ordinary acceptance test pipeline in CI. The scenarios that fail are checked against the
expected failures. If there are any differences then the CI pipeline fails. expected failures. If there are any differences then the CI pipeline fails.
@@ -251,7 +251,7 @@ If you want to work on a specific issue
```bash ```bash
make test-acceptance-api \ make test-acceptance-api \
TEST_SERVER_URL=https://localhost:9200 \ TEST_SERVER_URL=https://localhost:9200 \
STORAGE_DRIVER=ocis \ STORAGE_DRIVER=decomposed \
BEHAT_FEATURE='tests/acceptance/features/coreApiVersions/fileVersions.feature:141' BEHAT_FEATURE='tests/acceptance/features/coreApiVersions/fileVersions.feature:141'
``` ```
tests/acceptance/TestHelpers/OcHelper.php
+5 -5
View File
@@ -31,10 +31,10 @@ use GuzzleHttp\Exception\GuzzleException;
* @package TestHelpers * @package TestHelpers
*/ */
abstract class StorageDriver { abstract class StorageDriver {
public const OCIS = "OCIS"; public const DECOMPOSED = "DECOMPOSED";
public const EOS = "EOS"; public const EOS = "EOS";
public const OWNCLOUD = "OWNCLOUD"; public const OWNCLOUD = "OWNCLOUD";
public const S3NG = "S3NG"; public const DECOMPOSEDS3 = "DECOMPOSED_S3";
public const POSIX = "POSIX"; public const POSIX = "POSIX";
} }
@@ -47,10 +47,10 @@ abstract class StorageDriver {
*/ */
class OcHelper { class OcHelper {
public const STORAGE_DRIVERS = [ public const STORAGE_DRIVERS = [
StorageDriver::OCIS, StorageDriver::DECOMPOSED,
StorageDriver::EOS, StorageDriver::EOS,
StorageDriver::OWNCLOUD, StorageDriver::OWNCLOUD,
StorageDriver::S3NG, StorageDriver::DECOMPOSEDS3,
StorageDriver::POSIX StorageDriver::POSIX
]; ];
@@ -109,7 +109,7 @@ class OcHelper {
public static function getStorageDriver(): string { public static function getStorageDriver(): string {
$storageDriver = (\getenv("STORAGE_DRIVER")); $storageDriver = (\getenv("STORAGE_DRIVER"));
if ($storageDriver === false) { if ($storageDriver === false) {
return StorageDriver::OCIS; return StorageDriver::DECOMPOSED;
} }
$storageDriver = \strtoupper($storageDriver); $storageDriver = \strtoupper($storageDriver);
if (!\in_array($storageDriver, self::STORAGE_DRIVERS)) { if (!\in_array($storageDriver, self::STORAGE_DRIVERS)) {
tests/acceptance/bootstrap/FeatureContext.php
+3 -3
View File
@@ -752,7 +752,7 @@ class FeatureContext extends BehatVariablesContext {
*/ */
public function getStorageUsersRoot(): string { public function getStorageUsersRoot(): string {
$ocDataPath = getenv("OC_BASE_DATA_PATH") ? getenv("OC_BASE_DATA_PATH") : getenv("HOME") . '/.opencloud'; $ocDataPath = getenv("OC_BASE_DATA_PATH") ? getenv("OC_BASE_DATA_PATH") : getenv("HOME") . '/.opencloud';
return getenv("STORAGE_USERS_OCIS_ROOT") ? getenv("STORAGE_USERS_OCIS_ROOT") : $ocDataPath . "/storage/users"; return getenv("STORAGE_USERS_DECOMPOSED_ROOT") ? getenv("STORAGE_USERS_DECOMPOSED_ROOT") : $ocDataPath . "/storage/users";
} }
/** /**
@@ -3008,9 +3008,9 @@ class FeatureContext extends BehatVariablesContext {
public static function isExpectedToFail(string $scenarioLine): bool { public static function isExpectedToFail(string $scenarioLine): bool {
$expectedFailFile = \getenv('EXPECTED_FAILURES_FILE'); $expectedFailFile = \getenv('EXPECTED_FAILURES_FILE');
if (!$expectedFailFile) { if (!$expectedFailFile) {
$expectedFailFile = __DIR__ . '/../expected-failures-localAPI-on-OCIS-storage.md'; $expectedFailFile = __DIR__ . '/../expected-failures-localAPI-on-decomposed-storage.md';
if (\strpos($scenarioLine, "coreApi") === 0) { if (\strpos($scenarioLine, "coreApi") === 0) {
$expectedFailFile = __DIR__ . '/../expected-failures-API-on-OCIS-storage.md'; $expectedFailFile = __DIR__ . '/../expected-failures-API-on-decomposed-storage.md';
} }
} }
tests/acceptance/docker/Makefile
+40 -40
View File
@@ -67,26 +67,26 @@ help:
@echo -e "where ${YELLOW}latest${RESET} is an example for any valid Docker image tag from" @echo -e "where ${YELLOW}latest${RESET} is an example for any valid Docker image tag from"
@echo -e "https://hub.docker.com/r/opencloud-eu/opencloud." @echo -e "https://hub.docker.com/r/opencloud-eu/opencloud."
@echo @echo
@echo -e "${GREEN}Run full OpenCloud test suites with ocis storage:${RESET}\n" @echo -e "${GREEN}Run full OpenCloud test suites with decomposed storage:${RESET}\n"
@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite, where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}" @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-decomposed\t\t${BLUE}run apiAccountsHashDifficulty test suite, where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}"
@echo @echo
@echo -e "${GREEN}Run full OpenCloud test suites with s3ng storage:${RESET}\n" @echo -e "${GREEN}Run full OpenCloud test suites with decomposed_s3 storage:${RESET}\n"
@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-s3ng\t\t${BLUE}run apiAccountsHashDifficulty test suite, where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}" @echo -e "\tmake localApiTests-apiAccountsHashDifficulty-decomposed_s3\t\t${BLUE}run apiAccountsHashDifficulty test suite, where available test suite are apiAccountsHashDifficulty apiArchiver apiContract apiGraph apiSpaces apiSpacesShares apiAsyncUpload apiCors${RESET}"
@echo @echo
@echo -e "${GREEN}Run full OpenCloud test suites with ocis storage:${RESET}\n" @echo -e "${GREEN}Run full OpenCloud test suites with decomposed storage:${RESET}\n"
@echo -e "\tmake Core-API-Tests-ocis-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @echo -e "\tmake Core-API-Tests-decomposed-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
@echo @echo
@echo -e "${GREEN}Run full OpenCloud test suites with s3ng storage:${RESET}\n" @echo -e "${GREEN}Run full OpenCloud test suites with decomposed_s3 storage:${RESET}\n"
@echo -e "\tmake Core-API-Tests-s3ng-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @echo -e "\tmake Core-API-Tests-decomposed_s3-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
@echo @echo
@echo -e "${GREEN}Run an OpenCloud feature test with ocis storage:${RESET}\n" @echo -e "${GREEN}Run an OpenCloud feature test with decomposed storage:${RESET}\n"
@echo -e "\tmake test-opencloud-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo -e "\tmake test-opencloud-feature-decomposed-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
@echo @echo
@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
@echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}"
@echo @echo
@echo -e "${GREEN}Run an OpenCloud feature test with s3ng storage:${RESET}\n" @echo -e "${GREEN}Run an OpenCloud feature test with decomposed_s3 storage:${RESET}\n"
@echo -e "\tmake test-opencloud-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo -e "\tmake test-opencloud-feature-decomposed_s3-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
@echo @echo
@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
@echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}"
@@ -94,14 +94,14 @@ help:
@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
@echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}"
@echo @echo
@echo -e "${GREEN}Run a core test against OpenCloud with ocis storage:${RESET}\n" @echo -e "${GREEN}Run a core test against OpenCloud with decomposed storage:${RESET}\n"
@echo -e "\tmake test-core-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo -e "\tmake test-core-feature-decomposed-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
@echo @echo
@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
@echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}" @echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}"
@echo @echo
@echo -e "${GREEN}Run a core test against OpenCloud with s3ng storage:${RESET}\n" @echo -e "${GREEN}Run a core test against OpenCloud with decomposed_s3 storage:${RESET}\n"
@echo -e "\tmake test-core-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo -e "\tmake test-core-feature-decomposed_s3-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
@echo @echo
@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
@echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}" @echo -e "\texample: ${RED}tests/acceptance/features/coreApiAuth/webDavAuth.feature${RESET}"
@@ -119,69 +119,69 @@ help:
@echo -e "\tmake clean-docker-container\t\t${BLUE}stops and removes used docker containers${RESET}" @echo -e "\tmake clean-docker-container\t\t${BLUE}stops and removes used docker containers${RESET}"
@echo -e "\tmake clean-docker-volumes\t\t${BLUE}removes used docker volumes (used for caching)${RESET}" @echo -e "\tmake clean-docker-volumes\t\t${BLUE}removes used docker volumes (used for caching)${RESET}"
@echo @echo
.PHONY: test-opencloud-feature-ocis-storage .PHONY: test-opencloud-feature-decomposed-storage
test-opencloud-feature-ocis-storage: ## test a OpenCloud feature with ocis storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' test-opencloud-feature-decomposed-storage: ## test a OpenCloud feature with decomposed storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10'
@TEST_SOURCE=opencloud \ @TEST_SOURCE=opencloud \
STORAGE_DRIVER=ocis \ STORAGE_DRIVER=decomposed \
BEHAT_FEATURE=$(BEHAT_FEATURE) \ BEHAT_FEATURE=$(BEHAT_FEATURE) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
.PHONY: test-opencloud-feature-s3ng-storage .PHONY: test-opencloud-feature-decomposed_s3-storage
test-opencloud-feature-s3ng-storage: ## test a OpenCloud feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' test-opencloud-feature-decomposed_s3-storage: ## test a OpenCloud feature with decomposed_s3 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10'
@TEST_SOURCE=opencloud \ @TEST_SOURCE=opencloud \
STORAGE_DRIVER=s3ng \ STORAGE_DRIVER=decomposed_s3 \
BEHAT_FEATURE=$(BEHAT_FEATURE) \ BEHAT_FEATURE=$(BEHAT_FEATURE) \
START_CEPH=1 \ START_CEPH=1 \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
.PHONY: test-core-feature-ocis-storage .PHONY: test-core-feature-decomposed-storage
test-core-feature-ocis-storage: ## test a core feature with ocis storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature' test-core-feature-decomposed-storage: ## test a core feature with decomposed storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature'
@TEST_SOURCE=core \ @TEST_SOURCE=core \
STORAGE_DRIVER=ocis \ STORAGE_DRIVER=decomposed \
BEHAT_FEATURE=$(BEHAT_FEATURE) \ BEHAT_FEATURE=$(BEHAT_FEATURE) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
.PHONY: test-core-feature-s3ng-storage .PHONY: test-core-feature-decomposed_s3-storage
test-core-feature-s3ng-storage: ## test a core feature with s3ng storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature' test-core-feature-decomposed_s3-storage: ## test a core feature with decomposed_s3 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/coreApiAuth/webDavAuth.feature'
@TEST_SOURCE=core \ @TEST_SOURCE=core \
STORAGE_DRIVER=s3ng \ STORAGE_DRIVER=decomposed_s3 \
BEHAT_FEATURE=$(BEHAT_FEATURE) \ BEHAT_FEATURE=$(BEHAT_FEATURE) \
START_CEPH=1 \ START_CEPH=1 \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
localSuiteOpencloud = $(addprefix localApiTests-, $(addsuffix -ocis,${LOCAL_API_SUITES})) localSuiteOpencloud = $(addprefix localApiTests-, $(addsuffix -decomposed,${LOCAL_API_SUITES}))
.PHONY: $(localSuiteOpencloud) .PHONY: $(localSuiteOpencloud)
$(localSuiteOpencloud): ## run local api test suite with ocis storage $(localSuiteOpencloud): ## run local api test suite with decomposed storage
@$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2)) @$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2))
@TEST_SOURCE=opencloud \ @TEST_SOURCE=opencloud \
STORAGE_DRIVER=ocis \ STORAGE_DRIVER=decomposed \
BEHAT_SUITE=$(BEHAT_SUITE) \ BEHAT_SUITE=$(BEHAT_SUITE) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
localSuites3ng = $(addprefix localApiTests-, $(addsuffix -s3ng,${LOCAL_API_SUITES})) localSuiteDecomposedS3 = $(addprefix localApiTests-, $(addsuffix -decomposed_s3,${LOCAL_API_SUITES}))
.PHONY: $(localSuites3ng) .PHONY: $(localSuiteDecomposedS3)
$(localSuites3ng): ## run local api test suite with s3 storage $(localSuiteDecomposedS3): ## run local api test suite with s3 storage
@$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2)) @$(eval BEHAT_SUITE=$(shell echo "$@" | cut -d'-' -f2))
@TEST_SOURCE=opencloud \ @TEST_SOURCE=opencloud \
STORAGE_DRIVER=s3ng \ STORAGE_DRIVER=decomposed_s3 \
BEHAT_SUITE=$(BEHAT_SUITE) \ BEHAT_SUITE=$(BEHAT_SUITE) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
targetsOC = $(addprefix Core-API-Tests-ocis-storage-,$(PARTS)) targetsOC = $(addprefix Core-API-Tests-decomposed-storage-,$(PARTS))
.PHONY: $(targetsOC) .PHONY: $(targetsOC)
$(targetsOC): $(targetsOC):
@$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9')) @$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9'))
@TEST_SOURCE=core \ @TEST_SOURCE=core \
STORAGE_DRIVER=ocis \ STORAGE_DRIVER=decomposed \
RUN_PART=$(RUN_PART) \ RUN_PART=$(RUN_PART) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
targetsS3ng = $(addprefix Core-API-Tests-s3ng-storage-,$(PARTS)) targetsDecomposedS3 = $(addprefix Core-API-Tests-decomposed_s3-storage-,$(PARTS))
.PHONY: $(targetsS3ng) .PHONY: $(targetsDecomposedS3)
$(targets): $(targets):
@$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9')) @$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9'))
@TEST_SOURCE=core \ @TEST_SOURCE=core \
STORAGE_DRIVER=s3ng \ STORAGE_DRIVER=decomposed_s3 \
RUN_PART=$(RUN_PART) \ RUN_PART=$(RUN_PART) \
$(MAKE) --no-print-directory testSuite $(MAKE) --no-print-directory testSuite
tests/acceptance/docker/src/run-tests.sh
+9 -9
View File
@@ -9,12 +9,12 @@ git config --global advice.detachedHead false
if [ "$TEST_SOURCE" = "core" ]; then if [ "$TEST_SOURCE" = "core" ]; then
export ACCEPTANCE_TEST_TYPE='core-api' export ACCEPTANCE_TEST_TYPE='core-api'
if [ "$STORAGE_DRIVER" = "ocis" ]; then if [ "$STORAGE_DRIVER" = "decomposed" ]; then
export OC_REVA_DATA_ROOT='' export OC_REVA_DATA_ROOT=''
export BEHAT_FILTER_TAGS='~@skipOnOpencloud-OCIS-Storage' export BEHAT_FILTER_TAGS='~@skipOnOpencloud-decomposed-Storage'
export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-OCIS-storage.md' export EXPECTED_FAILURES_FILE='/drone/src/tests/acceptance/expected-failures-API-on-decomposed-storage.md'
elif [ "$STORAGE_DRIVER" = "s3ng" ]; then elif [ "$STORAGE_DRIVER" = "decomposed_s3" ]; then
export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-S3NG-Storage' export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-decomposed_s3-Storage'
export OC_REVA_DATA_ROOT='' export OC_REVA_DATA_ROOT=''
else else
echo "non existing STORAGE selected" echo "non existing STORAGE selected"
@@ -24,11 +24,11 @@ if [ "$TEST_SOURCE" = "core" ]; then
unset BEHAT_SUITE unset BEHAT_SUITE
elif [ "$TEST_SOURCE" = "opencloud" ]; then elif [ "$TEST_SOURCE" = "opencloud" ]; then
if [ "$STORAGE_DRIVER" = "ocis" ]; then if [ "$STORAGE_DRIVER" = "decomposed" ]; then
export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-OCIS-Storage' export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-decomposed-Storage'
export OC_REVA_DATA_ROOT='' export OC_REVA_DATA_ROOT=''
elif [ "$STORAGE_DRIVER" = "s3ng" ]; then elif [ "$STORAGE_DRIVER" = "decomposed_s3" ]; then
export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-S3NG-Storage' export BEHAT_FILTER_TAGS='~@skip&&~@skipOnOpencloud-decomposed_s3-Storage'
export OC_REVA_DATA_ROOT='' export OC_REVA_DATA_ROOT=''
else else
echo "non existing storage selected" echo "non existing storage selected"