storage: add group provider service and sharing SQL driver

2021-02-16 15:55:42 +01:00
parent 0726156f1b
commit d8ade4a917
13 changed files with 585 additions and 70 deletions
@@ -104,6 +104,13 @@ func GatewayWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_GATEWAY_HOME_MAPPING"},
 			Destination: &cfg.Reva.Gateway.HomeMapping,
 		},
+		&cli.IntFlag{
+			Name:        "etag-cache-ttl",
+			Value:       0,
+			Usage:       "TTL for the home and shares directory etags cache",
+			EnvVars:     []string{"STORAGE_GATEWAY_ETAG_CACHE_TTL"},
+			Destination: &cfg.Reva.Gateway.EtagCacheTTL,
+		},

 		// other services

@@ -0,0 +1,148 @@
+package flagset
+
+import (
+	"github.com/micro/cli/v2"
+	"github.com/owncloud/ocis/storage/pkg/config"
+)
+
+// GroupsWithConfig applies cfg to the root flagset
+func GroupsWithConfig(cfg *config.Config) []cli.Flag {
+	flags := []cli.Flag{
+
+		// debug ports are the odd ports
+		&cli.StringFlag{
+			Name:        "debug-addr",
+			Value:       "0.0.0.0:9161",
+			Usage:       "Address to bind debug server",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_DEBUG_ADDR"},
+			Destination: &cfg.Reva.Groups.DebugAddr,
+		},
+
+		// Services
+
+		// Groupprovider
+
+		&cli.StringFlag{
+			Name:        "network",
+			Value:       "tcp",
+			Usage:       "Network to use for the storage service, can be 'tcp', 'udp' or 'unix'",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_NETWORK"},
+			Destination: &cfg.Reva.Groups.GRPCNetwork,
+		},
+		&cli.StringFlag{
+			Name:        "addr",
+			Value:       "0.0.0.0:9160",
+			Usage:       "Address to bind storage service",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_ADDR"},
+			Destination: &cfg.Reva.Groups.GRPCAddr,
+		},
+		&cli.StringFlag{
+			Name:        "endpoint",
+			Value:       "localhost:9160",
+			Usage:       "URL to use for the storage service",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_ENDPOINT"},
+			Destination: &cfg.Reva.Groups.Endpoint,
+		},
+		&cli.StringSliceFlag{
+			Name:    "service",
+			Value:   cli.NewStringSlice("groupprovider"), // TODO preferences
+			Usage:   "--service groupprovider [--service otherservice]",
+			EnvVars: []string{"STORAGE_GROUPPROVIDER_SERVICES"},
+		},
+
+		&cli.StringFlag{
+			Name:        "driver",
+			Value:       "ldap",
+			Usage:       "group driver: 'json', 'ldap', or 'rest'",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_DRIVER"},
+			Destination: &cfg.Reva.Groups.Driver,
+		},
+		&cli.StringFlag{
+			Name:        "json-config",
+			Value:       "",
+			Usage:       "Path to groups.json file",
+			EnvVars:     []string{"STORAGE_GROUPPROVIDER_JSON"},
+			Destination: &cfg.Reva.Groups.JSON,
+		},
+    &cli.IntFlag{
+			Name:        "group-members-cache-expiration",
+			Value:       5,
+			Usage:       "Time in minutes for redis cache expiration.",
+			EnvVars:     []string{"STORAGE_GROUP_CACHE_EXPIRATION"},
+			Destination: &cfg.Reva.Groups.GroupMembersCacheExpiration,
+		},
+
+    // rest driver
+
+    &cli.StringFlag{
+			Name:        "rest-client-id",
+			Value:       "",
+			Usage:       "User/group rest driver Client ID",
+			EnvVars:     []string{"STORAGE_REST_CLIENT_ID"},
+			Destination: &cfg.Reva.UserGroupRest.ClientID,
+		},
+		&cli.StringFlag{
+			Name:        "rest-client-secret",
+			Value:       "",
+			Usage:       "User/group rest driver Client Secret",
+			EnvVars:     []string{"STORAGE_REST_CLIENT_SECRET"},
+			Destination: &cfg.Reva.UserGroupRest.ClientSecret,
+		},
+		&cli.StringFlag{
+			Name:        "rest-redis-address",
+			Value:       "localhost:6379",
+			Usage:       "Address for redis server",
+			EnvVars:     []string{"STORAGE_REST_REDIS_ADDRESS"},
+			Destination: &cfg.Reva.UserGroupRest.RedisAddress,
+		},
+		&cli.StringFlag{
+			Name:        "rest-redis-username",
+			Value:       "",
+			Usage:       "Username for redis server",
+			EnvVars:     []string{"STORAGE_REST_REDIS_USERNAME"},
+			Destination: &cfg.Reva.UserGroupRest.RedisUsername,
+		},
+		&cli.StringFlag{
+			Name:        "rest-redis-password",
+			Value:       "",
+			Usage:       "Password for redis server",
+			EnvVars:     []string{"STORAGE_REST_REDIS_PASSWORD"},
+			Destination: &cfg.Reva.UserGroupRest.RedisPassword,
+		},
+		&cli.StringFlag{
+			Name:        "rest-id-provider",
+			Value:       "",
+			Usage:       "The OIDC Provider",
+			EnvVars:     []string{"STORAGE_REST_ID_PROVIDER"},
+			Destination: &cfg.Reva.UserGroupRest.IDProvider,
+		},
+		&cli.StringFlag{
+			Name:        "rest-api-base-url",
+			Value:       "",
+			Usage:       "Base API Endpoint",
+			EnvVars:     []string{"STORAGE_REST_API_BASE_URL"},
+			Destination: &cfg.Reva.UserGroupRest.APIBaseURL,
+		},
+		&cli.StringFlag{
+			Name:        "rest-oidc-token-endpoint",
+			Value:       "",
+			Usage:       "Endpoint to generate token to access the API",
+			EnvVars:     []string{"STORAGE_REST_OIDC_TOKEN_ENDPOINT"},
+			Destination: &cfg.Reva.UserGroupRest.OIDCTokenEndpoint,
+		},
+		&cli.StringFlag{
+			Name:        "rest-target-api",
+			Value:       "",
+			Usage:       "The target application",
+			EnvVars:     []string{"STORAGE_REST_TARGET_API"},
+			Destination: &cfg.Reva.UserGroupRest.TargetAPI,
+		},
+	}
+
+	flags = append(flags, TracingWithConfig(cfg)...)
+	flags = append(flags, DebugWithConfig(cfg)...)
+	flags = append(flags, SecretWithConfig(cfg)...)
+	flags = append(flags, LDAPWithConfig(cfg)...)
+
+	return flags
+}
@@ -36,6 +36,9 @@ func LDAPWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_LDAP_LOGINFILTER"},
 			Destination: &cfg.Reva.LDAP.LoginFilter,
 		},
+
+		// User specific filters
+
 		&cli.StringFlag{
 			Name:        "ldap-userfilter",
 			Value:       "(&(objectclass=posixAccount)(|(ownclouduuid={{.OpaqueId}})(cn={{.OpaqueId}})))",
@@ -44,28 +47,61 @@ func LDAPWithConfig(cfg *config.Config) []cli.Flag {
 			Destination: &cfg.Reva.LDAP.UserFilter,
 		},
 		&cli.StringFlag{
-			Name:        "ldap-attributefilter",
+			Name:        "ldap-userattributefilter",
 			Value:       "(&(objectclass=posixAccount)({{attr}}={{value}}))",
 			Usage:       "LDAP filter used when searching for a user by claim/attribute. {{attr}} will be replaced with the attribute, {{value}} with the value.",
-			EnvVars:     []string{"STORAGE_LDAP_ATTRIBUTEFILTER"},
-			Destination: &cfg.Reva.LDAP.AttributeFilter,
+			EnvVars:     []string{"STORAGE_LDAP_USERATTRIBUTEFILTER"},
+			Destination: &cfg.Reva.LDAP.UserAttributeFilter,
 		},
 		&cli.StringFlag{
-			Name:        "ldap-findfilter",
+			Name:        "ldap-userfindfilter",
 			Value:       "(&(objectclass=posixAccount)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))",
-			Usage:       "LDAP filter used when searching for recipients. {{query}} will be replaced with the search query",
-			EnvVars:     []string{"STORAGE_LDAP_FINDFILTER"},
-			Destination: &cfg.Reva.LDAP.FindFilter,
+			Usage:       "LDAP filter used when searching for user recipients. {{query}} will be replaced with the search query",
+			EnvVars:     []string{"STORAGE_LDAP_USERFINDFILTER"},
+			Destination: &cfg.Reva.LDAP.UserFindFilter,
 		},
 		&cli.StringFlag{
-			Name: "ldap-groupfilter",
+			Name: "ldap-usergroupfilter",
 			// FIXME the storage implementation needs to use the memberof overlay to get the cn when it only has the uuid,
 			// because the ldap schema either uses the dn or the member(of) attributes to establish membership
 			Value:       "(&(objectclass=posixGroup)(ownclouduuid={{.OpaqueId}}*))", // This filter will never work
 			Usage:       "LDAP filter used when getting the groups of a user. The CS3 userid properties {{.OpaqueId}} and {{.Idp}} are available.",
+			EnvVars:     []string{"STORAGE_LDAP_USERGROUPFILTER"},
+			Destination: &cfg.Reva.LDAP.UserGroupFilter,
+		},
+
+		// Group specific filters
+		// These might not work at the moment. Need to be fixed
+
+		&cli.StringFlag{
+			Name:        "ldap-groupfilter",
+			Value:       "(&(objectclass=posixGroup)(|(ownclouduuid={{.OpaqueId}})(cn={{.OpaqueId}})))",
+			Usage:       "LDAP filter used when getting a group. The CS3 groupid properties {{.OpaqueId}} and {{.Idp}} are available.",
 			EnvVars:     []string{"STORAGE_LDAP_GROUPFILTER"},
 			Destination: &cfg.Reva.LDAP.GroupFilter,
 		},
+		&cli.StringFlag{
+			Name:        "ldap-groupattributefilter",
+			Value:       "(&(objectclass=posixGroup)({{attr}}={{value}}))",
+			Usage:       "LDAP filter used when searching for a group by claim/attribute. {{attr}} will be replaced with the attribute, {{value}} with the value.",
+			EnvVars:     []string{"STORAGE_LDAP_GROUPATTRIBUTEFILTER"},
+			Destination: &cfg.Reva.LDAP.GroupAttributeFilter,
+		},
+		&cli.StringFlag{
+			Name:        "ldap-groupfindfilter",
+			Value:       "(&(objectclass=posixGroup)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))",
+			Usage:       "LDAP filter used when searching for group recipients. {{query}} will be replaced with the search query",
+			EnvVars:     []string{"STORAGE_LDAP_GROUPFINDFILTER"},
+			Destination: &cfg.Reva.LDAP.GroupFindFilter,
+		},
+		&cli.StringFlag{
+			Name: "ldap-groupmemberfilter",
+			// FIXME the storage implementation needs to use the members overlay to get the cn when it only has the uuid
+			Value:       "(&(objectclass=posixAccount)(ownclouduuid={{.OpaqueId}}*))", // This filter will never work
+			Usage:       "LDAP filter used when getting the members of a group. The CS3 groupid properties {{.OpaqueId}} and {{.Idp}} are available.",
+			EnvVars:     []string{"STORAGE_LDAP_GROUPMEMBERFILTER"},
+			Destination: &cfg.Reva.LDAP.GroupMemberFilter,
+		},
 		&cli.StringFlag{
 			Name:        "ldap-bind-dn",
 			Value:       "cn=reva,ou=sysusers,dc=example,dc=org",
@@ -95,6 +131,13 @@ func LDAPWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_LDAP_SCHEMA_UID"},
 			Destination: &cfg.Reva.LDAP.Schema.UID,
 		},
+		&cli.StringFlag{
+			Name:        "ldap-schema-gid",
+			Value:       "ownclouduuid",
+			Usage:       "LDAP schema gid",
+			EnvVars:     []string{"STORAGE_LDAP_SCHEMA_GID"},
+			Destination: &cfg.Reva.LDAP.Schema.GID,
+		},
 		&cli.StringFlag{
 			Name:        "ldap-schema-mail",
 			Value:       "mail",
@@ -56,6 +56,41 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_SHARING_USER_JSON_FILE"},
 			Destination: &cfg.Reva.Sharing.UserJSONFile,
 		},
+		&cli.StringFlag{
+			Name:        "user-sql-username",
+			Value:       "",
+			Usage:       "Username to be used to connect to the SQL database",
+			EnvVars:     []string{"STORAGE_SHARING_USER_SQL_USERNAME"},
+			Destination: &cfg.Reva.Sharing.UserSQLUsername,
+		},
+		&cli.StringFlag{
+			Name:        "user-sql-password",
+			Value:       "",
+			Usage:       "Password to be used to connect to the SQL database",
+			EnvVars:     []string{"STORAGE_SHARING_USER_SQL_PASSWORD"},
+			Destination: &cfg.Reva.Sharing.UserSQLPassword,
+		},
+		&cli.StringFlag{
+			Name:        "user-sql-host",
+			Value:       "",
+			Usage:       "Hostname of the SQL database",
+			EnvVars:     []string{"STORAGE_SHARING_USER_SQL_HOST"},
+			Destination: &cfg.Reva.Sharing.UserSQLHost,
+		},
+		&cli.IntFlag{
+			Name:        "user-sql-port",
+			Value:       1433,
+			Usage:       "The port on which the SQL database is exposed",
+			EnvVars:     []string{"STORAGE_SHARING_USER_SQL_PORT"},
+			Destination: &cfg.Reva.Sharing.UserSQLPort,
+		},
+		&cli.StringFlag{
+			Name:        "user-sql-name",
+			Value:       "",
+			Usage:       "Name of the SQL database",
+			EnvVars:     []string{"STORAGE_SHARING_USER_SQL_Name"},
+			Destination: &cfg.Reva.Sharing.UserSQLName,
+		},
 		&cli.StringFlag{
 			Name:        "public-driver",
 			Value:       "json",
@@ -64,78 +64,78 @@ func UsersWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_USERPROVIDER_JSON"},
 			Destination: &cfg.Reva.Users.JSON,
 		},
+		&cli.IntFlag{
+			Name:        "user-groups-cache-expiration",
+			Value:       5,
+			Usage:       "Time in minutes for redis cache expiration.",
+			EnvVars:     []string{"STORAGE_USER_CACHE_EXPIRATION"},
+			Destination: &cfg.Reva.Users.UserGroupsCacheExpiration,
+		},

 		// rest driver

 		&cli.StringFlag{
 			Name:        "rest-client-id",
 			Value:       "",
-			Usage:       "User rest driver Client ID",
+			Usage:       "User/group rest driver Client ID",
 			EnvVars:     []string{"STORAGE_REST_CLIENT_ID"},
-			Destination: &cfg.Reva.UserRest.ClientID,
+			Destination: &cfg.Reva.UserGroupRest.ClientID,
 		},
 		&cli.StringFlag{
 			Name:        "rest-client-secret",
 			Value:       "",
-			Usage:       "User rest driver Client Secret",
+			Usage:       "User/group rest driver Client Secret",
 			EnvVars:     []string{"STORAGE_REST_CLIENT_SECRET"},
-			Destination: &cfg.Reva.UserRest.ClientSecret,
+			Destination: &cfg.Reva.UserGroupRest.ClientSecret,
 		},
 		&cli.StringFlag{
 			Name:        "rest-redis-address",
 			Value:       "localhost:6379",
 			Usage:       "Address for redis server",
 			EnvVars:     []string{"STORAGE_REST_REDIS_ADDRESS"},
-			Destination: &cfg.Reva.UserRest.RedisAddress,
+			Destination: &cfg.Reva.UserGroupRest.RedisAddress,
 		},
 		&cli.StringFlag{
 			Name:        "rest-redis-username",
 			Value:       "",
 			Usage:       "Username for redis server",
 			EnvVars:     []string{"STORAGE_REST_REDIS_USERNAME"},
-			Destination: &cfg.Reva.UserRest.RedisUsername,
+			Destination: &cfg.Reva.UserGroupRest.RedisUsername,
 		},
 		&cli.StringFlag{
 			Name:        "rest-redis-password",
 			Value:       "",
 			Usage:       "Password for redis server",
 			EnvVars:     []string{"STORAGE_REST_REDIS_PASSWORD"},
-			Destination: &cfg.Reva.UserRest.RedisPassword,
-		},
-		&cli.IntFlag{
-			Name:        "rest-user-groups-cache-expiration",
-			Value:       5,
-			Usage:       "Time in minutes for redis cache expiration.",
-			EnvVars:     []string{"STORAGE_REST_CACHE_EXPIRATION"},
-			Destination: &cfg.Reva.UserRest.UserGroupsCacheExpiration,
+			Destination: &cfg.Reva.UserGroupRest.RedisPassword,
 		},
 		&cli.StringFlag{
 			Name:        "rest-id-provider",
 			Value:       "",
 			Usage:       "The OIDC Provider",
 			EnvVars:     []string{"STORAGE_REST_ID_PROVIDER"},
-			Destination: &cfg.Reva.UserRest.IDProvider,
+			Destination: &cfg.Reva.UserGroupRest.IDProvider,
 		},
 		&cli.StringFlag{
 			Name:        "rest-api-base-url",
 			Value:       "",
 			Usage:       "Base API Endpoint",
 			EnvVars:     []string{"STORAGE_REST_API_BASE_URL"},
-			Destination: &cfg.Reva.UserRest.APIBaseURL,
+			Destination: &cfg.Reva.UserGroupRest.APIBaseURL,
 		},
 		&cli.StringFlag{
 			Name:        "rest-oidc-token-endpoint",
 			Value:       "",
 			Usage:       "Endpoint to generate token to access the API",
 			EnvVars:     []string{"STORAGE_REST_OIDC_TOKEN_ENDPOINT"},
-			Destination: &cfg.Reva.UserRest.OIDCTokenEndpoint,
+			Destination: &cfg.Reva.UserGroupRest.OIDCTokenEndpoint,
 		},
 		&cli.StringFlag{
 			Name:        "rest-target-api",
 			Value:       "",
 			Usage:       "The target application",
 			EnvVars:     []string{"STORAGE_REST_TARGET_API"},
-			Destination: &cfg.Reva.UserRest.TargetAPI,
+			Destination: &cfg.Reva.UserGroupRest.TargetAPI,
 		},
 	}