switch to go vendoring

This commit is contained in:
Michael Barz
2023-04-19 20:24:34 +02:00
parent 632fa05ef9
commit afc6ed1e41
8527 changed files with 3004916 additions and 2 deletions
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package app
import (
"context"
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
registry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
// Registry is the interface that application registries implement
// for discovering application providers
type Registry interface {
FindProviders(ctx context.Context, mimeType string) ([]*registry.ProviderInfo, error)
ListProviders(ctx context.Context) ([]*registry.ProviderInfo, error)
ListSupportedMimeTypes(ctx context.Context) ([]*registry.MimeTypeInfo, error)
AddProvider(ctx context.Context, p *registry.ProviderInfo) error
GetDefaultProviderForMimeType(ctx context.Context, mimeType string) (*registry.ProviderInfo, error)
SetDefaultProviderForMimeType(ctx context.Context, mimeType string, p *registry.ProviderInfo) error
}
// Provider is the interface that application providers implement
// for interacting with external apps that serve the requested resource.
type Provider interface {
GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.OpenInAppRequest_ViewMode, token, language string) (*appprovider.OpenInAppURL, error)
GetAppProviderInfo(ctx context.Context) (*registry.ProviderInfo, error)
}
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package demo
import (
"context"
"fmt"
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/app"
"github.com/cs3org/reva/v2/pkg/app/provider/registry"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/mitchellh/mapstructure"
)
func init() {
registry.Register("demo", New)
}
type demoProvider struct {
iframeUIProvider string
}
func (p *demoProvider) GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.OpenInAppRequest_ViewMode, token, language string) (*appprovider.OpenInAppURL, error) {
url := fmt.Sprintf("<iframe src=%s/open/%s?view-mode=%s&access-token=%s />", p.iframeUIProvider, storagespace.FormatResourceID(*resource.Id), viewMode.String(), token)
return &appprovider.OpenInAppURL{
AppUrl: url,
Method: "GET",
}, nil
}
func (p *demoProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) {
return &appregistry.ProviderInfo{
Name: "demo-app",
}, nil
}
type config struct {
IFrameUIProvider string `mapstructure:"iframe_ui_provider"`
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
// New returns an implementation to of the app.Provider interface that
// connects to an application in the backend.
func New(m map[string]interface{}) (app.Provider, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
return &demoProvider{iframeUIProvider: c.IFrameUIProvider}, nil
}
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core application providers.
_ "github.com/cs3org/reva/v2/pkg/app/provider/demo"
_ "github.com/cs3org/reva/v2/pkg/app/provider/wopi"
// Add your own here
)
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/app"
// NewFunc is the function that app provider implementations
// should register to at init time.
type NewFunc func(map[string]interface{}) (app.Provider, error)
// NewFuncs is a map containing all the registered app providers.
var NewFuncs = map[string]NewFunc{}
// Register registers a new app provider new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+468
View File
@@ -0,0 +1,468 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package wopi
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/beevik/etree"
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/app"
"github.com/cs3org/reva/v2/pkg/app/provider/registry"
"github.com/cs3org/reva/v2/pkg/appctx"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/mime"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/cs3org/reva/v2/pkg/sharedconf"
"github.com/cs3org/reva/v2/pkg/storage/utils/templates"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/golang-jwt/jwt"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("wopi", New)
}
type config struct {
IOPSecret string `mapstructure:"iop_secret" docs:";The IOP secret used to connect to the wopiserver."`
WopiURL string `mapstructure:"wopi_url" docs:";The wopiserver's URL."`
WopiFolderURLBaseURL string `mapstructure:"wopi_folder_url_base_url" docs:";The base URL to generate links to navigate back to the containing folder."`
WopiFolderURLPathTemplate string `mapstructure:"wopi_folder_url_path_template" docs:";The template to generate the folderurl path segments."`
AppName string `mapstructure:"app_name" docs:";The App user-friendly name."`
AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."`
AppURL string `mapstructure:"app_url" docs:";The App URL."`
AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"`
AppAPIKey string `mapstructure:"app_api_key" docs:";The API key used by the app, if applicable."`
JWTSecret string `mapstructure:"jwt_secret" docs:";The JWT secret to be used to retrieve the token TTL."`
AppDesktopOnly bool `mapstructure:"app_desktop_only" docs:"false;Specifies if the app can be opened only on desktop."`
InsecureConnections bool `mapstructure:"insecure_connections"`
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
type wopiProvider struct {
conf *config
wopiClient *http.Client
appURLs map[string]map[string]string // map[viewMode]map[extension]appURL
}
// New returns an implementation of the app.Provider interface that
// connects to an application in the backend.
func New(m map[string]interface{}) (app.Provider, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
if c.AppIntURL == "" {
c.AppIntURL = c.AppURL
}
if c.IOPSecret == "" {
c.IOPSecret = os.Getenv("REVA_APPPROVIDER_IOPSECRET")
}
c.JWTSecret = sharedconf.GetJWTSecret(c.JWTSecret)
appURLs, err := getAppURLs(c)
if err != nil {
return nil, err
}
wopiClient := rhttp.GetHTTPClient(
rhttp.Timeout(time.Duration(5*int64(time.Second))),
rhttp.Insecure(c.InsecureConnections),
)
wopiClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
return &wopiProvider{
conf: c,
wopiClient: wopiClient,
appURLs: appURLs,
}, nil
}
func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.OpenInAppRequest_ViewMode, token, language string) (*appprovider.OpenInAppURL, error) {
log := appctx.GetLogger(ctx)
ext := path.Ext(resource.Path)
wopiurl, err := url.Parse(p.conf.WopiURL)
if err != nil {
return nil, err
}
wopiurl.Path = path.Join(wopiurl.Path, "/wopi/iop/openinapp")
httpReq, err := rhttp.NewRequest(ctx, "GET", wopiurl.String(), nil)
if err != nil {
return nil, err
}
q := httpReq.URL.Query()
q.Add("endpoint", storagespace.FormatStorageID(resource.GetId().GetStorageId(), resource.GetId().GetSpaceId()))
q.Add("fileid", resource.GetId().OpaqueId)
q.Add("viewmode", viewMode.String())
folderURLPath := templates.WithResourceInfo(resource, p.conf.WopiFolderURLPathTemplate)
folderURLBaseURL, err := url.Parse(p.conf.WopiFolderURLBaseURL)
if err != nil {
return nil, err
}
if folderURLPath != "" {
folderURLBaseURL.Path = path.Join(folderURLBaseURL.Path, folderURLPath)
q.Add("folderurl", folderURLBaseURL.String())
}
u, ok := ctxpkg.ContextGetUser(ctx)
if ok { // else defaults to "Guest xyz"
if u.Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT || u.Id.Type == userpb.UserType_USER_TYPE_FEDERATED {
q.Add("userid", resource.Owner.OpaqueId+"@"+resource.Owner.Idp)
} else {
q.Add("userid", u.Id.OpaqueId+"@"+u.Id.Idp)
}
var isPublicShare bool
if u.Opaque != nil {
if _, ok := u.Opaque.Map["public-share-role"]; ok {
isPublicShare = true
}
}
if !isPublicShare {
q.Add("username", u.DisplayName)
}
}
q.Add("appname", p.conf.AppName)
var viewAppURL string
if viewAppURLs, ok := p.appURLs["view"]; ok {
if viewAppURL, ok = viewAppURLs[ext]; ok {
q.Add("appviewurl", viewAppURL)
}
}
access := "edit"
if resource.GetSize() == 0 {
if _, ok := p.appURLs["editnew"]; ok {
access = "editnew"
}
}
if editAppURLs, ok := p.appURLs[access]; ok {
if editAppURL, ok := editAppURLs[ext]; ok {
q.Add("appurl", editAppURL)
}
}
if q.Get("appurl") == "" {
// assuming that a view action is always available in the /hosting/discovery manifest
// eg. Collabora does support viewing jpgs but no editing
// eg. OnlyOffice does support viewing pdfs but no editing
// there is no known case of supporting edit only without view
q.Add("appurl", viewAppURL)
}
if q.Get("appurl") == "" && q.Get("appviewurl") == "" {
return nil, errors.New("wopi: neither edit nor view app url found")
}
if p.conf.AppIntURL != "" {
q.Add("appinturl", p.conf.AppIntURL)
}
httpReq.URL.RawQuery = q.Encode()
if p.conf.AppAPIKey != "" {
httpReq.Header.Set("ApiKey", p.conf.AppAPIKey)
}
httpReq.Header.Set("Authorization", "Bearer "+p.conf.IOPSecret)
httpReq.Header.Set("TokenHeader", token)
// Call the WOPI server and parse the response (body will always contain a payload)
openRes, err := p.wopiClient.Do(httpReq)
if err != nil {
return nil, errors.Wrap(err, "wopi: error performing open request to WOPI server")
}
defer openRes.Body.Close()
body, err := io.ReadAll(openRes.Body)
if err != nil {
return nil, err
}
if openRes.StatusCode != http.StatusOK {
// WOPI returned failure: body contains a user-friendly error message (yet perform a sanity check)
sbody := ""
if body != nil {
sbody = string(body)
}
log.Warn().Msg(fmt.Sprintf("wopi: WOPI server returned HTTP %s to request %s, error was: %s", openRes.Status, httpReq.URL.String(), sbody))
return nil, errors.New(sbody)
}
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, err
}
tokenTTL, err := p.getAccessTokenTTL(ctx)
if err != nil {
return nil, err
}
appFullURL := result["app-url"].(string)
if language != "" {
url, err := url.Parse(appFullURL)
if err != nil {
return nil, err
}
urlQuery := url.Query()
urlQuery.Set("ui", language) // OnlyOffice
urlQuery.Set("lang", language) // Collabora
urlQuery.Set("UI_LLCC", language) // Office365
url.RawQuery = urlQuery.Encode()
appFullURL = url.String()
}
// Depending on whether wopi server returned any form parameters or not,
// we decide whether the request method is POST or GET
var formParams map[string]string
method := "GET"
if form, ok := result["form-parameters"].(map[string]interface{}); ok {
if tkn, ok := form["access_token"].(string); ok {
formParams = map[string]string{
"access_token": tkn,
"access_token_ttl": tokenTTL,
}
method = "POST"
}
}
log.Info().Msg(fmt.Sprintf("wopi: returning app URL %s", appFullURL))
return &appprovider.OpenInAppURL{
AppUrl: appFullURL,
Method: method,
FormParameters: formParams,
}, nil
}
func (p *wopiProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) {
// Initially we store the mime types in a map to avoid duplicates
mimeTypesMap := make(map[string]bool)
for _, extensions := range p.appURLs {
for ext := range extensions {
m := mime.Detect(false, ext)
mimeTypesMap[m] = true
}
}
mimeTypes := make([]string, 0, len(mimeTypesMap))
for m := range mimeTypesMap {
mimeTypes = append(mimeTypes, m)
}
return &appregistry.ProviderInfo{
Name: p.conf.AppName,
Icon: p.conf.AppIconURI,
DesktopOnly: p.conf.AppDesktopOnly,
MimeTypes: mimeTypes,
}, nil
}
func getAppURLs(c *config) (map[string]map[string]string, error) {
// Initialize WOPI URLs by discovery
httpcl := rhttp.GetHTTPClient(
rhttp.Timeout(time.Duration(5*int64(time.Second))),
rhttp.Insecure(c.InsecureConnections),
)
appurl, err := url.Parse(c.AppIntURL)
if err != nil {
return nil, err
}
appurl.Path = path.Join(appurl.Path, "/hosting/discovery")
discReq, err := http.NewRequest("GET", appurl.String(), nil)
if err != nil {
return nil, err
}
discRes, err := httpcl.Do(discReq)
if err != nil {
return nil, err
}
defer discRes.Body.Close()
var appURLs map[string]map[string]string
if discRes.StatusCode == http.StatusOK {
appURLs, err = parseWopiDiscovery(discRes.Body)
if err != nil {
return nil, errors.Wrap(err, "error parsing wopi discovery response")
}
} else if discRes.StatusCode == http.StatusNotFound {
// this may be a bridge-supported app
discReq, err = http.NewRequest("GET", c.AppIntURL, nil)
if err != nil {
return nil, err
}
discRes, err = httpcl.Do(discReq)
if err != nil {
return nil, err
}
defer discRes.Body.Close()
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(discRes.Body)
if err != nil {
return nil, err
}
// scrape app's home page to find the appname
if !strings.Contains(buf.String(), c.AppName) {
return nil, errors.New("Application server at " + c.AppURL + " does not match this AppProvider for " + c.AppName)
}
// register the supported mimetypes in the AppRegistry: this is hardcoded for the time being
// TODO(lopresti) move to config
switch c.AppName {
case "CodiMD":
appURLs = getCodimdExtensions(c.AppURL)
case "Etherpad":
appURLs = getEtherpadExtensions(c.AppURL)
default:
return nil, errors.New("Application server " + c.AppName + " running at " + c.AppURL + " is unsupported")
}
}
return appURLs, nil
}
func (p *wopiProvider) getAccessTokenTTL(ctx context.Context) (string, error) {
tkn := ctxpkg.ContextMustGetToken(ctx)
token, err := jwt.ParseWithClaims(tkn, &jwt.StandardClaims{}, func(token *jwt.Token) (interface{}, error) {
return []byte(p.conf.JWTSecret), nil
})
if err != nil {
return "", err
}
if claims, ok := token.Claims.(*jwt.StandardClaims); ok && token.Valid {
// milliseconds since Jan 1, 1970 UTC as required in https://wopi.readthedocs.io/projects/wopirest/en/latest/concepts.html?highlight=access_token_ttl#term-access-token-ttl
return strconv.FormatInt(claims.ExpiresAt*1000, 10), nil
}
return "", errtypes.InvalidCredentials("wopi: invalid token present in ctx")
}
func parseWopiDiscovery(body io.Reader) (map[string]map[string]string, error) {
appURLs := make(map[string]map[string]string)
doc := etree.NewDocument()
if _, err := doc.ReadFrom(body); err != nil {
return nil, err
}
root := doc.SelectElement("wopi-discovery")
if root == nil {
return nil, errors.New("wopi-discovery response malformed")
}
for _, netzone := range root.SelectElements("net-zone") {
if strings.Contains(netzone.SelectAttrValue("name", ""), "external") {
for _, app := range netzone.SelectElements("app") {
for _, action := range app.SelectElements("action") {
access := action.SelectAttrValue("name", "")
if access == "view" || access == "edit" || access == "editnew" {
ext := action.SelectAttrValue("ext", "")
urlString := action.SelectAttrValue("urlsrc", "")
if ext == "" || urlString == "" {
continue
}
u, err := url.Parse(urlString)
if err != nil {
// it sucks we cannot log here because this function is run
// on init without any context.
// TODO(labkode): add logging when we'll have static logging in boot phase.
continue
}
// remove any malformed query parameter from discovery urls
q := u.Query()
for k := range q {
if strings.Contains(k, "<") || strings.Contains(k, ">") {
q.Del(k)
}
}
u.RawQuery = q.Encode()
if _, ok := appURLs[access]; !ok {
appURLs[access] = make(map[string]string)
}
appURLs[access]["."+ext] = u.String()
}
}
}
}
}
return appURLs, nil
}
func getCodimdExtensions(appURL string) map[string]map[string]string {
// Register custom mime types
mime.RegisterMime(".zmd", "application/compressed-markdown")
appURLs := make(map[string]map[string]string)
appURLs["edit"] = map[string]string{
".txt": appURL,
".md": appURL,
".zmd": appURL,
}
return appURLs
}
func getEtherpadExtensions(appURL string) map[string]map[string]string {
appURLs := make(map[string]map[string]string)
appURLs["edit"] = map[string]string{
".epd": appURL,
}
return appURLs
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core app registry drivers.
_ "github.com/cs3org/reva/v2/pkg/app/registry/static"
// Add your own here
)
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/app"
// NewFunc is the function that app provider implementations
// should register to at init time.
type NewFunc func(map[string]interface{}) (app.Registry, error)
// NewFuncs is a map containing all the registered app registry backends.
var NewFuncs = map[string]NewFunc{}
// Register registers a new app registry new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+377
View File
@@ -0,0 +1,377 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package static
import (
"container/heap"
"context"
"fmt"
"strconv"
"strings"
"sync"
registrypb "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1"
"github.com/cs3org/reva/v2/pkg/app"
"github.com/cs3org/reva/v2/pkg/app/registry/registry"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/mitchellh/mapstructure"
"github.com/rs/zerolog/log"
orderedmap "github.com/wk8/go-ordered-map"
)
func init() {
registry.Register("static", New)
}
const defaultPriority = 0
type mimeTypeConfig struct {
MimeType string `mapstructure:"mime_type"`
Extension string `mapstructure:"extension"`
Name string `mapstructure:"name"`
Description string `mapstructure:"description"`
Icon string `mapstructure:"icon"`
DefaultApp string `mapstructure:"default_app"`
AllowCreation bool `mapstructure:"allow_creation"`
apps providerHeap
}
type config struct {
Providers []*registrypb.ProviderInfo `mapstructure:"providers"`
MimeTypes []*mimeTypeConfig `mapstructure:"mime_types"`
}
func (c *config) init() {
if len(c.Providers) == 0 {
c.Providers = []*registrypb.ProviderInfo{}
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
type manager struct {
providers map[string]*registrypb.ProviderInfo
mimetypes *orderedmap.OrderedMap // map[string]*mimeTypeConfig -> map the mime type to the addresses of the corresponding providers
sync.RWMutex
}
// New returns an implementation of the app.Registry interface.
func New(m map[string]interface{}) (app.Registry, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
c.init()
mimetypes := orderedmap.New()
for _, mime := range c.MimeTypes {
mimetypes.Set(mime.MimeType, mime)
}
providerMap := make(map[string]*registrypb.ProviderInfo)
for _, p := range c.Providers {
providerMap[p.Address] = p
}
// register providers configured manually from the config
// (different from the others that are registering themselves -
// dinamically added invoking the AddProvider function)
for _, p := range c.Providers {
if p != nil {
for _, m := range p.MimeTypes {
if v, ok := mimetypes.Get(m); ok {
mtc := v.(*mimeTypeConfig)
registerProvider(p, mtc)
} else {
return nil, errtypes.NotFound(fmt.Sprintf("mimetype %s not found in the configuration", m))
}
}
}
}
newManager := manager{
providers: providerMap,
mimetypes: mimetypes,
}
return &newManager, nil
}
// remove a provider from the provider list in a mime type
// it's a no-op if the provider is not in the list of providers in the mime type
func unregisterProvider(p *registrypb.ProviderInfo, mime *mimeTypeConfig) {
if index, in := getIndex(mime.apps, p); in {
// remove the provider from the list
heap.Remove(&mime.apps, index)
}
}
func registerProvider(p *registrypb.ProviderInfo, mime *mimeTypeConfig) {
// the app provider could be previously registered to the same mime type list
// so we will remove it
unregisterProvider(p, mime)
heap.Push(&mime.apps, providerWithPriority{
provider: p,
priority: getPriority(p),
})
}
func getPriority(p *registrypb.ProviderInfo) uint64 {
if p.Opaque != nil && len(p.Opaque.Map) != 0 {
if priority, ok := p.Opaque.Map["priority"]; ok {
if pr, err := strconv.ParseUint(string(priority.GetValue()), 10, 64); err == nil {
return pr
}
}
}
return defaultPriority
}
func (m *manager) FindProviders(ctx context.Context, mimeType string) ([]*registrypb.ProviderInfo, error) {
// find longest match
var match string
m.RLock()
defer m.RUnlock()
for pair := m.mimetypes.Oldest(); pair != nil; pair = pair.Next() {
prefix := pair.Key.(string)
if strings.HasPrefix(mimeType, prefix) && len(prefix) > len(match) {
match = prefix
}
}
if match == "" {
return nil, errtypes.NotFound("application provider not found for mime type " + mimeType)
}
mimeInterface, _ := m.mimetypes.Get(match)
mimeMatch := mimeInterface.(*mimeTypeConfig)
var providers = make([]*registrypb.ProviderInfo, 0, len(mimeMatch.apps))
for _, p := range mimeMatch.apps {
providers = append(providers, m.providers[p.provider.Address])
}
return providers, nil
}
func (m *manager) AddProvider(ctx context.Context, p *registrypb.ProviderInfo) error {
m.Lock()
defer m.Unlock()
// check if the provider was already registered
// if it's the case, we have to unregister it
// from all the old mime types
if oldP, ok := m.providers[p.Address]; ok {
oldMimeTypes := oldP.MimeTypes
for _, mimeName := range oldMimeTypes {
mimeIf, ok := m.mimetypes.Get(mimeName)
if !ok {
continue
}
mime := mimeIf.(*mimeTypeConfig)
unregisterProvider(p, mime)
}
}
m.providers[p.Address] = p
for _, mime := range p.MimeTypes {
if mimeTypeInterface, ok := m.mimetypes.Get(mime); ok {
mimeType := mimeTypeInterface.(*mimeTypeConfig)
registerProvider(p, mimeType)
} else {
// the mime type should be already registered as config in the AppRegistry
// we will create a new entry fot the mimetype, but leaving a warning for
// future log inspection for weird behaviour
// log.Warn().Msgf("config for mimetype '%s' not found while adding a new AppProvider", m)
m.mimetypes.Set(mime, dummyMimeType(mime, []*registrypb.ProviderInfo{p}))
}
}
return nil
}
func (m *manager) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) {
m.RLock()
defer m.RUnlock()
providers := make([]*registrypb.ProviderInfo, 0, len(m.providers))
for _, p := range m.providers {
providers = append(providers, p)
}
return providers, nil
}
func (m *manager) ListSupportedMimeTypes(ctx context.Context) ([]*registrypb.MimeTypeInfo, error) {
m.RLock()
defer m.RUnlock()
res := make([]*registrypb.MimeTypeInfo, 0, m.mimetypes.Len())
for pair := m.mimetypes.Oldest(); pair != nil; pair = pair.Next() {
mime := pair.Value.(*mimeTypeConfig)
res = append(res, &registrypb.MimeTypeInfo{
MimeType: mime.MimeType,
Ext: mime.Extension,
Name: mime.Name,
Description: mime.Description,
Icon: mime.Icon,
AppProviders: mime.apps.getOrderedProviderByPriority(),
AllowCreation: mime.AllowCreation,
DefaultApplication: mime.DefaultApp,
})
}
return res, nil
}
func (h providerHeap) getOrderedProviderByPriority() []*registrypb.ProviderInfo {
providers := make([]*registrypb.ProviderInfo, 0, h.Len())
for _, pp := range h {
providers = append(providers, pp.provider)
}
return providers
}
func getIndex(h providerHeap, s *registrypb.ProviderInfo) (int, bool) {
for i, e := range h {
if equalsProviderInfo(e.provider, s) {
return i, true
}
}
return -1, false
}
func (m *manager) SetDefaultProviderForMimeType(ctx context.Context, mimeType string, p *registrypb.ProviderInfo) error {
m.Lock()
defer m.Unlock()
mimeInterface, ok := m.mimetypes.Get(mimeType)
if ok {
mime := mimeInterface.(*mimeTypeConfig)
mime.DefaultApp = p.Address
registerProvider(p, mime)
} else {
// the mime type should be already registered as config in the AppRegistry
// we will create a new entry fot the mimetype, but leaving a warning for
// future log inspection for weird behaviour
log.Warn().Msgf("config for mimetype '%s' not found while setting a new default AppProvider", mimeType)
m.mimetypes.Set(mimeType, dummyMimeType(mimeType, []*registrypb.ProviderInfo{p}))
}
return nil
}
func dummyMimeType(m string, apps []*registrypb.ProviderInfo) *mimeTypeConfig {
appsHeap := providerHeap{}
for _, p := range apps {
heap.Push(&appsHeap, providerWithPriority{
provider: p,
priority: getPriority(p),
})
}
return &mimeTypeConfig{
MimeType: m,
apps: appsHeap,
//Extension: "", // there is no meaningful general extension, so omit it
//Name: "", // there is no meaningful general name, so omit it
//Description: "", // there is no meaningful general description, so omit it
}
}
func (m *manager) GetDefaultProviderForMimeType(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) {
m.RLock()
defer m.RUnlock()
mimeInterface, ok := m.mimetypes.Get(mimeType)
if ok {
mime := mimeInterface.(*mimeTypeConfig)
// default by provider address
if p, ok := m.providers[mime.DefaultApp]; ok {
return p, nil
}
// default by provider name
for _, p := range m.providers {
if p.Name == mime.DefaultApp {
return p, nil
}
}
}
return nil, errtypes.NotFound("default application provider not set for mime type " + mimeType)
}
func equalsProviderInfo(p1, p2 *registrypb.ProviderInfo) bool {
return p1.Name == p2.Name
}
// check that all providers in the two lists are equals
func providersEquals(l1, l2 []*registrypb.ProviderInfo) bool {
if len(l1) != len(l2) {
return false
}
for i := 0; i < len(l1); i++ {
if !equalsProviderInfo(l1[i], l2[i]) {
return false
}
}
return true
}
type providerWithPriority struct {
provider *registrypb.ProviderInfo
priority uint64
}
type providerHeap []providerWithPriority
func (h providerHeap) Len() int {
return len(h)
}
func (h providerHeap) Less(i, j int) bool {
return h[i].priority > h[j].priority
}
func (h providerHeap) Swap(i, j int) {
h[i], h[j] = h[j], h[i]
}
func (h *providerHeap) Push(x interface{}) {
*h = append(*h, x.(providerWithPriority))
}
func (h *providerHeap) Pop() interface{} {
last := len(*h) - 1
x := (*h)[last]
*h = (*h)[:last]
return x
}
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package appauth
import (
"context"
apppb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
)
// Manager is the interface that manages application authentication mechanisms.
type Manager interface {
// GenerateAppPassword creates a password with specified scope to be used by
// third-party applications.
GenerateAppPassword(ctx context.Context, scope map[string]*authpb.Scope, label string, expiration *typespb.Timestamp) (*apppb.AppPassword, error)
// ListAppPasswords lists the application passwords created by a user.
ListAppPasswords(ctx context.Context) ([]*apppb.AppPassword, error)
// InvalidateAppPassword invalidates a generated password.
InvalidateAppPassword(ctx context.Context, secret string) error
// GetAppPassword retrieves the password information by the combination of username and password.
GetAppPassword(ctx context.Context, user *userpb.UserId, secret string) (*apppb.AppPassword, error)
}
+254
View File
@@ -0,0 +1,254 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package json
import (
"context"
"encoding/json"
"io"
"os"
"sync"
"time"
apppb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/appauth"
"github.com/cs3org/reva/v2/pkg/appauth/manager/registry"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/sethvargo/go-password/password"
"golang.org/x/crypto/bcrypt"
)
func init() {
registry.Register("json", New)
}
type config struct {
File string `mapstructure:"file"`
TokenStrength int `mapstructure:"token_strength"`
PasswordHashCost int `mapstructure:"password_hash_cost"`
}
type jsonManager struct {
sync.Mutex
config *config
// map[userid][password]AppPassword
passwords map[string]map[string]*apppb.AppPassword
}
// New returns a new mgr.
func New(m map[string]interface{}) (appauth.Manager, error) {
c, err := parseConfig(m)
if err != nil {
return nil, errors.Wrap(err, "error creating a new manager")
}
c.init()
// load or create file
manager, err := loadOrCreate(c.File)
if err != nil {
return nil, errors.Wrap(err, "error loading the file containing the application passwords")
}
manager.config = c
return manager, nil
}
func (c *config) init() {
if c.File == "" {
c.File = "/var/tmp/reva/appauth.json"
}
if c.TokenStrength == 0 {
c.TokenStrength = 16
}
if c.PasswordHashCost == 0 {
c.PasswordHashCost = 11
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
func loadOrCreate(file string) (*jsonManager, error) {
stat, err := os.Stat(file)
if os.IsNotExist(err) || stat.Size() == 0 {
if err = os.WriteFile(file, []byte("{}"), 0644); err != nil {
return nil, errors.Wrapf(err, "error creating the file %s", file)
}
}
fd, err := os.OpenFile(file, os.O_RDONLY, 0)
if err != nil {
return nil, errors.Wrapf(err, "error opening the file %s", file)
}
defer fd.Close()
data, err := io.ReadAll(fd)
if err != nil {
return nil, errors.Wrapf(err, "error reading the file %s", file)
}
m := &jsonManager{}
if err = json.Unmarshal(data, &m.passwords); err != nil {
return nil, errors.Wrapf(err, "error parsing the file %s", file)
}
if m.passwords == nil {
m.passwords = make(map[string]map[string]*apppb.AppPassword)
}
return m, nil
}
func (mgr *jsonManager) GenerateAppPassword(ctx context.Context, scope map[string]*authpb.Scope, label string, expiration *typespb.Timestamp) (*apppb.AppPassword, error) {
token, err := password.Generate(mgr.config.TokenStrength, mgr.config.TokenStrength/2, 0, false, false)
if err != nil {
return nil, errors.Wrap(err, "error creating new token")
}
tokenHashed, err := bcrypt.GenerateFromPassword([]byte(token), mgr.config.PasswordHashCost)
if err != nil {
return nil, errors.Wrap(err, "error creating new token")
}
userID := ctxpkg.ContextMustGetUser(ctx).GetId()
ctime := now()
password := string(tokenHashed)
appPass := &apppb.AppPassword{
Password: password,
TokenScope: scope,
Label: label,
Expiration: expiration,
Ctime: ctime,
Utime: ctime,
User: userID,
}
mgr.Lock()
defer mgr.Unlock()
// check if user has some previous password
if _, ok := mgr.passwords[userID.String()]; !ok {
mgr.passwords[userID.String()] = make(map[string]*apppb.AppPassword)
}
mgr.passwords[userID.String()][password] = appPass
err = mgr.save()
if err != nil {
return nil, errors.Wrap(err, "error saving new token")
}
clonedAppPass := *appPass
clonedAppPass.Password = token
return &clonedAppPass, nil
}
func (mgr *jsonManager) ListAppPasswords(ctx context.Context) ([]*apppb.AppPassword, error) {
userID := ctxpkg.ContextMustGetUser(ctx).GetId()
mgr.Lock()
defer mgr.Unlock()
appPasswords := []*apppb.AppPassword{}
for _, pw := range mgr.passwords[userID.String()] {
appPasswords = append(appPasswords, pw)
}
return appPasswords, nil
}
func (mgr *jsonManager) InvalidateAppPassword(ctx context.Context, password string) error {
userID := ctxpkg.ContextMustGetUser(ctx).GetId()
mgr.Lock()
defer mgr.Unlock()
// see if user has a list of passwords
appPasswords, ok := mgr.passwords[userID.String()]
if !ok || len(appPasswords) == 0 {
return errtypes.NotFound("password not found")
}
if _, ok := appPasswords[password]; !ok {
return errtypes.NotFound("password not found")
}
delete(mgr.passwords[userID.String()], password)
// if user has 0 passwords, delete user key from state map
if len(mgr.passwords[userID.String()]) == 0 {
delete(mgr.passwords, userID.String())
}
return mgr.save()
}
func (mgr *jsonManager) GetAppPassword(ctx context.Context, userID *userpb.UserId, password string) (*apppb.AppPassword, error) {
mgr.Lock()
defer mgr.Unlock()
appPassword, ok := mgr.passwords[userID.String()]
if !ok {
return nil, errtypes.NotFound("password not found")
}
for hash, pw := range appPassword {
err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password))
if err == nil {
// password found
if pw.Expiration != nil && pw.Expiration.Seconds != 0 && uint64(time.Now().Unix()) > pw.Expiration.Seconds {
// password expired
return nil, errtypes.NotFound("password not found")
}
// password not expired
// update last used time
pw.Utime = now()
if err := mgr.save(); err != nil {
return nil, errors.Wrap(err, "error saving file")
}
return pw, nil
}
}
return nil, errtypes.NotFound("password not found")
}
func now() *typespb.Timestamp {
return &typespb.Timestamp{Seconds: uint64(time.Now().Unix())}
}
func (mgr *jsonManager) save() error {
data, err := json.Marshal(mgr.passwords)
if err != nil {
return errors.Wrap(err, "error encoding json file")
}
if err = os.WriteFile(mgr.config.File, data, 0644); err != nil {
return errors.Wrapf(err, "error writing to file %s", mgr.config.File)
}
return nil
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core application auth manager drivers.
_ "github.com/cs3org/reva/v2/pkg/appauth/manager/json"
// Add your own here
)
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/appauth"
// NewFunc is the function that application auth implementations
// should register at init time.
type NewFunc func(map[string]interface{}) (appauth.Manager, error)
// NewFuncs is a map containing all the registered application auth managers.
var NewFuncs = map[string]NewFunc{}
// Register registers a new application auth manager new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package appctx
import (
"context"
rtrace "github.com/cs3org/reva/v2/pkg/trace"
"github.com/go-chi/chi/v5/middleware"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/trace"
)
// DeletingSharedResource flags to a storage a shared resource is being deleted not by the owner.
var DeletingSharedResource struct{}
// WithLogger returns a context with an associated logger.
func WithLogger(ctx context.Context, l *zerolog.Logger) context.Context {
return l.WithContext(ctx)
}
// GetLogger returns the logger associated with the given context
// or a disabled logger in case no logger is stored inside the context.
func GetLogger(ctx context.Context) *zerolog.Logger {
logger := zerolog.Ctx(ctx)
reqID := middleware.GetReqID(ctx)
if reqID != "" {
sublogger := logger.With().Str("request-id", reqID).Logger()
logger = &sublogger
}
return logger
}
// WithTracerProvider returns a context with an associated TracerProvider
func WithTracerProvider(ctx context.Context, p trace.TracerProvider) context.Context {
return rtrace.ContextSetTracerProvider(ctx, p)
}
// GetTracerProvider returns the TracerProvider associated with
// the given context. (Or the global default TracerProvider if there
// is no TracerProvider in the context)
func GetTracerProvider(ctx context.Context) trace.TracerProvider {
return rtrace.ContextGetTracerProvider(ctx)
}
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package appctx
import (
"context"
"reflect"
"unsafe"
)
// PutKeyValuesToCtx puts all the key-value pairs from the provided map to a background context.
func PutKeyValuesToCtx(m map[interface{}]interface{}) context.Context {
ctx := context.Background()
for key, value := range m {
ctx = context.WithValue(ctx, key, value)
}
return ctx
}
// GetKeyValuesFromCtx retrieves all the key-value pairs from the provided context.
func GetKeyValuesFromCtx(ctx context.Context) map[interface{}]interface{} {
m := make(map[interface{}]interface{})
getKeyValue(ctx, m)
return m
}
func getKeyValue(ctx interface{}, m map[interface{}]interface{}) {
ctxVals := reflect.ValueOf(ctx).Elem()
ctxType := reflect.TypeOf(ctx).Elem()
if ctxType.Kind() == reflect.Struct {
for i := 0; i < ctxVals.NumField(); i++ {
currField, currIf := extractField(ctxVals, ctxType, i)
switch currField {
case "Context":
getKeyValue(currIf, m)
case "key":
nextField, nextIf := extractField(ctxVals, ctxType, i+1)
if nextField == "val" {
m[currIf] = nextIf
i++
}
}
}
}
}
func extractField(vals reflect.Value, fieldType reflect.Type, pos int) (string, interface{}) {
currVal := vals.Field(pos)
currVal = reflect.NewAt(currVal.Type(), unsafe.Pointer(currVal.UnsafeAddr())).Elem()
currField := fieldType.Field(pos)
return currField.Name, currVal.Interface()
}
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package auth
import (
"context"
"net/http"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
registry "github.com/cs3org/go-cs3apis/cs3/auth/registry/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/plugin"
)
// Manager is the interface to implement to authenticate users
type Manager interface {
plugin.Plugin
Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error)
}
// Credentials contains the auth type, client id and secret.
type Credentials struct {
Type string
ClientID string
ClientSecret string
}
// CredentialStrategy obtains Credentials from the request.
type CredentialStrategy interface {
GetCredentials(w http.ResponseWriter, r *http.Request) (*Credentials, error)
AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, realm string)
}
// TokenStrategy obtains a token from the request.
// If token does not exist returns an empty string.
type TokenStrategy interface {
GetToken(r *http.Request) string
}
// TokenWriter stores the token in a http response.
type TokenWriter interface {
WriteToken(token string, w http.ResponseWriter)
}
// Registry is the interface that auth registries implement
// for discovering auth providers
type Registry interface {
ListProviders(ctx context.Context) ([]*registry.ProviderInfo, error)
GetProvider(ctx context.Context, authType string) (*registry.ProviderInfo, error)
}
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package appauth
import (
"context"
appauthpb "github.com/cs3org/go-cs3apis/cs3/auth/applications/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("appauth", New)
}
type manager struct {
GatewayAddr string `mapstructure:"gateway_addr"`
}
// New returns a new auth Manager.
func New(m map[string]interface{}) (auth.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
return nil, err
}
return mgr, nil
}
func (m *manager) Configure(ml map[string]interface{}) error {
err := mapstructure.Decode(ml, m)
if err != nil {
return errors.Wrap(err, "error decoding conf")
}
return nil
}
func (m *manager) Authenticate(ctx context.Context, username, password string) (*user.User, map[string]*authpb.Scope, error) {
gtw, err := pool.GetGatewayServiceClient(m.GatewayAddr)
if err != nil {
return nil, nil, err
}
// get user info
userResponse, err := gtw.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
Claim: "username",
Value: username,
})
switch {
case err != nil:
return nil, nil, err
case userResponse.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND:
return nil, nil, errtypes.NotFound(userResponse.Status.Message)
case userResponse.Status.Code != rpcv1beta1.Code_CODE_OK:
return nil, nil, errtypes.InternalError(userResponse.Status.Message)
}
// get the app password associated with the user and password
appAuthResponse, err := gtw.GetAppPassword(ctx, &appauthpb.GetAppPasswordRequest{
User: userResponse.GetUser().Id,
Password: password,
})
switch {
case err != nil:
return nil, nil, err
case appAuthResponse.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND:
return nil, nil, errtypes.NotFound(appAuthResponse.Status.Message)
case appAuthResponse.Status.Code != rpcv1beta1.Code_CODE_OK:
return nil, nil, errtypes.InternalError(appAuthResponse.Status.Message)
}
return userResponse.GetUser(), appAuthResponse.GetAppPassword().TokenScope, nil
}
+127
View File
@@ -0,0 +1,127 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package demo
import (
"context"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
)
func init() {
registry.Register("demo", New)
}
type manager struct {
credentials map[string]Credentials
}
// Credentials holds a pair of secret and userid
type Credentials struct {
User *user.User
Secret string
}
// New returns a new auth Manager.
func New(m map[string]interface{}) (auth.Manager, error) {
// m not used
mgr := &manager{}
err := mgr.Configure(m)
return mgr, err
}
func (m *manager) Configure(ml map[string]interface{}) error {
creds := getCredentials()
m.credentials = creds
return nil
}
func (m *manager) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
if c, ok := m.credentials[clientID]; ok {
if c.Secret == clientSecret {
var scopes map[string]*authpb.Scope
var err error
if c.User.Id != nil && (c.User.Id.Type == user.UserType_USER_TYPE_LIGHTWEIGHT || c.User.Id.Type == user.UserType_USER_TYPE_FEDERATED) {
scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
if err != nil {
return nil, nil, err
}
} else {
scopes, err = scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
}
return c.User, scopes, nil
}
}
return nil, nil, errtypes.InvalidCredentials(clientID)
}
func getCredentials() map[string]Credentials {
return map[string]Credentials{
"einstein": {
Secret: "relativity",
User: &user.User{
Id: &user.UserId{
Idp: "http://localhost:9998",
OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51",
Type: user.UserType_USER_TYPE_PRIMARY,
},
Username: "einstein",
Groups: []string{"sailing-lovers", "violin-haters", "physics-lovers"},
Mail: "einstein@example.org",
DisplayName: "Albert Einstein",
},
},
"marie": {
Secret: "radioactivity",
User: &user.User{
Id: &user.UserId{
Idp: "http://localhost:9998",
OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c",
Type: user.UserType_USER_TYPE_PRIMARY,
},
Username: "marie",
Groups: []string{"radium-lovers", "polonium-lovers", "physics-lovers"},
Mail: "marie@example.org",
DisplayName: "Marie Curie",
},
},
"richard": {
Secret: "superfluidity",
User: &user.User{
Id: &user.UserId{
Idp: "http://localhost:9998",
OpaqueId: "932b4540-8d16-481e-8ef4-588e4b6b151c",
Type: user.UserType_USER_TYPE_PRIMARY,
},
Username: "richard",
Groups: []string{"quantum-lovers", "philosophy-haters", "physics-lovers"},
Mail: "richard@example.org",
DisplayName: "Richard Feynman",
},
},
}
}
@@ -0,0 +1,67 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package impersonator
import (
"context"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
)
func init() {
registry.Register("impersonator", New)
}
type mgr struct{}
// New returns an auth manager implementation that allows to authenticate with any credentials.
func New(c map[string]interface{}) (auth.Manager, error) {
return &mgr{}, nil
}
func (m *mgr) Configure(ml map[string]interface{}) error {
return nil
}
func (m *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
// allow passing in uid as <opaqueid>@<idp>
at := strings.LastIndex(clientID, "@")
uid := &user.UserId{Type: user.UserType_USER_TYPE_PRIMARY}
if at < 0 {
uid.OpaqueId = clientID
} else {
uid.OpaqueId = clientID[:at]
uid.Idp = clientID[at+1:]
}
scope, err := scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
return &user.User{
Id: uid,
// not much else to provide
}, scope, nil
}
+146
View File
@@ -0,0 +1,146 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package json
import (
"context"
"encoding/json"
"os"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("json", New)
}
// Credentials holds a pair of secret and userid
type Credentials struct {
ID *user.UserId `mapstructure:"id" json:"id"`
Username string `mapstructure:"username" json:"username"`
Mail string `mapstructure:"mail" json:"mail"`
MailVerified bool `mapstructure:"mail_verified" json:"mail_verified"`
DisplayName string `mapstructure:"display_name" json:"display_name"`
Secret string `mapstructure:"secret" json:"secret"`
Groups []string `mapstructure:"groups" json:"groups"`
UIDNumber int64 `mapstructure:"uid_number" json:"uid_number"`
GIDNumber int64 `mapstructure:"gid_number" json:"gid_number"`
Opaque *typespb.Opaque `mapstructure:"opaque" json:"opaque"`
}
type manager struct {
credentials map[string]*Credentials
}
type config struct {
// Users holds a path to a file containing json conforming the Users struct
Users string `mapstructure:"users"`
}
func (c *config) init() {
if c.Users == "" {
c.Users = "/etc/revad/users.json"
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
c.init()
return c, nil
}
// New returns a new auth Manager.
func New(m map[string]interface{}) (auth.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
return nil, err
}
return mgr, nil
}
func (m *manager) Configure(ml map[string]interface{}) error {
c, err := parseConfig(ml)
if err != nil {
return err
}
m.credentials = map[string]*Credentials{}
f, err := os.ReadFile(c.Users)
if err != nil {
return err
}
credentials := []*Credentials{}
err = json.Unmarshal(f, &credentials)
if err != nil {
return err
}
for _, c := range credentials {
m.credentials[c.Username] = c
}
return nil
}
func (m *manager) Authenticate(ctx context.Context, username string, secret string) (*user.User, map[string]*authpb.Scope, error) {
if c, ok := m.credentials[username]; ok {
if c.Secret == secret {
var scopes map[string]*authpb.Scope
var err error
if c.ID != nil && (c.ID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT || c.ID.Type == user.UserType_USER_TYPE_FEDERATED) {
scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
if err != nil {
return nil, nil, err
}
} else {
scopes, err = scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
}
return &user.User{
Id: c.ID,
Username: c.Username,
Mail: c.Mail,
MailVerified: c.MailVerified,
DisplayName: c.DisplayName,
Groups: c.Groups,
UidNumber: c.UIDNumber,
GidNumber: c.GIDNumber,
Opaque: c.Opaque,
// TODO add arbitrary keys as opaque data
}, scopes, nil
}
}
return nil, nil, errtypes.InvalidCredentials(username)
}
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ldap
import (
"context"
"fmt"
"strconv"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/sharedconf"
"github.com/cs3org/reva/v2/pkg/utils"
ldapIdentity "github.com/cs3org/reva/v2/pkg/utils/ldap"
"github.com/go-ldap/ldap/v3"
"github.com/google/uuid"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("ldap", New)
}
type mgr struct {
c *config
ldapClient ldap.Client
}
type config struct {
utils.LDAPConn `mapstructure:",squash"`
LDAPIdentity ldapIdentity.Identity `mapstructure:",squash"`
Idp string `mapstructure:"idp"`
GatewaySvc string `mapstructure:"gatewaysvc"`
Nobody int64 `mapstructure:"nobody"`
LoginAttributes []string `mapstructure:"login_attributes"`
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{
LDAPIdentity: ldapIdentity.New(),
LoginAttributes: []string{"cn"},
}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns an auth manager implementation that connects to a LDAP server to validate the user.
func New(m map[string]interface{}) (auth.Manager, error) {
manager := &mgr{}
err := manager.Configure(m)
if err != nil {
return nil, err
}
manager.ldapClient, err = utils.GetLDAPClientWithReconnect(&manager.c.LDAPConn)
if err != nil {
return nil, err
}
return manager, nil
}
func (am *mgr) Configure(m map[string]interface{}) error {
c, err := parseConfig(m)
if err != nil {
return err
}
if c.Nobody == 0 {
c.Nobody = 99
}
if err = c.LDAPIdentity.Setup(); err != nil {
return fmt.Errorf("error setting up Identity config: %w", err)
}
c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
am.c = c
return nil
}
func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
log := appctx.GetLogger(ctx)
filter := am.getLoginFilter(clientID)
userEntry, err := am.c.LDAPIdentity.GetLDAPUserByFilter(log, am.ldapClient, filter)
if err != nil {
return nil, nil, err
}
// Bind as the user to verify their password
la, err := utils.GetLDAPClientForAuth(&am.c.LDAPConn)
if err != nil {
return nil, nil, err
}
defer la.Close()
err = la.Bind(userEntry.DN, clientSecret)
switch {
case err == nil:
break
case ldap.IsErrorWithCode(err, ldap.LDAPResultInvalidCredentials):
return nil, nil, errtypes.InvalidCredentials(clientID)
default:
log.Debug().Err(err).Interface("userdn", userEntry.DN).Msg("bind with user credentials failed")
return nil, nil, err
}
var uid string
if am.c.LDAPIdentity.User.Schema.IDIsOctetString {
rawValue := userEntry.GetEqualFoldRawAttributeValue(am.c.LDAPIdentity.User.Schema.ID)
if value, err := uuid.FromBytes(rawValue); err == nil {
uid = value.String()
}
} else {
uid = userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.ID)
}
userID := &user.UserId{
Idp: am.c.Idp,
OpaqueId: uid,
Type: am.c.LDAPIdentity.GetUserType(userEntry),
}
gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc)
if err != nil {
return nil, nil, errors.Wrap(err, "ldap: error getting gateway grpc client")
}
getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{
UserId: userID,
})
if err != nil {
log.Warn().Err(err).Msg("error getting user groups")
return nil, nil, errors.Wrap(err, "ldap: error getting user groups")
}
if getGroupsResp.Status.Code != rpc.Code_CODE_OK {
log.Warn().Err(err).Str("msg", getGroupsResp.Status.Message).Msg("grpc getting user groups failed")
return nil, nil, fmt.Errorf("ldap: grpc getting user groups failed: '%s'", getGroupsResp.Status.Message)
}
gidNumber := am.c.Nobody
gidValue := userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.GIDNumber)
if gidValue != "" {
gidNumber, err = strconv.ParseInt(gidValue, 10, 64)
if err != nil {
return nil, nil, err
}
}
uidNumber := am.c.Nobody
uidValue := userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.UIDNumber)
if uidValue != "" {
uidNumber, err = strconv.ParseInt(uidValue, 10, 64)
if err != nil {
return nil, nil, err
}
}
u := &user.User{
Id: userID,
// TODO add more claims from the StandardClaims, eg EmailVerified
Username: userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.Username),
// TODO groups
Groups: getGroupsResp.Groups,
Mail: userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.Mail),
DisplayName: userEntry.GetEqualFoldAttributeValue(am.c.LDAPIdentity.User.Schema.DisplayName),
UidNumber: uidNumber,
GidNumber: gidNumber,
}
var scopes map[string]*authpb.Scope
if userID != nil && userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT {
scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
if err != nil {
return nil, nil, err
}
} else {
scopes, err = scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
}
log.Debug().Interface("entry", userEntry).Interface("user", u).Msg("authenticated user")
return u, scopes, nil
}
func (am *mgr) getLoginFilter(login string) string {
var filter string
for _, attr := range am.c.LoginAttributes {
filter = fmt.Sprintf("%s(%s=%s)", filter, attr, ldap.EscapeFilter(login))
}
return fmt.Sprintf("(&%s(objectclass=%s)(|%s))",
am.c.LDAPIdentity.User.Filter,
am.c.LDAPIdentity.User.Objectclass,
filter,
)
}
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core authentication managers.
_ "github.com/cs3org/reva/v2/pkg/auth/manager/appauth"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/demo"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/impersonator"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/json"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/ldap"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/machine"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/nextcloud"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/oidc"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/owncloudsql"
_ "github.com/cs3org/reva/v2/pkg/auth/manager/publicshares"
// Add your own here
)
+125
View File
@@ -0,0 +1,125 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package machine
import (
"context"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
// 'machine' is an authentication method used to impersonate users.
// To impersonate the given user it's only needed an api-key, saved
// in a config file.
// supported claims
var claims = []string{"mail", "uid", "username", "gid", "userid"}
type manager struct {
APIKey string `mapstructure:"api_key"`
GatewayAddr string `mapstructure:"gateway_addr"`
}
func init() {
registry.Register("machine", New)
}
// Configure parses the map conf
func (m *manager) Configure(conf map[string]interface{}) error {
err := mapstructure.Decode(conf, m)
if err != nil {
return errors.Wrap(err, "error decoding conf")
}
return nil
}
// New creates a new manager for the 'machine' authentication
func New(conf map[string]interface{}) (auth.Manager, error) {
m := &manager{}
err := m.Configure(conf)
if err != nil {
return nil, err
}
return m, nil
}
// Authenticate impersonate an user if the provided secret is equal to the api-key
func (m *manager) Authenticate(ctx context.Context, user, secret string) (*userpb.User, map[string]*authpb.Scope, error) {
if m.APIKey != secret {
return nil, nil, errtypes.InvalidCredentials("")
}
gtw, err := pool.GetGatewayServiceClient(m.GatewayAddr)
if err != nil {
return nil, nil, err
}
// username could be either a normal username or a string <claim>:<value>
// in the first case the claim is "username"
claim, value := parseUser(user)
userResponse, err := gtw.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{
Claim: claim,
Value: value,
})
switch {
case err != nil:
return nil, nil, err
case userResponse.Status.Code == rpc.Code_CODE_NOT_FOUND:
return nil, nil, errtypes.NotFound(userResponse.Status.Message)
case userResponse.Status.Code != rpc.Code_CODE_OK:
return nil, nil, errtypes.InternalError(userResponse.Status.Message)
}
scope, err := scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
return userResponse.GetUser(), scope, nil
}
func contains(lst []string, s string) bool {
for _, e := range lst {
if e == s {
return true
}
}
return false
}
func parseUser(user string) (string, string) {
s := strings.SplitN(user, ":", 2)
if len(s) == 2 && contains(claims, s[0]) {
return s[0], s[1]
}
return "username", user
}
@@ -0,0 +1,197 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package nextcloud verifies a clientID and clientSecret against a Nextcloud backend.
package nextcloud
import (
"context"
"encoding/json"
"io"
"net/http"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("nextcloud", New)
}
// Manager is the Nextcloud-based implementation of the auth.Manager interface
// see https://github.com/cs3org/reva/blob/v1.13.0/pkg/auth/auth.go#L32-L35
type Manager struct {
client *http.Client
sharedSecret string
endPoint string
}
// AuthManagerConfig contains config for a Nextcloud-based AuthManager
type AuthManagerConfig struct {
EndPoint string `mapstructure:"endpoint" docs:";The Nextcloud backend endpoint for user check"`
SharedSecret string `mapstructure:"shared_secret"`
MockHTTP bool `mapstructure:"mock_http"`
}
// Action describes a REST request to forward to the Nextcloud backend
type Action struct {
verb string
username string
argS string
}
func (c *AuthManagerConfig) init() {
}
func parseConfig(m map[string]interface{}) (*AuthManagerConfig, error) {
c := &AuthManagerConfig{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns an auth manager implementation that verifies against a Nextcloud backend.
func New(m map[string]interface{}) (auth.Manager, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
c.init()
return NewAuthManager(c)
}
// NewAuthManager returns a new Nextcloud-based AuthManager
func NewAuthManager(c *AuthManagerConfig) (*Manager, error) {
var client *http.Client
if c.MockHTTP {
// called := make([]string, 0)
// nextcloudServerMock := GetNextcloudServerMock(&called)
// client, _ = TestingHTTPClient(nextcloudServerMock)
// Wait for SetHTTPClient to be called later
client = nil
} else {
if len(c.EndPoint) == 0 {
return nil, errors.New("Please specify 'endpoint' in '[grpc.services.authprovider.auth_managers.nextcloud]'")
}
client = &http.Client{}
}
return &Manager{
endPoint: c.EndPoint, // e.g. "http://nc/apps/sciencemesh/"
sharedSecret: c.SharedSecret,
client: client,
}, nil
}
// Configure method as defined in https://github.com/cs3org/reva/blob/v1.13.0/pkg/auth/auth.go#L32-L35
func (am *Manager) Configure(ml map[string]interface{}) error {
return nil
}
// SetHTTPClient sets the HTTP client
func (am *Manager) SetHTTPClient(c *http.Client) {
am.client = c
}
func (am *Manager) do(ctx context.Context, a Action) (int, []byte, error) {
log := appctx.GetLogger(ctx)
url := am.endPoint + "~" + a.username + "/api/auth/" + a.verb
log.Info().Msgf("am.do %s %s %s", url, a.argS, am.sharedSecret)
req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(a.argS))
if err != nil {
return 0, nil, err
}
req.Header.Set("X-Reva-Secret", am.sharedSecret)
req.Header.Set("Content-Type", "application/json")
resp, err := am.client.Do(req)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return 0, nil, err
}
log.Info().Msgf("am.do response %d %s", resp.StatusCode, body)
return resp.StatusCode, body, nil
}
// Authenticate method as defined in https://github.com/cs3org/reva/blob/28500a8/pkg/auth/auth.go#L31-L33
func (am *Manager) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
type paramsObj struct {
ClientID string `json:"clientID"`
ClientSecret string `json:"clientSecret"`
// Scope authpb.Scope
}
bodyObj := &paramsObj{
ClientID: clientID,
ClientSecret: clientSecret,
// Scope: authpb.Scope{
// Resource: &types.OpaqueEntry{
// Decoder: "json",
// Value: []byte(`{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"some/file/path.txt"}`),
// },
// Role: authpb.Role_ROLE_OWNER,
// },
}
bodyStr, err := json.Marshal(bodyObj)
if err != nil {
return nil, nil, err
}
log := appctx.GetLogger(ctx)
log.Info().Msgf("Authenticate %s %s", clientID, bodyStr)
statusCode, body, err := am.do(ctx, Action{"Authenticate", clientID, string(bodyStr)})
if err != nil {
return nil, nil, err
}
if statusCode != 200 {
return nil, nil, errors.New("Username/password not recognized by Nextcloud backend")
}
type resultsObj struct {
User user.User `json:"user"`
Scopes map[string]authpb.Scope `json:"scopes"`
}
result := &resultsObj{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, nil, err
}
var pointersMap = make(map[string]*authpb.Scope)
for k := range result.Scopes {
scope := result.Scopes[k]
pointersMap[k] = &scope
}
return &result.User, pointersMap, nil
}
@@ -0,0 +1,99 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package nextcloud
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
)
// Response contains data for the Nextcloud mock server to respond
// and to switch to a new server state
type Response struct {
code int
body string
newServerState string
}
const serverStateError = "ERROR"
const serverStateEmpty = "EMPTY"
const serverStateHome = "HOME"
var serverState = serverStateEmpty
var responses = map[string]Response{
`POST /apps/sciencemesh/~einstein/api/auth/Authenticate {"clientID":"einstein","clientSecret":"relativity"}`: {200, `{"user":{"id":{"idp":"some-idp","opaque_id":"some-opaque-user-id","type":1}},"scopes":{"user":{"resource":{"decoder":"json","value":"eyJyZXNvdXJjZV9pZCI6eyJzdG9yYWdlX2lkIjoic3RvcmFnZS1pZCIsIm9wYXF1ZV9pZCI6Im9wYXF1ZS1pZCJ9LCJwYXRoIjoic29tZS9maWxlL3BhdGgudHh0In0="},"role":1}}}`, serverStateHome},
}
// GetNextcloudServerMock returns a handler that pretends to be a remote Nextcloud server
func GetNextcloudServerMock(called *[]string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buf := new(strings.Builder)
_, err := io.Copy(buf, r.Body)
if err != nil {
panic("Error reading response into buffer")
}
var key = fmt.Sprintf("%s %s %s", r.Method, r.URL, buf.String())
*called = append(*called, key)
response := responses[key]
if (response == Response{}) {
key = fmt.Sprintf("%s %s %s %s", r.Method, r.URL, buf.String(), serverState)
response = responses[key]
}
if (response == Response{}) {
fmt.Printf("%s %s %s %s", r.Method, r.URL, buf.String(), serverState)
response = Response{500, fmt.Sprintf("response not defined! %s", key), serverStateEmpty}
}
serverState = responses[key].newServerState
if serverState == `` {
serverState = serverStateError
}
w.WriteHeader(response.code)
// w.Header().Set("Etag", "mocker-etag")
_, err = w.Write([]byte(responses[key].body))
if err != nil {
panic(err)
}
})
}
// TestingHTTPClient thanks to https://itnext.io/how-to-stub-requests-to-remote-hosts-with-go-6c2c1db32bf2
// Ideally, this function would live in tests/helpers, but
// if we put it there, it gets excluded by .dockerignore, and the
// Docker build fails (see https://github.com/cs3org/reva/issues/1999)
// So putting it here for now - open to suggestions if someone knows
// a better way to inject this.
func TestingHTTPClient(handler http.Handler) (*http.Client, func()) {
s := httptest.NewServer(handler)
cli := &http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, network, _ string) (net.Conn, error) {
return net.Dial(network, s.Listener.Addr().String())
},
},
}
return cli, s.Close
}
+363
View File
@@ -0,0 +1,363 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package oidc verifies an OIDC token against the configured OIDC provider
// and obtains the necessary claims to obtain user information.
package oidc
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
oidc "github.com/coreos/go-oidc"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/rgrpc/status"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/cs3org/reva/v2/pkg/sharedconf"
"github.com/juliangruber/go-intersect"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
func init() {
registry.Register("oidc", New)
}
type mgr struct {
provider *oidc.Provider // cached on first request
c *config
oidcUsersMapping map[string]*oidcUserMapping
}
type config struct {
Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."`
Issuer string `mapstructure:"issuer" docs:";The issuer of the OIDC token."`
IDClaim string `mapstructure:"id_claim" docs:"sub;The claim containing the ID of the user."`
UIDClaim string `mapstructure:"uid_claim" docs:";The claim containing the UID of the user."`
GIDClaim string `mapstructure:"gid_claim" docs:";The claim containing the GID of the user."`
GatewaySvc string `mapstructure:"gatewaysvc" docs:";The endpoint at which the GRPC gateway is exposed."`
UsersMapping string `mapstructure:"users_mapping" docs:"; The optional OIDC users mapping file path"`
GroupClaim string `mapstructure:"group_claim" docs:"; The group claim to be looked up to map the user (default to 'groups')."`
}
type oidcUserMapping struct {
OIDCIssuer string `mapstructure:"oidc_issuer" json:"oidc_issuer"`
OIDCGroup string `mapstructure:"oidc_group" json:"oidc_group"`
Username string `mapstructure:"username" json:"username"`
}
func (c *config) init() {
if c.IDClaim == "" {
// sub is stable and defined as unique. the user manager needs to take care of the sub to user metadata lookup
c.IDClaim = "sub"
}
if c.GroupClaim == "" {
c.GroupClaim = "groups"
}
if c.UIDClaim == "" {
c.UIDClaim = "uid"
}
if c.GIDClaim == "" {
c.GIDClaim = "gid"
}
c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns an auth manager implementation that verifies the oidc token and obtains the user claims.
func New(m map[string]interface{}) (auth.Manager, error) {
manager := &mgr{}
err := manager.Configure(m)
if err != nil {
return nil, err
}
return manager, nil
}
func (am *mgr) Configure(m map[string]interface{}) error {
c, err := parseConfig(m)
if err != nil {
return err
}
c.init()
am.c = c
am.oidcUsersMapping = map[string]*oidcUserMapping{}
if c.UsersMapping == "" {
// no mapping defined, leave the map empty and move on
return nil
}
f, err := os.ReadFile(c.UsersMapping)
if err != nil {
return fmt.Errorf("oidc: error reading the users mapping file: +%v", err)
}
oidcUsers := []*oidcUserMapping{}
err = json.Unmarshal(f, &oidcUsers)
if err != nil {
return fmt.Errorf("oidc: error unmarshalling the users mapping file: +%v", err)
}
for _, u := range oidcUsers {
if _, found := am.oidcUsersMapping[u.OIDCGroup]; found {
return fmt.Errorf("oidc: mapping error, group \"%s\" is mapped to multiple users", u.OIDCGroup)
}
am.oidcUsersMapping[u.OIDCGroup] = u
}
return nil
}
// The clientID would be empty as we only need to validate the clientSecret variable
// which contains the access token that we can use to contact the UserInfo endpoint
// and get the user claims.
func (am *mgr) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
ctx = am.getOAuthCtx(ctx)
log := appctx.GetLogger(ctx)
oidcProvider, err := am.getOIDCProvider(ctx)
if err != nil {
return nil, nil, fmt.Errorf("oidc: error creating oidc provider: +%v", err)
}
oauth2Token := &oauth2.Token{
AccessToken: clientSecret,
}
// query the oidc provider for user info
userInfo, err := oidcProvider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token))
if err != nil {
return nil, nil, fmt.Errorf("oidc: error getting userinfo: +%v", err)
}
// claims contains the standard OIDC claims like iss, iat, aud, ... and any other non-standard one.
// TODO(labkode): make claims configuration dynamic from the config file so we can add arbitrary mappings from claims to user struct.
// For now, only the group claim is dynamic.
// TODO(labkode): may do like K8s does it: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go
var claims map[string]interface{}
if err := userInfo.Claims(&claims); err != nil {
return nil, nil, fmt.Errorf("oidc: error unmarshaling userinfo claims: %v", err)
}
log.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Msg("unmarshalled userinfo")
if claims["iss"] == nil { // This is not set in simplesamlphp
claims["iss"] = am.c.Issuer
}
if claims["email_verified"] == nil { // This is not set in simplesamlphp
claims["email_verified"] = false
}
if claims["preferred_username"] == nil {
claims["preferred_username"] = claims[am.c.IDClaim]
}
if claims["preferred_username"] == nil {
claims["preferred_username"] = claims["email"]
}
if claims["name"] == nil {
claims["name"] = claims[am.c.IDClaim]
}
if claims["name"] == nil {
return nil, nil, fmt.Errorf("no \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope")
}
if claims["email"] == nil {
return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope")
}
uid, _ := claims[am.c.UIDClaim].(float64)
claims[am.c.UIDClaim] = int64(uid) // in case the uid claim is missing and a mapping is to be performed, resolveUser() will populate it
// Note that if not, will silently carry a user with 0 uid, potentially problematic with storage providers
gid, _ := claims[am.c.GIDClaim].(float64)
claims[am.c.GIDClaim] = int64(gid)
err = am.resolveUser(ctx, claims)
if err != nil {
return nil, nil, errors.Wrapf(err, "oidc: error resolving username for external user '%v'", claims["email"])
}
userID := &user.UserId{
OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id
Idp: claims["iss"].(string), // in the scope of this issuer
Type: getUserType(claims[am.c.IDClaim].(string)),
}
gwc, err := pool.GetGatewayServiceClient(am.c.GatewaySvc)
if err != nil {
return nil, nil, errors.Wrap(err, "oidc: error getting gateway grpc client")
}
getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{
UserId: userID,
})
if err != nil {
return nil, nil, errors.Wrapf(err, "oidc: error getting user groups for '%+v'", userID)
}
if getGroupsResp.Status.Code != rpc.Code_CODE_OK {
return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidc")
}
u := &user.User{
Id: userID,
Username: claims["preferred_username"].(string),
Groups: getGroupsResp.Groups,
Mail: claims["email"].(string),
MailVerified: claims["email_verified"].(bool),
DisplayName: claims["name"].(string),
UidNumber: claims[am.c.UIDClaim].(int64),
GidNumber: claims[am.c.GIDClaim].(int64),
}
var scopes map[string]*authpb.Scope
if userID != nil && (userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT || userID.Type == user.UserType_USER_TYPE_FEDERATED) {
scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
if err != nil {
return nil, nil, err
}
} else {
scopes, err = scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
}
return u, scopes, nil
}
func (am *mgr) getOAuthCtx(ctx context.Context) context.Context {
// Sometimes for testing we need to skip the TLS check, that's why we need a
// custom HTTP client.
customHTTPClient := rhttp.GetHTTPClient(
rhttp.Context(ctx),
rhttp.Timeout(time.Second*10),
rhttp.Insecure(am.c.Insecure),
// Fixes connection fd leak which might be caused by provider-caching
rhttp.DisableKeepAlive(true),
)
ctx = context.WithValue(ctx, oauth2.HTTPClient, customHTTPClient)
return ctx
}
// getOIDCProvider returns a singleton OIDC provider
func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) {
ctx = am.getOAuthCtx(ctx)
log := appctx.GetLogger(ctx)
if am.provider != nil {
return am.provider, nil
}
// Initialize a provider by specifying the issuer URL.
// Once initialized this is a singleton that is reused for further requests.
// The provider is responsible to verify the token sent by the client
// against the security keys oftentimes available in the .well-known endpoint.
provider, err := oidc.NewProvider(ctx, am.c.Issuer)
if err != nil {
log.Error().Err(err).Msg("oidc: error creating a new oidc provider")
return nil, fmt.Errorf("oidc: error creating a new oidc provider: %+v", err)
}
am.provider = provider
return am.provider, nil
}
func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}) error {
if len(am.oidcUsersMapping) > 0 {
var username string
// map and discover the user's username when a mapping is defined
if claims[am.c.GroupClaim] == nil {
// we are required to perform a user mapping but the group claim is not available
return fmt.Errorf("no \"%s\" claim found in userinfo to map user", am.c.GroupClaim)
}
mappings := make([]string, 0, len(am.oidcUsersMapping))
for _, m := range am.oidcUsersMapping {
if m.OIDCIssuer == claims["iss"] {
mappings = append(mappings, m.OIDCGroup)
}
}
intersection := intersect.Simple(claims[am.c.GroupClaim], mappings)
if len(intersection) > 1 {
// multiple mappings are not implemented as we cannot decide which one to choose
return errtypes.PermissionDenied("more than one user mapping entry exists for the given group claims")
}
if len(intersection) == 0 {
return errtypes.PermissionDenied("no user mapping found for the given group claim(s)")
}
for _, m := range intersection {
username = am.oidcUsersMapping[m.(string)].Username
}
upsc, err := pool.GetUserProviderServiceClient(am.c.GatewaySvc)
if err != nil {
return errors.Wrap(err, "error getting user provider grpc client")
}
getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{
Claim: "username",
Value: username,
})
if err != nil {
return errors.Wrapf(err, "error getting user by username '%v'", username)
}
if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK {
return status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidc")
}
// take the properties of the mapped target user to override the claims
claims["preferred_username"] = username
claims[am.c.IDClaim] = getUserByClaimResp.GetUser().GetId().OpaqueId
claims["iss"] = getUserByClaimResp.GetUser().GetId().Idp
claims[am.c.UIDClaim] = getUserByClaimResp.GetUser().UidNumber
claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber
appctx.GetLogger(ctx).Debug().Str("username", username).Interface("claims", claims).Msg("resolveUser: claims overridden from mapped user")
}
return nil
}
func getUserType(upn string) user.UserType {
var t user.UserType
switch {
case strings.HasPrefix(upn, "guest"):
t = user.UserType_USER_TYPE_LIGHTWEIGHT
case strings.Contains(upn, "@"):
t = user.UserType_USER_TYPE_FEDERATED
default:
t = user.UserType_USER_TYPE_PRIMARY
}
return t
}
@@ -0,0 +1,165 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package accounts
import (
"context"
"database/sql"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/pkg/errors"
)
// Accounts represents oc10-style Accounts
type Accounts struct {
driver string
db *sql.DB
joinUsername, joinUUID, enableMedialSearch bool
selectSQL string
}
// NewMysql returns a new accounts instance connecting to a MySQL database
func NewMysql(dsn string, joinUsername, joinUUID, enableMedialSearch bool) (*Accounts, error) {
sqldb, err := sql.Open("mysql", dsn)
if err != nil {
return nil, errors.Wrap(err, "error connecting to the database")
}
// FIXME make configurable
sqldb.SetConnMaxLifetime(time.Minute * 3)
sqldb.SetConnMaxIdleTime(time.Second * 30)
sqldb.SetMaxOpenConns(100)
sqldb.SetMaxIdleConns(10)
err = sqldb.Ping()
if err != nil {
return nil, errors.Wrap(err, "error connecting to the database")
}
return New("mysql", sqldb, joinUsername, joinUUID, enableMedialSearch)
}
// New returns a new accounts instance connecting to the given sql.DB
func New(driver string, sqldb *sql.DB, joinUsername, joinUUID, enableMedialSearch bool) (*Accounts, error) {
sel := "SELECT id, email, user_id, display_name, quota, last_login, backend, home, state, password"
from := `
FROM oc_accounts a
LEFT JOIN oc_users u
ON a.user_id=u.uid
`
if joinUsername {
sel += ", p.configvalue AS username"
from += `LEFT JOIN oc_preferences p
ON a.user_id=p.userid
AND p.appid='core'
AND p.configkey='username'`
} else {
// fallback to user_id as username
sel += ", user_id AS username"
}
if joinUUID {
sel += ", p2.configvalue AS ownclouduuid"
from += `LEFT JOIN oc_preferences p2
ON a.user_id=p2.userid
AND p2.appid='core'
AND p2.configkey='ownclouduuid'`
} else {
// fallback to user_id as ownclouduuid
sel += ", user_id AS ownclouduuid"
}
return &Accounts{
driver: driver,
db: sqldb,
joinUsername: joinUsername,
joinUUID: joinUUID,
enableMedialSearch: enableMedialSearch,
selectSQL: sel + from,
}, nil
}
// Account stores information about accounts.
type Account struct {
ID uint64
Email sql.NullString
UserID string
DisplayName sql.NullString
Quota sql.NullString
LastLogin int
Backend string
Home string
State int8
PasswordHash string // from oc_users
Username sql.NullString // optional comes from the oc_preferences
OwnCloudUUID sql.NullString // optional comes from the oc_preferences
}
func (as *Accounts) rowToAccount(ctx context.Context, row Scannable) (*Account, error) {
a := Account{}
if err := row.Scan(&a.ID, &a.Email, &a.UserID, &a.DisplayName, &a.Quota, &a.LastLogin, &a.Backend, &a.Home, &a.State, &a.PasswordHash, &a.Username, &a.OwnCloudUUID); err != nil {
appctx.GetLogger(ctx).Error().Err(err).Msg("could not scan row, skipping")
return nil, err
}
return &a, nil
}
// Scannable describes the interface providing a Scan method
type Scannable interface {
Scan(...interface{}) error
}
// GetAccountByLogin fetches an account by mail or username
func (as *Accounts) GetAccountByLogin(ctx context.Context, login string) (*Account, error) {
var row *sql.Row
username := strings.ToLower(login) // usernames are lowercased in owncloud classic
if as.joinUsername {
row = as.db.QueryRowContext(ctx, as.selectSQL+" WHERE a.email=? OR a.lower_user_id=? OR p.configvalue=?", login, username, login)
} else {
row = as.db.QueryRowContext(ctx, as.selectSQL+" WHERE a.email=? OR a.lower_user_id=?", login, username)
}
return as.rowToAccount(ctx, row)
}
// GetAccountGroups reads the groups for an account
func (as *Accounts) GetAccountGroups(ctx context.Context, uid string) ([]string, error) {
rows, err := as.db.QueryContext(ctx, "SELECT gid FROM oc_group_user WHERE uid=?", uid)
if err != nil {
return nil, err
}
defer rows.Close()
var group string
groups := []string{}
for rows.Next() {
if err := rows.Scan(&group); err != nil {
appctx.GetLogger(ctx).Error().Err(err).Msg("could not scan row, skipping")
continue
}
groups = append(groups, group)
}
if err = rows.Err(); err != nil {
return nil, err
}
return groups, nil
}
@@ -0,0 +1,192 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package owncloudsql
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/hex"
"fmt"
"strings"
"golang.org/x/crypto/bcrypt"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/owncloudsql/accounts"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
// Provides mysql drivers
_ "github.com/go-sql-driver/mysql"
)
func init() {
registry.Register("owncloudsql", NewMysql)
}
type manager struct {
c *config
db *accounts.Accounts
}
type config struct {
DbUsername string `mapstructure:"dbusername"`
DbPassword string `mapstructure:"dbpassword"`
DbHost string `mapstructure:"dbhost"`
DbPort int `mapstructure:"dbport"`
DbName string `mapstructure:"dbname"`
Idp string `mapstructure:"idp"`
Nobody int64 `mapstructure:"nobody"`
LegacySalt string `mapstructure:"legacy_salt"`
JoinUsername bool `mapstructure:"join_username"`
JoinOwnCloudUUID bool `mapstructure:"join_ownclouduuid"`
}
// NewMysql returns a new auth manager connection to an owncloud mysql database
func NewMysql(m map[string]interface{}) (auth.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
err = errors.Wrap(err, "error creating a new auth manager")
return nil, err
}
mgr.db, err = accounts.NewMysql(
fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", mgr.c.DbUsername, mgr.c.DbPassword, mgr.c.DbHost, mgr.c.DbPort, mgr.c.DbName),
mgr.c.JoinUsername,
mgr.c.JoinOwnCloudUUID,
false,
)
if err != nil {
return nil, err
}
return mgr, nil
}
func (m *manager) Configure(ml map[string]interface{}) error {
c, err := parseConfig(ml)
if err != nil {
return err
}
if c.Nobody == 0 {
c.Nobody = 99
}
m.c = c
return nil
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, &c); err != nil {
return nil, err
}
return c, nil
}
func (m *manager) Authenticate(ctx context.Context, login, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
log := appctx.GetLogger(ctx)
// 1. find user by login
account, err := m.db.GetAccountByLogin(ctx, login)
if err != nil {
return nil, nil, errtypes.NotFound(login)
}
// 2. verify the user password
if !m.verify(clientSecret, account.PasswordHash) {
return nil, nil, errtypes.InvalidCredentials(login)
}
userID := &user.UserId{
Idp: m.c.Idp,
OpaqueId: account.OwnCloudUUID.String,
Type: user.UserType_USER_TYPE_PRIMARY, // TODO: assign the appropriate user type for guest accounts
}
u := &user.User{
Id: userID,
// TODO add more claims from the StandardClaims, eg EmailVerified and lastlogin
Username: account.Username.String,
Mail: account.Email.String,
DisplayName: account.DisplayName.String,
//UidNumber: uidNumber,
//GidNumber: gidNumber,
}
if u.Groups, err = m.db.GetAccountGroups(ctx, account.UserID); err != nil {
return nil, nil, err
}
var scopes map[string]*authpb.Scope
if userID != nil && (userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT || userID.Type == user.UserType_USER_TYPE_FEDERATED) {
scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil)
if err != nil {
return nil, nil, err
}
} else {
scopes, err = scope.AddOwnerScope(nil)
if err != nil {
return nil, nil, err
}
}
// do not log password hash
account.PasswordHash = "***redacted***"
log.Debug().Interface("account", account).Interface("user", u).Msg("authenticated user")
return u, scopes, nil
}
func (m *manager) verify(password, hash string) bool {
splitHash := strings.SplitN(hash, "|", 2)
switch len(splitHash) {
case 2:
if splitHash[0] == "1" {
return m.verifyHashV1(password, splitHash[1])
}
case 1:
return m.legacyHashVerify(password, hash)
}
return false
}
func (m *manager) legacyHashVerify(password, hash string) bool {
// TODO rehash $newHash = $this->hash($message);
switch len(hash) {
case 60: // legacy PHPass hash
return nil == bcrypt.CompareHashAndPassword([]byte(hash), []byte(password+m.c.LegacySalt))
case 40: // legacy sha1 hash
h := sha1.Sum([]byte(password))
return hmac.Equal([]byte(hash), []byte(hex.EncodeToString(h[:])))
}
return false
}
func (m *manager) verifyHashV1(password, hash string) bool {
// TODO implement password_needs_rehash
return nil == bcrypt.CompareHashAndPassword([]byte(hash), []byte(password))
}
@@ -0,0 +1,173 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package publicshares
import (
"context"
"strings"
"time"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
userprovider "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/manager/registry"
"github.com/cs3org/reva/v2/pkg/auth/scope"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("publicshares", New)
}
type manager struct {
c *config
}
type config struct {
GatewayAddr string `mapstructure:"gateway_addr"`
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns a new auth Manager.
func New(m map[string]interface{}) (auth.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
return nil, err
}
return mgr, nil
}
func (m *manager) Configure(ml map[string]interface{}) error {
conf, err := parseConfig(ml)
if err != nil {
return err
}
m.c = conf
return nil
}
func (m *manager) Authenticate(ctx context.Context, token, secret string) (*user.User, map[string]*authpb.Scope, error) {
gwConn, err := pool.GetGatewayServiceClient(m.c.GatewayAddr)
if err != nil {
return nil, nil, err
}
var auth *link.PublicShareAuthentication
if strings.HasPrefix(secret, "password|") {
secret = strings.TrimPrefix(secret, "password|")
auth = &link.PublicShareAuthentication{
Spec: &link.PublicShareAuthentication_Password{
Password: secret,
},
}
} else if strings.HasPrefix(secret, "signature|") {
secret = strings.TrimPrefix(secret, "signature|")
parts := strings.Split(secret, "|")
sig, expiration := parts[0], parts[1]
exp, _ := time.Parse(time.RFC3339, expiration)
auth = &link.PublicShareAuthentication{
Spec: &link.PublicShareAuthentication_Signature{
Signature: &link.ShareSignature{
Signature: sig,
SignatureExpiration: &types.Timestamp{
Seconds: uint64(exp.UnixNano() / 1000000000),
Nanos: uint32(exp.UnixNano() % 1000000000),
},
},
},
}
}
publicShareResponse, err := gwConn.GetPublicShareByToken(ctx, &link.GetPublicShareByTokenRequest{
Token: token,
Authentication: auth,
Sign: true,
})
switch {
case err != nil:
return nil, nil, err
case publicShareResponse.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND:
return nil, nil, errtypes.NotFound(publicShareResponse.Status.Message)
case publicShareResponse.Status.Code == rpcv1beta1.Code_CODE_PERMISSION_DENIED:
return nil, nil, errtypes.InvalidCredentials(publicShareResponse.Status.Message)
case publicShareResponse.Status.Code != rpcv1beta1.Code_CODE_OK:
return nil, nil, errtypes.InternalError(publicShareResponse.Status.Message)
}
var owner *user.User
// FIXME use new user type SPACE_OWNER
if publicShareResponse.GetShare().GetOwner().GetType() == 8 {
owner = &user.User{Id: publicShareResponse.GetShare().GetOwner(), DisplayName: "Public", Username: "public"}
} else {
getUserResponse, err := gwConn.GetUser(ctx, &userprovider.GetUserRequest{
UserId: publicShareResponse.GetShare().GetCreator(),
})
if err != nil {
return nil, nil, err
}
owner = getUserResponse.GetUser()
}
share := publicShareResponse.GetShare()
role := authpb.Role_ROLE_VIEWER
roleStr := "viewer"
if share.Permissions.Permissions.InitiateFileUpload && !share.Permissions.Permissions.InitiateFileDownload {
role = authpb.Role_ROLE_UPLOADER
roleStr = "uploader"
} else if share.Permissions.Permissions.InitiateFileUpload {
role = authpb.Role_ROLE_EDITOR
roleStr = "editor"
}
scope, err := scope.AddPublicShareScope(share, role, nil)
if err != nil {
return nil, nil, err
}
owner.Opaque = &types.Opaque{
Map: map[string]*types.OpaqueEntry{
"public-share-role": {
Decoder: "plain",
Value: []byte(roleStr),
},
},
}
return owner, scope, nil
}
// ErrPasswordNotProvided is returned when the public share is password protected, but there was no password on the request
var ErrPasswordNotProvided = errors.New("public share is password protected, but password was not provided")
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/auth"
// NewFunc is the function that auth implementations
// should register to at init time.
type NewFunc func(map[string]interface{}) (auth.Manager, error)
// NewFuncs is a map containing all the registered auth managers.
var NewFuncs = map[string]NewFunc{}
// Register registers a new auth manager new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core storage broker drivers.
_ "github.com/cs3org/reva/v2/pkg/auth/registry/static"
// Add your own here
)
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/auth"
// NewFunc is the function that auth registry implementations
// should register at init time.
type NewFunc func(map[string]interface{}) (auth.Registry, error)
// NewFuncs is a map containing all the registered auth registries.
var NewFuncs = map[string]NewFunc{}
// Register registers a new auth registry new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package static
import (
"context"
registrypb "github.com/cs3org/go-cs3apis/cs3/auth/registry/v1beta1"
"github.com/cs3org/reva/v2/pkg/auth"
"github.com/cs3org/reva/v2/pkg/auth/registry/registry"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/sharedconf"
"github.com/mitchellh/mapstructure"
)
func init() {
registry.Register("static", New)
}
type config struct {
Rules map[string]string `mapstructure:"rules"`
}
func (c *config) init() {
if len(c.Rules) == 0 {
c.Rules = map[string]string{
"basic": sharedconf.GetGatewaySVC(""),
}
}
}
type reg struct {
rules map[string]string
}
func (r *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) {
providers := []*registrypb.ProviderInfo{}
for k, v := range r.rules {
providers = append(providers, &registrypb.ProviderInfo{
ProviderType: k,
Address: v,
})
}
return providers, nil
}
func (r *reg) GetProvider(ctx context.Context, authType string) (*registrypb.ProviderInfo, error) {
for k, v := range r.rules {
if k == authType {
return &registrypb.ProviderInfo{
ProviderType: k,
Address: v,
}, nil
}
}
return nil, errtypes.NotFound("static: auth type not found: " + authType)
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
// New returns an implementation of the auth.Registry interface.
func New(m map[string]interface{}) (auth.Registry, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
c.init()
return &reg{rules: c.Rules}, nil
}
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package auth
import (
"context"
"net/rpc"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/plugin"
hcplugin "github.com/hashicorp/go-plugin"
)
func init() {
plugin.Register("authprovider", &ProviderPlugin{})
}
// ProviderPlugin is the implementation of plugin.Plugin so we can serve/consume this.
type ProviderPlugin struct {
Impl Manager
}
// Server returns the RPC Server which serves the methods that the Client calls over net/rpc
func (p *ProviderPlugin) Server(*hcplugin.MuxBroker) (interface{}, error) {
return &RPCServer{Impl: p.Impl}, nil
}
// Client returns interface implementation for the plugin that communicates to the server end of the plugin
func (p *ProviderPlugin) Client(b *hcplugin.MuxBroker, c *rpc.Client) (interface{}, error) {
return &RPCClient{Client: c}, nil
}
// RPCClient is an implementation of Manager that talks over RPC.
type RPCClient struct{ Client *rpc.Client }
// ConfigureArg for RPC
type ConfigureArg struct {
Ml map[string]interface{}
}
// ConfigureReply for RPC
type ConfigureReply struct {
Err error
}
// Configure RPCClient configure method
func (m *RPCClient) Configure(ml map[string]interface{}) error {
args := ConfigureArg{Ml: ml}
resp := ConfigureReply{}
err := m.Client.Call("Plugin.Configure", args, &resp)
if err != nil {
return err
}
return resp.Err
}
// AuthenticateArgs for RPC
type AuthenticateArgs struct {
Ctx map[interface{}]interface{}
ClientID string
ClientSecret string
}
// AuthenticateReply for RPC
type AuthenticateReply struct {
User *user.User
Auth map[string]*authpb.Scope
Error error
}
// Authenticate RPCClient Authenticate method
func (m *RPCClient) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) {
ctxVal := appctx.GetKeyValuesFromCtx(ctx)
args := AuthenticateArgs{Ctx: ctxVal, ClientID: clientID, ClientSecret: clientSecret}
reply := AuthenticateReply{}
err := m.Client.Call("Plugin.Authenticate", args, &reply)
if err != nil {
return nil, nil, err
}
return reply.User, reply.Auth, reply.Error
}
// RPCServer is the server that RPCClient talks to, conforming to the requirements of net/rpc
type RPCServer struct {
// This is the real implementation
Impl Manager
}
// Configure RPCServer Configure method
func (m *RPCServer) Configure(args ConfigureArg, resp *ConfigureReply) error {
resp.Err = m.Impl.Configure(args.Ml)
return nil
}
// Authenticate RPCServer Authenticate method
func (m *RPCServer) Authenticate(args AuthenticateArgs, resp *AuthenticateReply) error {
ctx := appctx.PutKeyValuesToCtx(args.Ctx)
resp.User, resp.Auth, resp.Error = m.Impl.Authenticate(ctx, args.ClientID, args.ClientSecret)
return nil
}
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/rs/zerolog"
)
func lightweightAccountScope(_ context.Context, scope *authpb.Scope, resource interface{}, _ *zerolog.Logger) (bool, error) {
// Lightweight accounts have access to resources shared with them.
// These cannot be resolved from here, but need to be added to the scope from
// where the call to mint tokens is made.
// From here, we only allow ListReceivedShares calls
switch v := resource.(type) {
case *collaboration.ListReceivedSharesRequest:
return true, nil
case string:
return checkLightweightPath(v), nil
}
return false, nil
}
func checkLightweightPath(path string) bool {
paths := []string{
"/ocs/v2.php/apps/files_sharing/api/v1/shares",
"/ocs/v1.php/apps/files_sharing/api/v1/shares",
"/ocs/v2.php/apps/files_sharing//api/v1/shares",
"/ocs/v1.php/apps/files_sharing//api/v1/shares",
"/ocs/v2.php/cloud/capabilities",
"/ocs/v1.php/cloud/capabilities",
"/ocs/v2.php/cloud/user",
"/ocs/v1.php/cloud/user",
"/remote.php/webdav",
"/remote.php/dav/files",
"/app/open",
"/app/new",
"/archiver",
"/dataprovider",
"/data",
}
for _, p := range paths {
if strings.HasPrefix(path, p) {
return true
}
}
return false
}
// AddLightweightAccountScope adds the scope to allow access to lightweight user.
func AddLightweightAccountScope(role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
ref := &provider.Reference{Path: "/"}
val, err := utils.MarshalProtoV1ToJSON(ref)
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["lightweight"] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: role,
}
return scopes, nil
}
+204
View File
@@ -0,0 +1,204 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"strings"
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/rs/zerolog"
)
// PublicStorageProviderID is the space id used for the public links storage space
const PublicStorageProviderID = "7993447f-687f-490d-875c-ac95e89a62a4"
func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interface{}, logger *zerolog.Logger) (bool, error) {
var share link.PublicShare
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &share)
if err != nil {
return false, err
}
switch v := resource.(type) {
// Viewer role
case *registry.GetStorageProvidersRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *registry.ListStorageProvidersRequest:
ref := &provider.Reference{}
if v.Opaque != nil && v.Opaque.Map != nil {
if e, ok := v.Opaque.Map["storage_id"]; ok {
if ref.ResourceId == nil {
ref.ResourceId = &provider.ResourceId{}
}
ref.ResourceId.StorageId = string(e.Value)
}
if e, ok := v.Opaque.Map["space_id"]; ok {
if ref.ResourceId == nil {
ref.ResourceId = &provider.ResourceId{}
}
ref.ResourceId.SpaceId = string(e.Value)
}
if e, ok := v.Opaque.Map["opaque_id"]; ok {
if ref.ResourceId == nil {
ref.ResourceId = &provider.ResourceId{}
}
ref.ResourceId.OpaqueId = string(e.Value)
}
if e, ok := v.Opaque.Map["path"]; ok {
ref.Path = string(e.Value)
}
}
return checkStorageRef(ctx, &share, ref), nil
case *provider.CreateHomeRequest:
return false, nil
case *provider.GetPathRequest:
return checkStorageRef(ctx, &share, &provider.Reference{ResourceId: v.GetResourceId()}), nil
case *provider.StatRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.GetLockRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.UnlockRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.RefreshLockRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.SetLockRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.ListContainerRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.InitiateFileDownloadRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *appprovider.OpenInAppRequest:
return checkStorageRef(ctx, &share, &provider.Reference{ResourceId: v.ResourceInfo.Id}), nil
case *gateway.OpenInAppRequest:
return checkStorageRef(ctx, &share, v.GetRef()), nil
case *permissionsv1beta1.CheckPermissionRequest:
return true, nil
// Editor role
// need to return appropriate status codes in the ocs/ocdav layers.
case *provider.CreateContainerRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.TouchFileRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.DeleteRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.MoveRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetSource()) && checkStorageRef(ctx, &share, v.GetDestination()), nil
case *provider.InitiateFileUploadRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.SetArbitraryMetadataRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
case *provider.UnsetArbitraryMetadataRequest:
return hasRoleEditor(*scope) && checkStorageRef(ctx, &share, v.GetRef()), nil
// App provider requests
case *appregistry.GetDefaultAppProviderForMimeTypeRequest:
return true, nil
case *appregistry.GetAppProvidersRequest:
return true, nil
case *userv1beta1.GetUserByClaimRequest:
return true, nil
case *userv1beta1.GetUserRequest:
return true, nil
case *provider.ListStorageSpacesRequest:
return true, nil
case *link.GetPublicShareRequest:
return checkPublicShareRef(&share, v.GetRef()), nil
case *link.ListPublicSharesRequest:
// public links must not leak info about other links
return false, nil
case *collaboration.ListReceivedSharesRequest:
// public links must not leak info about collaborative shares
return false, nil
case string:
return checkResourcePath(v), nil
}
msg := "public resource type assertion failed"
logger.Debug().Str("scope", "publicshareScope").Interface("resource", resource).Msg(msg)
return false, errtypes.InternalError(msg)
}
func checkStorageRef(ctx context.Context, s *link.PublicShare, r *provider.Reference) bool {
// r: <resource_id:<storage_id:$storageID space_id:$spaceID opaque_id:$opaqueID> path:$path > >
if utils.ResourceIDEqual(s.ResourceId, r.GetResourceId()) {
return true
}
// r: <path:"/public/$token" >
if strings.HasPrefix(r.GetPath(), "/public/"+s.Token) || strings.HasPrefix(r.GetPath(), "./"+s.Token) {
return true
}
// r: <resource_id:<storage_id: space_id: opaque_id:$token> path:$path>
if id := r.GetResourceId(); id.GetStorageId() == PublicStorageProviderID {
// access to /public
if id.GetOpaqueId() == PublicStorageProviderID {
return true
}
// access relative to /public/$token
if id.GetOpaqueId() == s.Token {
return true
}
}
return false
}
func checkPublicShareRef(s *link.PublicShare, ref *link.PublicShareReference) bool {
// ref: <token:$token >
return ref.GetToken() == s.Token
}
// AddPublicShareScope adds the scope to allow access to a public share and
// the shared resource.
func AddPublicShareScope(share *link.PublicShare, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
// Create a new "scope share" to only expose the required fields `ResourceId` and `Token` to the scope.
scopeShare := &link.PublicShare{ResourceId: share.ResourceId, Token: share.Token}
val, err := utils.MarshalProtoV1ToJSON(scopeShare)
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["publicshare:"+share.Id.OpaqueId] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: role,
}
return scopes, nil
}
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"fmt"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/rs/zerolog"
)
func receivedShareScope(_ context.Context, scope *authpb.Scope, resource interface{}, logger *zerolog.Logger) (bool, error) {
var share collaboration.ReceivedShare
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &share)
if err != nil {
return false, err
}
switch v := resource.(type) {
case *collaboration.GetReceivedShareRequest:
return checkShareRef(share.Share, v.GetRef()), nil
case *collaboration.UpdateReceivedShareRequest:
return checkShare(share.Share, v.GetShare().GetShare()), nil
case string:
return checkSharePath(v) || checkResourcePath(v), nil
}
msg := fmt.Sprintf("resource type assertion failed: %+v", resource)
logger.Debug().Str("scope", "receivedShareScope").Msg(msg)
return false, errtypes.InternalError(msg)
}
// AddReceivedShareScope adds the scope to allow access to a received user/group share and
// the shared resource.
func AddReceivedShareScope(share *collaboration.ReceivedShare, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
// Create a new "scope share" to only expose the required fields to the scope.
scopeShare := &collaboration.Share{Id: share.Share.Id, Owner: share.Share.Owner, Creator: share.Share.Creator, ResourceId: share.Share.ResourceId}
val, err := utils.MarshalProtoV1ToJSON(&collaboration.ReceivedShare{Share: scopeShare})
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["receivedshare:"+share.Share.Id.OpaqueId] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: role,
}
return scopes, nil
}
+166
View File
@@ -0,0 +1,166 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"fmt"
"strings"
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1"
"github.com/rs/zerolog"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
)
func resourceinfoScope(_ context.Context, scope *authpb.Scope, resource interface{}, logger *zerolog.Logger) (bool, error) {
var r provider.ResourceInfo
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &r)
if err != nil {
return false, err
}
switch v := resource.(type) {
// Viewer role
case *registry.GetStorageProvidersRequest:
return checkResourceInfo(&r, v.GetRef()), nil
case *registry.ListStorageProvidersRequest:
// the call will only return spaces the current user has access to
ref := &provider.Reference{}
if v.Opaque != nil && v.Opaque.Map != nil {
if e, ok := v.Opaque.Map["storage_id"]; ok {
ref.ResourceId = &provider.ResourceId{
StorageId: string(e.Value),
}
}
if e, ok := v.Opaque.Map["opaque_id"]; ok {
if ref.ResourceId == nil {
ref.ResourceId = &provider.ResourceId{}
}
ref.ResourceId.OpaqueId = string(e.Value)
}
if e, ok := v.Opaque.Map["path"]; ok {
ref.Path = string(e.Value)
}
}
return checkResourceInfo(&r, ref), nil
case *provider.ListStorageSpacesRequest:
// the call will only return spaces the current user has access to
return true, nil
case *provider.StatRequest:
return checkResourceInfo(&r, v.GetRef()), nil
case *provider.ListContainerRequest:
return checkResourceInfo(&r, v.GetRef()), nil
case *provider.InitiateFileDownloadRequest:
return checkResourceInfo(&r, v.GetRef()), nil
case *appprovider.OpenInAppRequest:
return checkResourceInfo(&r, &provider.Reference{ResourceId: v.ResourceInfo.Id}), nil
case *gateway.OpenInAppRequest:
return checkResourceInfo(&r, v.GetRef()), nil
// Editor role
// need to return appropriate status codes in the ocs/ocdav layers.
case *provider.CreateContainerRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case *provider.TouchFileRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case *provider.DeleteRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case *provider.MoveRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetSource()) && checkResourceInfo(&r, v.GetDestination()), nil
case *provider.InitiateFileUploadRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case *provider.SetArbitraryMetadataRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case *provider.UnsetArbitraryMetadataRequest:
return hasRoleEditor(*scope) && checkResourceInfo(&r, v.GetRef()), nil
case string:
return checkResourcePath(v), nil
}
msg := fmt.Sprintf("resource type assertion failed: %+v", resource)
logger.Debug().Str("scope", "resourceinfoScope").Msg(msg)
return false, errtypes.InternalError(msg)
}
func checkResourceInfo(inf *provider.ResourceInfo, ref *provider.Reference) bool {
// ref: <resource_id:<storage_id:$storageID opaque_id:$opaqueID path:$path> >
if ref.ResourceId != nil { // path can be empty or a relative path
if inf.Id.SpaceId == ref.ResourceId.SpaceId && inf.Id.OpaqueId == ref.ResourceId.OpaqueId {
if ref.Path == "" {
// id only reference
return true
}
// check path has same prefix below
} else {
return false
}
}
// ref: <path:$path >
if strings.HasPrefix(ref.GetPath(), inf.Path) {
return true
}
return false
}
func checkResourcePath(path string) bool {
paths := []string{
"/dataprovider",
"/data",
"/app/open",
"/app/new",
"/archiver",
"/ocs/v2.php/cloud/capabilities",
"/ocs/v1.php/cloud/capabilities",
}
for _, p := range paths {
if strings.HasPrefix(path, p) {
return true
}
}
return false
}
// AddResourceInfoScope adds the scope to allow access to a resource info object.
func AddResourceInfoScope(r *provider.ResourceInfo, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
// Create a new "scope info" to only expose the required fields `Id` and `Path` to the scope.
scopeInfo := &provider.ResourceInfo{Id: r.Id, Path: r.Path}
val, err := utils.MarshalProtoV1ToJSON(scopeInfo)
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["resourceinfo:"+r.Id.String()] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: role,
}
return scopes, nil
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/rs/zerolog"
)
// Verifier is the function signature which every scope verifier should implement.
type Verifier func(context.Context, *authpb.Scope, interface{}, *zerolog.Logger) (bool, error)
var supportedScopes = map[string]Verifier{
"user": userScope,
"publicshare": publicshareScope,
"resourceinfo": resourceinfoScope,
"share": shareScope,
"receivedshare": receivedShareScope,
"lightweight": lightweightAccountScope,
}
// VerifyScope is the function to be called when dismantling tokens to check if
// the token has access to a particular resource.
func VerifyScope(ctx context.Context, scopeMap map[string]*authpb.Scope, resource interface{}) (bool, error) {
logger := appctx.GetLogger(ctx)
for k, scope := range scopeMap {
for s, f := range supportedScopes {
if strings.HasPrefix(k, s) {
if valid, err := f(ctx, scope, resource, logger); err == nil && valid {
return true, nil
}
}
}
}
return false, nil
}
func hasRoleEditor(scope authpb.Scope) bool {
return scope.Role == authpb.Role_ROLE_OWNER || scope.Role == authpb.Role_ROLE_EDITOR || scope.Role == authpb.Role_ROLE_UPLOADER
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
"fmt"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/rs/zerolog"
)
func shareScope(_ context.Context, scope *authpb.Scope, resource interface{}, logger *zerolog.Logger) (bool, error) {
var share collaboration.Share
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &share)
if err != nil {
return false, err
}
switch v := resource.(type) {
// Viewer role
case *registry.GetStorageProvidersRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.StatRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.ListContainerRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.InitiateFileDownloadRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
// Editor role
// TODO(ishank011): Add role checks,
// need to return appropriate status codes in the ocs/ocdav layers.
case *provider.CreateContainerRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.TouchFileRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.DeleteRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *provider.MoveRequest:
return checkShareStorageRef(&share, v.GetSource()) && checkShareStorageRef(&share, v.GetDestination()), nil
case *provider.InitiateFileUploadRequest:
return checkShareStorageRef(&share, v.GetRef()), nil
case *collaboration.ListReceivedSharesRequest:
return true, nil
case *collaboration.GetReceivedShareRequest:
return checkShareRef(&share, v.GetRef()), nil
case string:
return checkSharePath(v) || checkResourcePath(v), nil
}
msg := fmt.Sprintf("resource type assertion failed: %+v", resource)
logger.Debug().Str("scope", "shareScope").Msg(msg)
return false, errtypes.InternalError(msg)
}
func checkShareStorageRef(s *collaboration.Share, r *provider.Reference) bool {
// ref: <id:<storage_id:$storageID opaque_id:$opaqueID > >
if r.GetResourceId() != nil && r.Path == "" { // path must be empty
return utils.ResourceIDEqual(s.ResourceId, r.GetResourceId())
}
return false
}
func checkShareRef(s *collaboration.Share, ref *collaboration.ShareReference) bool {
if ref.GetId() != nil {
return ref.GetId().OpaqueId == s.Id.OpaqueId
}
if key := ref.GetKey(); key != nil {
return (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) &&
utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee)
}
return false
}
func checkShare(s1 *collaboration.Share, s2 *collaboration.Share) bool {
if s2.GetId() != nil {
return s2.GetId().OpaqueId == s1.Id.OpaqueId
}
return false
}
func checkSharePath(path string) bool {
paths := []string{
"/ocs/v2.php/apps/files_sharing/api/v1/shares",
"/ocs/v1.php/apps/files_sharing/api/v1/shares",
"/remote.php/webdav",
"/remote.php/dav/files",
}
for _, p := range paths {
if strings.HasPrefix(path, p) {
return true
}
}
return false
}
// AddShareScope adds the scope to allow access to a user/group share and
// the shared resource.
func AddShareScope(share *collaboration.Share, role authpb.Role, scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
// Create a new "scope share" to only expose the required fields to the scope.
scopeShare := &collaboration.Share{Id: share.Id, Owner: share.Owner, Creator: share.Creator, ResourceId: share.ResourceId}
val, err := utils.MarshalProtoV1ToJSON(scopeShare)
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["share:"+share.Id.OpaqueId] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: role,
}
return scopes, nil
}
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"context"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/rs/zerolog"
)
func userScope(_ context.Context, scope *authpb.Scope, resource interface{}, _ *zerolog.Logger) (bool, error) {
// Always return true. Registered users can access all paths.
// TODO(ishank011): Add checks for read/write permissions.
return true, nil
}
// AddOwnerScope adds the default owner scope with access to all resources.
func AddOwnerScope(scopes map[string]*authpb.Scope) (map[string]*authpb.Scope, error) {
ref := &provider.Reference{Path: "/"}
val, err := utils.MarshalProtoV1ToJSON(ref)
if err != nil {
return nil, err
}
if scopes == nil {
scopes = make(map[string]*authpb.Scope)
}
scopes["user"] = &authpb.Scope{
Resource: &types.OpaqueEntry{
Decoder: "json",
Value: val,
},
Role: authpb.Role_ROLE_OWNER,
}
return scopes, nil
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package scope
import (
"fmt"
"strings"
authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
)
// FormatScope create a pretty print of the scope
func FormatScope(scopeType string, scope *authpb.Scope) (string, error) {
// TODO(gmgigi96): check decoder type
switch {
case strings.HasPrefix(scopeType, "user"):
// user scope
var ref provider.Reference
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &ref)
if err != nil {
return "", err
}
return fmt.Sprintf("%s %s", ref.String(), scope.Role.String()), nil
case strings.HasPrefix(scopeType, "publicshare"):
// public share
var pShare link.PublicShare
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &pShare)
if err != nil {
return "", err
}
return fmt.Sprintf("share:\"%s\" %s", pShare.Id.OpaqueId, scope.Role.String()), nil
case strings.HasPrefix(scopeType, "resourceinfo"):
var resInfo provider.ResourceInfo
err := utils.UnmarshalJSONToProtoV1(scope.Resource.Value, &resInfo)
if err != nil {
return "", err
}
return fmt.Sprintf("path:\"%s\" %s", resInfo.Path, scope.Role.String()), nil
default:
return "", errtypes.NotSupported("scope not yet supported")
}
}
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package bytesize provides easy conversions from human readable strings (eg. 10MB) to bytes
package bytesize
import (
"fmt"
"strconv"
"strings"
)
// ByteSize is the size in bytes
type ByteSize uint64
// List of available byte sizes
// NOTE: max is exabyte as we convert to uint64
const (
KB ByteSize = 1000
MB ByteSize = 1000 * KB
GB ByteSize = 1000 * MB
TB ByteSize = 1000 * GB
PB ByteSize = 1000 * TB
EB ByteSize = 1000 * PB
KiB ByteSize = 1024
MiB ByteSize = 1024 * KiB
GiB ByteSize = 1024 * MiB
TiB ByteSize = 1024 * GiB
PiB ByteSize = 1024 * TiB
EiB ByteSize = 1024 * PiB
)
// Parse parses a Bytesize from a string
func Parse(s string) (ByteSize, error) {
sanitized := strings.TrimSpace(s)
if !strings.HasSuffix(sanitized, "B") {
u, err := strconv.Atoi(sanitized)
return ByteSize(u), err
}
var (
value int
unit string
)
template := "%d%s"
_, err := fmt.Sscanf(sanitized, template, &value, &unit)
if err != nil {
return 0, err
}
bytes := ByteSize(value)
switch unit {
case "KB":
bytes *= KB
case "MB":
bytes *= MB
case "GB":
bytes *= GB
case "TB":
bytes *= TB
case "PB":
bytes *= PB
case "EB":
bytes *= EB
case "KiB":
bytes *= KiB
case "MiB":
bytes *= MiB
case "GiB":
bytes *= GiB
case "TiB":
bytes *= TiB
case "PiB":
bytes *= PiB
case "EiB":
bytes *= EiB
default:
return 0, fmt.Errorf("unknown unit '%s'. Use common abbreviations such as KB, MiB, GB", unit)
}
return bytes, nil
}
// Bytes converts the ByteSize to an uint64
func (b ByteSize) Bytes() uint64 {
return uint64(b)
}
// String converts the ByteSize to a string
func (b ByteSize) String() string {
return strconv.FormatUint(uint64(b), 10)
}
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package cbox
import (
"context"
"database/sql"
"fmt"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/cbox/utils"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/storage/favorite"
"github.com/cs3org/reva/v2/pkg/storage/favorite/registry"
"github.com/mitchellh/mapstructure"
)
func init() {
registry.Register("sql", New)
}
type config struct {
DbUsername string `mapstructure:"db_username"`
DbPassword string `mapstructure:"db_password"`
DbHost string `mapstructure:"db_host"`
DbPort int `mapstructure:"db_port"`
DbName string `mapstructure:"db_name"`
}
type mgr struct {
c *config
db *sql.DB
}
// New returns an instance of the cbox sql favorites manager.
func New(m map[string]interface{}) (favorite.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DbUsername, c.DbPassword, c.DbHost, c.DbPort, c.DbName))
if err != nil {
return nil, err
}
return &mgr{
c: c,
db: db,
}, nil
}
func (m *mgr) ListFavorites(ctx context.Context, userID *user.UserId) ([]*provider.ResourceId, error) {
user := ctxpkg.ContextMustGetUser(ctx)
infos := []*provider.ResourceId{}
query := `SELECT fileid_prefix, fileid FROM cbox_metadata WHERE uid=? AND tag_key="fav"`
rows, err := m.db.Query(query, user.Id.OpaqueId)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var info provider.ResourceId
if err := rows.Scan(&info.SpaceId, &info.OpaqueId); err != nil {
return nil, err
}
infos = append(infos, &info)
}
if err = rows.Err(); err != nil {
return nil, err
}
return infos, nil
}
func (m *mgr) SetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error {
user := ctxpkg.ContextMustGetUser(ctx)
spaceID := resourceInfo.Id.SpaceId
// The primary key is just the ID in the table, it should ideally be (uid, fileid_prefix, fileid, tag_key)
// For the time being, just check if the favorite already exists. If it does, return early
var id int
query := `SELECT id FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"`
if err := m.db.QueryRow(query, user.Id.OpaqueId, spaceID, resourceInfo.Id.OpaqueId).Scan(&id); err == nil {
// Favorite is already set, return
return nil
}
query = `INSERT INTO cbox_metadata SET item_type=?, uid=?, fileid_prefix=?, fileid=?, tag_key="fav"`
vals := []interface{}{utils.ResourceTypeToItemInt(resourceInfo.Type), user.Id.OpaqueId, spaceID, resourceInfo.Id.OpaqueId}
stmt, err := m.db.Prepare(query)
if err != nil {
return err
}
if _, err = stmt.Exec(vals...); err != nil {
return err
}
return nil
}
func (m *mgr) UnsetFavorite(ctx context.Context, userID *user.UserId, resourceInfo *provider.ResourceInfo) error {
user := ctxpkg.ContextMustGetUser(ctx)
spaceID := resourceInfo.Id.SpaceId
stmt, err := m.db.Prepare(`DELETE FROM cbox_metadata WHERE uid=? AND fileid_prefix=? AND fileid=? AND tag_key="fav"`)
if err != nil {
return err
}
res, err := stmt.Exec(user.Id.OpaqueId, spaceID, resourceInfo.Id.OpaqueId)
if err != nil {
return err
}
_, err = res.RowsAffected()
if err != nil {
return err
}
return nil
}
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rest
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/gomodule/redigo/redis"
)
const (
groupPrefix = "group:"
idPrefix = "id:"
namePrefix = "name:"
gidPrefix = "gid:"
groupMembersPrefix = "members:"
groupInternalIDPrefix = "internal:"
)
func initRedisPool(address, username, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 50,
MaxActive: 1000,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
var c redis.Conn
var err error
switch {
case username != "":
c, err = redis.Dial("tcp", address,
redis.DialUsername(username),
redis.DialPassword(password),
)
case password != "":
c, err = redis.Dial("tcp", address,
redis.DialPassword(password),
)
default:
c, err = redis.Dial("tcp", address)
}
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
func (m *manager) setVal(key, val string, expiration int) error {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
args := []interface{}{key, val}
if expiration != -1 {
args = append(args, "EX", expiration)
}
if _, err := conn.Do("SET", args...); err != nil {
return err
}
return nil
}
return errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) getVal(key string) (string, error) {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
val, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", err
}
return val, nil
}
return "", errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) fetchCachedInternalID(gid *grouppb.GroupId) (string, error) {
return m.getVal(groupPrefix + groupInternalIDPrefix + gid.OpaqueId)
}
func (m *manager) cacheInternalID(gid *grouppb.GroupId, internalID string) error {
return m.setVal(groupPrefix+groupInternalIDPrefix+gid.OpaqueId, internalID, -1)
}
func (m *manager) findCachedGroups(query string) ([]*grouppb.Group, error) {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
query = fmt.Sprintf("%s*%s*", groupPrefix, strings.ReplaceAll(strings.ToLower(query), " ", "_"))
keys, err := redis.Strings(conn.Do("KEYS", query))
if err != nil {
return nil, err
}
var args []interface{}
for _, k := range keys {
args = append(args, k)
}
// Fetch the groups for all these keys
groupStrings, err := redis.Strings(conn.Do("MGET", args...))
if err != nil {
return nil, err
}
groupMap := make(map[string]*grouppb.Group)
for _, group := range groupStrings {
g := grouppb.Group{}
if err = json.Unmarshal([]byte(group), &g); err == nil {
groupMap[g.Id.OpaqueId] = &g
}
}
var groups []*grouppb.Group
for _, g := range groupMap {
groups = append(groups, g)
}
return groups, nil
}
return nil, errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) fetchCachedGroupDetails(gid *grouppb.GroupId) (*grouppb.Group, error) {
group, err := m.getVal(groupPrefix + idPrefix + gid.OpaqueId)
if err != nil {
return nil, err
}
g := grouppb.Group{}
if err = json.Unmarshal([]byte(group), &g); err != nil {
return nil, err
}
return &g, nil
}
func (m *manager) cacheGroupDetails(g *grouppb.Group) error {
encodedGroup, err := json.Marshal(&g)
if err != nil {
return err
}
if err = m.setVal(groupPrefix+idPrefix+strings.ToLower(g.Id.OpaqueId), string(encodedGroup), -1); err != nil {
return err
}
if g.GidNumber != 0 {
if err = m.setVal(groupPrefix+gidPrefix+strconv.FormatInt(g.GidNumber, 10), g.Id.OpaqueId, -1); err != nil {
return err
}
}
if g.DisplayName != "" {
if err = m.setVal(groupPrefix+namePrefix+g.Id.OpaqueId+"_"+strings.ToLower(g.DisplayName), g.Id.OpaqueId, -1); err != nil {
return err
}
}
return nil
}
func (m *manager) fetchCachedGroupByParam(field, claim string) (*grouppb.Group, error) {
group, err := m.getVal(groupPrefix + field + ":" + strings.ToLower(claim))
if err != nil {
return nil, err
}
g := grouppb.Group{}
if err = json.Unmarshal([]byte(group), &g); err != nil {
return nil, err
}
return &g, nil
}
func (m *manager) fetchCachedGroupMembers(gid *grouppb.GroupId) ([]*userpb.UserId, error) {
members, err := m.getVal(groupPrefix + groupMembersPrefix + strings.ToLower(gid.OpaqueId))
if err != nil {
return nil, err
}
u := []*userpb.UserId{}
if err = json.Unmarshal([]byte(members), &u); err != nil {
return nil, err
}
return u, nil
}
func (m *manager) cacheGroupMembers(gid *grouppb.GroupId, members []*userpb.UserId) error {
u, err := json.Marshal(&members)
if err != nil {
return err
}
return m.setVal(groupPrefix+groupMembersPrefix+strings.ToLower(gid.OpaqueId), string(u), m.conf.GroupMembersCacheExpiration*60)
}
+329
View File
@@ -0,0 +1,329 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rest
import (
"context"
"errors"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
utils "github.com/cs3org/reva/v2/pkg/cbox/utils"
"github.com/cs3org/reva/v2/pkg/group"
"github.com/cs3org/reva/v2/pkg/group/manager/registry"
"github.com/gomodule/redigo/redis"
"github.com/mitchellh/mapstructure"
"github.com/rs/zerolog/log"
)
func init() {
registry.Register("rest", New)
}
type manager struct {
conf *config
redisPool *redis.Pool
apiTokenManager *utils.APITokenManager
}
type config struct {
// The address at which the redis server is running
RedisAddress string `mapstructure:"redis_address" docs:"localhost:6379"`
// The username for connecting to the redis server
RedisUsername string `mapstructure:"redis_username" docs:""`
// The password for connecting to the redis server
RedisPassword string `mapstructure:"redis_password" docs:""`
// The time in minutes for which the members of a group would be cached
GroupMembersCacheExpiration int `mapstructure:"group_members_cache_expiration" docs:"5"`
// The OIDC Provider
IDProvider string `mapstructure:"id_provider" docs:"http://cernbox.cern.ch"`
// Base API Endpoint
APIBaseURL string `mapstructure:"api_base_url" docs:"https://authorization-service-api-dev.web.cern.ch"`
// Client ID needed to authenticate
ClientID string `mapstructure:"client_id" docs:"-"`
// Client Secret
ClientSecret string `mapstructure:"client_secret" docs:"-"`
// Endpoint to generate token to access the API
OIDCTokenEndpoint string `mapstructure:"oidc_token_endpoint" docs:"https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token"`
// The target application for which token needs to be generated
TargetAPI string `mapstructure:"target_api" docs:"authorization-service-api"`
// The time in seconds between bulk fetch of groups
GroupFetchInterval int `mapstructure:"group_fetch_interval" docs:"3600"`
}
func (c *config) init() {
if c.GroupMembersCacheExpiration == 0 {
c.GroupMembersCacheExpiration = 5
}
if c.RedisAddress == "" {
c.RedisAddress = ":6379"
}
if c.APIBaseURL == "" {
c.APIBaseURL = "https://authorization-service-api-dev.web.cern.ch"
}
if c.TargetAPI == "" {
c.TargetAPI = "authorization-service-api"
}
if c.OIDCTokenEndpoint == "" {
c.OIDCTokenEndpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token"
}
if c.IDProvider == "" {
c.IDProvider = "http://cernbox.cern.ch"
}
if c.GroupFetchInterval == 0 {
c.GroupFetchInterval = 3600
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
// New returns a user manager implementation that makes calls to the GRAPPA API.
func New(m map[string]interface{}) (group.Manager, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
c.init()
redisPool := initRedisPool(c.RedisAddress, c.RedisUsername, c.RedisPassword)
apiTokenManager := utils.InitAPITokenManager(c.TargetAPI, c.OIDCTokenEndpoint, c.ClientID, c.ClientSecret)
mgr := &manager{
conf: c,
redisPool: redisPool,
apiTokenManager: apiTokenManager,
}
go mgr.fetchAllGroups()
return mgr, nil
}
func (m *manager) fetchAllGroups() {
_ = m.fetchAllGroupAccounts()
ticker := time.NewTicker(time.Duration(m.conf.GroupFetchInterval) * time.Second)
work := make(chan os.Signal, 1)
signal.Notify(work, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT)
for {
select {
case <-work:
return
case <-ticker.C:
_ = m.fetchAllGroupAccounts()
}
}
}
func (m *manager) fetchAllGroupAccounts() error {
ctx := context.Background()
url := fmt.Sprintf("%s/api/v1.0/Group?field=groupIdentifier&field=displayName&field=gid", m.conf.APIBaseURL)
for url != "" {
result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false)
if err != nil {
return err
}
responseData, ok := result["data"].([]interface{})
if !ok {
return errors.New("rest: error in type assertion")
}
for _, usr := range responseData {
groupData, ok := usr.(map[string]interface{})
if !ok {
continue
}
_, err = m.parseAndCacheGroup(ctx, groupData)
if err != nil {
continue
}
}
url = ""
if pagination, ok := result["pagination"].(map[string]interface{}); ok {
if links, ok := pagination["links"].(map[string]interface{}); ok {
if next, ok := links["next"].(string); ok {
url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, next)
}
}
}
}
return nil
}
func (m *manager) parseAndCacheGroup(ctx context.Context, groupData map[string]interface{}) (*grouppb.Group, error) {
id, ok := groupData["groupIdentifier"].(string)
if !ok {
return nil, errors.New("rest: missing upn in user data")
}
name, _ := groupData["displayName"].(string)
groupID := &grouppb.GroupId{
OpaqueId: id,
Idp: m.conf.IDProvider,
}
gid, ok := groupData["gid"].(int64)
if !ok {
gid = 0
}
g := &grouppb.Group{
Id: groupID,
GroupName: id,
Mail: id + "@cern.ch",
DisplayName: name,
GidNumber: gid,
}
if err := m.cacheGroupDetails(g); err != nil {
log.Error().Err(err).Msg("rest: error caching group details")
}
if internalID, ok := groupData["id"].(string); ok {
if err := m.cacheInternalID(groupID, internalID); err != nil {
log.Error().Err(err).Msg("rest: error caching group details")
}
}
return g, nil
}
func (m *manager) GetGroup(ctx context.Context, gid *grouppb.GroupId, skipFetchingMembers bool) (*grouppb.Group, error) {
g, err := m.fetchCachedGroupDetails(gid)
if err != nil {
return nil, err
}
if !skipFetchingMembers {
groupMembers, err := m.GetMembers(ctx, gid)
if err != nil {
return nil, err
}
g.Members = groupMembers
}
return g, nil
}
func (m *manager) GetGroupByClaim(ctx context.Context, claim, value string, skipFetchingMembers bool) (*grouppb.Group, error) {
if claim == "group_name" {
return m.GetGroup(ctx, &grouppb.GroupId{OpaqueId: value}, skipFetchingMembers)
}
g, err := m.fetchCachedGroupByParam(claim, value)
if err != nil {
return nil, err
}
if !skipFetchingMembers {
groupMembers, err := m.GetMembers(ctx, g.Id)
if err != nil {
return nil, err
}
g.Members = groupMembers
}
return g, nil
}
func (m *manager) FindGroups(ctx context.Context, query string, skipFetchingMembers bool) ([]*grouppb.Group, error) {
// Look at namespaces filters. If the query starts with:
// "a" or none => get egroups
// other filters => get empty list
parts := strings.SplitN(query, ":", 2)
if len(parts) == 2 {
if parts[0] == "a" {
query = parts[1]
} else {
return []*grouppb.Group{}, nil
}
}
return m.findCachedGroups(query)
}
func (m *manager) GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*userpb.UserId, error) {
users, err := m.fetchCachedGroupMembers(gid)
if err == nil {
return users, nil
}
internalID, err := m.fetchCachedInternalID(gid)
if err != nil {
return nil, err
}
url := fmt.Sprintf("%s/api/v1.0/Group/%s/memberidentities/precomputed", m.conf.APIBaseURL, internalID)
result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false)
if err != nil {
return nil, err
}
userData := result["data"].([]interface{})
users = []*userpb.UserId{}
for _, u := range userData {
userInfo, ok := u.(map[string]interface{})
if !ok {
return nil, errors.New("rest: error in type assertion")
}
if id, ok := userInfo["upn"].(string); ok {
users = append(users, &userpb.UserId{OpaqueId: id, Idp: m.conf.IDProvider})
}
}
if err = m.cacheGroupMembers(gid, users); err != nil {
log := appctx.GetLogger(ctx)
log.Error().Err(err).Msg("rest: error caching group members")
}
return users, nil
}
func (m *manager) HasMember(ctx context.Context, gid *grouppb.GroupId, uid *userpb.UserId) (bool, error) {
groupMemers, err := m.GetMembers(ctx, gid)
if err != nil {
return false, err
}
for _, u := range groupMemers {
if uid.OpaqueId == u.OpaqueId {
return true, nil
}
}
return false, nil
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load cbox specific drivers.
_ "github.com/cs3org/reva/v2/pkg/cbox/favorite/sql"
_ "github.com/cs3org/reva/v2/pkg/cbox/group/rest"
_ "github.com/cs3org/reva/v2/pkg/cbox/preferences/sql"
_ "github.com/cs3org/reva/v2/pkg/cbox/publicshare/sql"
_ "github.com/cs3org/reva/v2/pkg/cbox/share/sql"
_ "github.com/cs3org/reva/v2/pkg/cbox/storage/eoshomewrapper"
_ "github.com/cs3org/reva/v2/pkg/cbox/storage/eoswrapper"
_ "github.com/cs3org/reva/v2/pkg/cbox/user/rest"
)
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package sql
import (
"context"
"database/sql"
"fmt"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/preferences"
"github.com/cs3org/reva/v2/pkg/preferences/registry"
"github.com/mitchellh/mapstructure"
)
func init() {
registry.Register("sql", New)
}
type config struct {
DbUsername string `mapstructure:"db_username"`
DbPassword string `mapstructure:"db_password"`
DbHost string `mapstructure:"db_host"`
DbPort int `mapstructure:"db_port"`
DbName string `mapstructure:"db_name"`
}
type mgr struct {
c *config
db *sql.DB
}
// New returns an instance of the cbox sql preferences manager.
func New(m map[string]interface{}) (preferences.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DbUsername, c.DbPassword, c.DbHost, c.DbPort, c.DbName))
if err != nil {
return nil, err
}
return &mgr{
c: c,
db: db,
}, nil
}
func (m *mgr) SetKey(ctx context.Context, key, namespace, value string) error {
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return errtypes.UserRequired("preferences: error getting user from ctx")
}
query := `INSERT INTO oc_preferences(userid, appid, configkey, configvalue) values(?, ?, ?, ?) ON DUPLICATE KEY UPDATE configvalue = ?`
params := []interface{}{user.Id.OpaqueId, namespace, key, value, value}
stmt, err := m.db.Prepare(query)
if err != nil {
return err
}
if _, err = stmt.Exec(params...); err != nil {
return err
}
return nil
}
func (m *mgr) GetKey(ctx context.Context, key, namespace string) (string, error) {
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return "", errtypes.UserRequired("preferences: error getting user from ctx")
}
query := `SELECT configvalue FROM oc_preferences WHERE userid=? AND appid=? AND configkey=?`
var val string
if err := m.db.QueryRow(query, user.Id.OpaqueId, namespace, key).Scan(&val); err != nil {
if err == sql.ErrNoRows {
return "", errtypes.NotFound(namespace + ":" + key)
}
return "", err
}
return val, nil
}
+509
View File
@@ -0,0 +1,509 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package sql
import (
"context"
"database/sql"
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"golang.org/x/crypto/bcrypt"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
conversions "github.com/cs3org/reva/v2/pkg/cbox/utils"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/publicshare"
"github.com/cs3org/reva/v2/pkg/publicshare/manager/registry"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
const publicShareType = 3
func init() {
registry.Register("sql", New)
}
type config struct {
SharePasswordHashCost int `mapstructure:"password_hash_cost"`
JanitorRunInterval int `mapstructure:"janitor_run_interval"`
EnableExpiredSharesCleanup bool `mapstructure:"enable_expired_shares_cleanup"`
DbUsername string `mapstructure:"db_username"`
DbPassword string `mapstructure:"db_password"`
DbHost string `mapstructure:"db_host"`
DbPort int `mapstructure:"db_port"`
DbName string `mapstructure:"db_name"`
}
type manager struct {
c *config
db *sql.DB
}
func (c *config) init() {
if c.SharePasswordHashCost == 0 {
c.SharePasswordHashCost = 11
}
if c.JanitorRunInterval == 0 {
c.JanitorRunInterval = 3600
}
}
func (m *manager) startJanitorRun() {
if !m.c.EnableExpiredSharesCleanup {
return
}
ticker := time.NewTicker(time.Duration(m.c.JanitorRunInterval) * time.Second)
work := make(chan os.Signal, 1)
signal.Notify(work, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT)
for {
select {
case <-work:
return
case <-ticker.C:
_ = m.cleanupExpiredShares()
}
}
}
// New returns a new public share manager.
func New(m map[string]interface{}) (publicshare.Manager, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
c.init()
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DbUsername, c.DbPassword, c.DbHost, c.DbPort, c.DbName))
if err != nil {
return nil, err
}
mgr := manager{
c: c,
db: db,
}
go mgr.startJanitorRun()
return &mgr, nil
}
func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant) (*link.PublicShare, error) {
tkn := utils.RandString(15)
now := time.Now().Unix()
displayName, ok := rInfo.ArbitraryMetadata.Metadata["name"]
if !ok {
displayName = tkn
}
createdAt := &typespb.Timestamp{
Seconds: uint64(now),
}
creator := conversions.FormatUserID(u.Id)
owner := conversions.FormatUserID(rInfo.Owner)
permissions := conversions.SharePermToInt(g.Permissions.Permissions)
itemType := conversions.ResourceTypeToItem(rInfo.Type)
prefix := rInfo.Id.SpaceId
itemSource := rInfo.Id.OpaqueId
fileSource, err := strconv.ParseUint(itemSource, 10, 64)
if err != nil {
// it can be the case that the item source may be a character string
// we leave fileSource blank in that case
fileSource = 0
}
query := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,token=?,share_name=?"
params := []interface{}{publicShareType, owner, creator, itemType, prefix, itemSource, fileSource, permissions, now, tkn, displayName}
var passwordProtected bool
password := g.Password
if password != "" {
password, err = hashPassword(password, m.c.SharePasswordHashCost)
if err != nil {
return nil, errors.Wrap(err, "could not hash share password")
}
passwordProtected = true
query += ",share_with=?"
params = append(params, password)
}
if g.Expiration != nil && g.Expiration.Seconds != 0 {
t := time.Unix(int64(g.Expiration.Seconds), 0)
query += ",expiration=?"
params = append(params, t)
}
stmt, err := m.db.Prepare(query)
if err != nil {
return nil, err
}
result, err := stmt.Exec(params...)
if err != nil {
return nil, err
}
lastID, err := result.LastInsertId()
if err != nil {
return nil, err
}
return &link.PublicShare{
Id: &link.PublicShareId{
OpaqueId: strconv.FormatInt(lastID, 10),
},
Owner: rInfo.GetOwner(),
Creator: u.Id,
ResourceId: rInfo.Id,
Token: tkn,
Permissions: g.Permissions,
Ctime: createdAt,
Mtime: createdAt,
PasswordProtected: passwordProtected,
Expiration: g.Expiration,
DisplayName: displayName,
}, nil
}
func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest) (*link.PublicShare, error) {
query := "update oc_share set "
paramsMap := map[string]interface{}{}
params := []interface{}{}
now := time.Now().Unix()
uid := conversions.FormatUserID(u.Id)
switch req.GetUpdate().GetType() {
case link.UpdatePublicShareRequest_Update_TYPE_DISPLAYNAME:
paramsMap["share_name"] = req.Update.GetDisplayName()
case link.UpdatePublicShareRequest_Update_TYPE_PERMISSIONS:
paramsMap["permissions"] = conversions.SharePermToInt(req.Update.GetGrant().GetPermissions().Permissions)
case link.UpdatePublicShareRequest_Update_TYPE_EXPIRATION:
paramsMap["expiration"] = time.Unix(int64(req.Update.GetGrant().Expiration.Seconds), 0)
case link.UpdatePublicShareRequest_Update_TYPE_PASSWORD:
if req.Update.GetGrant().Password == "" {
paramsMap["share_with"] = ""
} else {
h, err := hashPassword(req.Update.GetGrant().Password, m.c.SharePasswordHashCost)
if err != nil {
return nil, errors.Wrap(err, "could not hash share password")
}
paramsMap["share_with"] = h
}
default:
return nil, fmt.Errorf("invalid update type: %v", req.GetUpdate().GetType())
}
for k, v := range paramsMap {
query += k + "=?"
params = append(params, v)
}
switch {
case req.Ref.GetId() != nil:
query += ",stime=? where id=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, now, req.Ref.GetId().OpaqueId, uid, uid)
case req.Ref.GetToken() != "":
query += ",stime=? where token=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, now, req.Ref.GetToken(), uid, uid)
default:
return nil, errtypes.NotFound(req.Ref.String())
}
stmt, err := m.db.Prepare(query)
if err != nil {
return nil, err
}
if _, err = stmt.Exec(params...); err != nil {
return nil, err
}
return m.GetPublicShare(ctx, u, req.Ref, false)
}
func (m *manager) getByToken(ctx context.Context, token string, u *user.User) (*link.PublicShare, string, error) {
s := conversions.DBShare{Token: token}
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND token=?"
if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions); err != nil {
if err == sql.ErrNoRows {
return nil, "", errtypes.NotFound(token)
}
return nil, "", err
}
return conversions.ConvertToCS3PublicShare(s), s.ShareWith, nil
}
func (m *manager) getByID(ctx context.Context, id *link.PublicShareId, u *user.User) (*link.PublicShare, string, error) {
uid := conversions.FormatUserID(u.Id)
s := conversions.DBShare{ID: id.OpaqueId}
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, stime, permissions FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND id=? AND (uid_owner=? OR uid_initiator=?)"
if err := m.db.QueryRow(query, publicShareType, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.STime, &s.Permissions); err != nil {
if err == sql.ErrNoRows {
return nil, "", errtypes.NotFound(id.OpaqueId)
}
return nil, "", err
}
return conversions.ConvertToCS3PublicShare(s), s.ShareWith, nil
}
func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference, sign bool) (*link.PublicShare, error) {
var s *link.PublicShare
var pw string
var err error
switch {
case ref.GetId() != nil:
s, pw, err = m.getByID(ctx, ref.GetId(), u)
case ref.GetToken() != "":
s, pw, err = m.getByToken(ctx, ref.GetToken(), u)
default:
err = errtypes.NotFound(ref.String())
}
if err != nil {
return nil, err
}
if expired(s) {
if err := m.cleanupExpiredShares(); err != nil {
return nil, err
}
return nil, errtypes.NotFound(ref.String())
}
if s.PasswordProtected && sign {
if err := publicshare.AddSignature(s, pw); err != nil {
return nil, err
}
}
return s, nil
}
func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, sign bool) ([]*link.PublicShare, error) {
uid := conversions.FormatUserID(u.Id)
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner=? or uid_initiator=?) AND (share_type=?)"
var resourceFilters, ownerFilters, creatorFilters string
var resourceParams, ownerParams, creatorParams []interface{}
params := []interface{}{uid, uid, publicShareType}
for _, f := range filters {
switch f.Type {
case link.ListPublicSharesRequest_Filter_TYPE_RESOURCE_ID:
if len(resourceFilters) != 0 {
resourceFilters += " OR "
}
resourceFilters += "(fileid_prefix=? AND item_source=?)"
resourceParams = append(resourceParams, f.GetResourceId().SpaceId, f.GetResourceId().OpaqueId)
case link.ListPublicSharesRequest_Filter_TYPE_OWNER:
if len(ownerFilters) != 0 {
ownerFilters += " OR "
}
ownerFilters += "(uid_owner=?)"
ownerParams = append(ownerParams, conversions.FormatUserID(f.GetOwner()))
case link.ListPublicSharesRequest_Filter_TYPE_CREATOR:
if len(creatorFilters) != 0 {
creatorFilters += " OR "
}
creatorFilters += "(uid_initiator=?)"
creatorParams = append(creatorParams, conversions.FormatUserID(f.GetCreator()))
}
}
if resourceFilters != "" {
query = fmt.Sprintf("%s AND (%s)", query, resourceFilters)
params = append(params, resourceParams...)
}
if ownerFilters != "" {
query = fmt.Sprintf("%s AND (%s)", query, ownerFilters)
params = append(params, ownerParams...)
}
if creatorFilters != "" {
query = fmt.Sprintf("%s AND (%s)", query, creatorFilters)
params = append(params, creatorParams...)
}
rows, err := m.db.Query(query, params...)
if err != nil {
return nil, err
}
defer rows.Close()
var s conversions.DBShare
shares := []*link.PublicShare{}
for rows.Next() {
if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions); err != nil {
continue
}
cs3Share := conversions.ConvertToCS3PublicShare(s)
if expired(cs3Share) {
_ = m.cleanupExpiredShares()
} else {
if cs3Share.PasswordProtected && sign {
if err := publicshare.AddSignature(cs3Share, s.ShareWith); err != nil {
return nil, err
}
}
shares = append(shares, cs3Share)
}
}
if err = rows.Err(); err != nil {
return nil, err
}
return shares, nil
}
func (m *manager) RevokePublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference) error {
uid := conversions.FormatUserID(u.Id)
query := "delete from oc_share where "
params := []interface{}{}
switch {
case ref.GetId() != nil && ref.GetId().OpaqueId != "":
query += "id=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, ref.GetId().OpaqueId, uid, uid)
case ref.GetToken() != "":
query += "token=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, ref.GetToken(), uid, uid)
default:
return errtypes.NotFound(ref.String())
}
stmt, err := m.db.Prepare(query)
if err != nil {
return err
}
res, err := stmt.Exec(params...)
if err != nil {
return err
}
rowCnt, err := res.RowsAffected()
if err != nil {
return err
}
if rowCnt == 0 {
return errtypes.NotFound(ref.String())
}
return nil
}
func (m *manager) GetPublicShareByToken(ctx context.Context, token string, auth *link.PublicShareAuthentication, sign bool) (*link.PublicShare, error) {
s := conversions.DBShare{Token: token}
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions FROM oc_share WHERE share_type=? AND token=?"
if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions); err != nil {
if err == sql.ErrNoRows {
return nil, errtypes.NotFound(token)
}
return nil, err
}
cs3Share := conversions.ConvertToCS3PublicShare(s)
if s.ShareWith != "" {
if !authenticate(cs3Share, s.ShareWith, auth) {
// if check := checkPasswordHash(auth.Password, s.ShareWith); !check {
return nil, errtypes.InvalidCredentials(token)
}
if sign {
if err := publicshare.AddSignature(cs3Share, s.ShareWith); err != nil {
return nil, err
}
}
}
if expired(cs3Share) {
if err := m.cleanupExpiredShares(); err != nil {
return nil, err
}
return nil, errtypes.NotFound(token)
}
return cs3Share, nil
}
func (m *manager) cleanupExpiredShares() error {
if !m.c.EnableExpiredSharesCleanup {
return nil
}
query := "update oc_share set orphan = 1 where expiration IS NOT NULL AND expiration < ?"
params := []interface{}{time.Now().Format("2006-01-02 03:04:05")}
stmt, err := m.db.Prepare(query)
if err != nil {
return err
}
if _, err = stmt.Exec(params...); err != nil {
return err
}
return nil
}
func expired(s *link.PublicShare) bool {
if s.Expiration != nil {
if t := time.Unix(int64(s.Expiration.GetSeconds()), int64(s.Expiration.GetNanos())); t.Before(time.Now()) {
return true
}
}
return false
}
func hashPassword(password string, cost int) (string, error) {
bytes, err := bcrypt.GenerateFromPassword([]byte(password), cost)
return "1|" + string(bytes), err
}
func checkPasswordHash(password, hash string) bool {
err := bcrypt.CompareHashAndPassword([]byte(strings.TrimPrefix(hash, "1|")), []byte(password))
return err == nil
}
func authenticate(share *link.PublicShare, pw string, auth *link.PublicShareAuthentication) bool {
switch {
case auth.GetPassword() != "":
return checkPasswordHash(auth.GetPassword(), pw)
case auth.GetSignature() != nil:
sig := auth.GetSignature()
now := time.Now()
expiration := time.Unix(int64(sig.GetSignatureExpiration().GetSeconds()), int64(sig.GetSignatureExpiration().GetNanos()))
if now.After(expiration) {
return false
}
s, err := publicshare.CreateSignature(share.Token, pw, expiration)
if err != nil {
// TODO(labkode): pass context to call to log err.
// No we are blind
return false
}
return sig.GetSignature() == s
}
return false
}
+557
View File
@@ -0,0 +1,557 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package sql
import (
"context"
"database/sql"
"fmt"
"path"
"strconv"
"strings"
"time"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
conversions "github.com/cs3org/reva/v2/pkg/cbox/utils"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/share"
"github.com/cs3org/reva/v2/pkg/share/manager/registry"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"google.golang.org/genproto/protobuf/field_mask"
// Provides mysql drivers
_ "github.com/go-sql-driver/mysql"
)
const (
shareTypeUser = 0
shareTypeGroup = 1
)
func init() {
registry.Register("sql", New)
}
type config struct {
DbUsername string `mapstructure:"db_username"`
DbPassword string `mapstructure:"db_password"`
DbHost string `mapstructure:"db_host"`
DbPort int `mapstructure:"db_port"`
DbName string `mapstructure:"db_name"`
}
type mgr struct {
c *config
db *sql.DB
}
// New returns a new share manager.
func New(m map[string]interface{}) (share.Manager, error) {
c, err := parseConfig(m)
if err != nil {
err = errors.Wrap(err, "error creating a new manager")
return nil, err
}
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DbUsername, c.DbPassword, c.DbHost, c.DbPort, c.DbName))
if err != nil {
return nil, err
}
return &mgr{
c: c,
db: db,
}, nil
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) {
user := ctxpkg.ContextMustGetUser(ctx)
// do not allow share to myself or the owner if share is for a user
// TODO(labkode): should not this be caught already at the gw level?
if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER &&
(utils.UserEqual(g.Grantee.GetUserId(), user.Id) || utils.UserEqual(g.Grantee.GetUserId(), md.Owner)) {
return nil, errors.New("sql: owner/creator and grantee are the same")
}
// check if share already exists.
key := &collaboration.ShareKey{
Owner: md.Owner,
ResourceId: md.Id,
Grantee: g.Grantee,
}
_, err := m.getByKey(ctx, key)
// share already exists
if err == nil {
return nil, errtypes.AlreadyExists(key.String())
}
now := time.Now().Unix()
ts := &typespb.Timestamp{
Seconds: uint64(now),
}
shareType, shareWith := conversions.FormatGrantee(g.Grantee)
itemType := conversions.ResourceTypeToItem(md.Type)
targetPath := path.Join("/", path.Base(md.Path))
permissions := conversions.SharePermToInt(g.Permissions.Permissions)
prefix := md.Id.SpaceId
itemSource := md.Id.OpaqueId
fileSource, err := strconv.ParseUint(itemSource, 10, 64)
if err != nil {
// it can be the case that the item source may be a character string
// we leave fileSource blank in that case
fileSource = 0
}
stmtString := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,share_with=?,file_target=?"
stmtValues := []interface{}{shareType, conversions.FormatUserID(md.Owner), conversions.FormatUserID(user.Id), itemType, prefix, itemSource, fileSource, permissions, now, shareWith, targetPath}
stmt, err := m.db.Prepare(stmtString)
if err != nil {
return nil, err
}
result, err := stmt.Exec(stmtValues...)
if err != nil {
return nil, err
}
lastID, err := result.LastInsertId()
if err != nil {
return nil, err
}
return &collaboration.Share{
Id: &collaboration.ShareId{
OpaqueId: strconv.FormatInt(lastID, 10),
},
ResourceId: md.Id,
Permissions: g.Permissions,
Grantee: g.Grantee,
Owner: md.Owner,
Creator: user.Id,
Ctime: ts,
Mtime: ts,
}, nil
}
func (m *mgr) getByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.Share, error) {
uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id)
s := conversions.DBShare{ID: id.OpaqueId}
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND id=? AND (uid_owner=? or uid_initiator=?)"
if err := m.db.QueryRow(query, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.STime, &s.Permissions, &s.ShareType); err != nil {
if err == sql.ErrNoRows {
return nil, errtypes.NotFound(id.OpaqueId)
}
return nil, err
}
return conversions.ConvertToCS3Share(s), nil
}
func (m *mgr) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) {
owner := conversions.FormatUserID(key.Owner)
uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id)
s := conversions.DBShare{}
shareType, shareWith := conversions.FormatGrantee(key.Grantee)
query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, id, stime, permissions, share_type FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)"
if err := m.db.QueryRow(query, owner, key.ResourceId.SpaceId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil {
if err == sql.ErrNoRows {
return nil, errtypes.NotFound(key.String())
}
return nil, err
}
return conversions.ConvertToCS3Share(s), nil
}
func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) {
var s *collaboration.Share
var err error
switch {
case ref.GetId() != nil:
s, err = m.getByID(ctx, ref.GetId())
case ref.GetKey() != nil:
s, err = m.getByKey(ctx, ref.GetKey())
default:
err = errtypes.NotFound(ref.String())
}
if err != nil {
return nil, err
}
return s, nil
}
func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error {
uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id)
var query string
params := []interface{}{}
switch {
case ref.GetId() != nil:
query = "delete from oc_share where id=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, ref.GetId().OpaqueId, uid, uid)
case ref.GetKey() != nil:
key := ref.GetKey()
shareType, shareWith := conversions.FormatGrantee(key.Grantee)
owner := conversions.FormatUserID(key.Owner)
query = "delete from oc_share where uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, owner, key.ResourceId.SpaceId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid)
default:
return errtypes.NotFound(ref.String())
}
stmt, err := m.db.Prepare(query)
if err != nil {
return err
}
res, err := stmt.Exec(params...)
if err != nil {
return err
}
rowCnt, err := res.RowsAffected()
if err != nil {
return err
}
if rowCnt == 0 {
return errtypes.NotFound(ref.String())
}
return nil
}
func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions, updated *collaboration.Share, fieldMask *field_mask.FieldMask) (*collaboration.Share, error) {
permissions := conversions.SharePermToInt(p.Permissions)
uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id)
var query string
params := []interface{}{}
switch {
case ref.GetId() != nil:
query = "update oc_share set permissions=?,stime=? where id=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, permissions, time.Now().Unix(), ref.GetId().OpaqueId, uid, uid)
case ref.GetKey() != nil:
key := ref.GetKey()
shareType, shareWith := conversions.FormatGrantee(key.Grantee)
owner := conversions.FormatUserID(key.Owner)
query = "update oc_share set permissions=?,stime=? where (uid_owner=? or uid_initiator=?) AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=? AND (uid_owner=? or uid_initiator=?)"
params = append(params, permissions, time.Now().Unix(), owner, owner, key.ResourceId.SpaceId, key.ResourceId.OpaqueId, shareType, shareWith, uid, uid)
default:
return nil, errtypes.NotFound(ref.String())
}
stmt, err := m.db.Prepare(query)
if err != nil {
return nil, err
}
if _, err = stmt.Exec(params...); err != nil {
return nil, err
}
return m.GetShare(ctx, ref)
}
func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) {
uid := conversions.FormatUserID(ctxpkg.ContextMustGetUser(ctx).Id)
query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with,
coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type,
id, stime, permissions, share_type
FROM oc_share
WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner=? or uid_initiator=?) AND (share_type=? OR share_type=?)`
params := []interface{}{uid, uid, shareTypeUser, shareTypeGroup}
if len(filters) > 0 {
filterQuery, filterParams, err := translateFilters(filters)
if err != nil {
return nil, err
}
params = append(params, filterParams...)
if filterQuery != "" {
query = fmt.Sprintf("%s AND (%s)", query, filterQuery)
}
}
rows, err := m.db.Query(query, params...)
if err != nil {
return nil, err
}
defer rows.Close()
var s conversions.DBShare
shares := []*collaboration.Share{}
for rows.Next() {
if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.ID, &s.STime, &s.Permissions, &s.ShareType); err != nil {
continue
}
shares = append(shares, conversions.ConvertToCS3Share(s))
}
if err = rows.Err(); err != nil {
return nil, err
}
return shares, nil
}
// we list the shares that are targeted to the user in context or to the user groups.
func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) {
user := ctxpkg.ContextMustGetUser(ctx)
uid := conversions.FormatUserID(user.Id)
params := []interface{}{uid, uid, uid, uid}
for _, v := range user.Groups {
params = append(params, v)
}
query := `SELECT coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with,
coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(file_target, '') as file_target,
ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state
FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?)
WHERE (orphan = 0 or orphan IS NULL) AND (uid_owner != ? AND uid_initiator != ?)`
if len(user.Groups) > 0 {
query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))"
} else {
query += " AND (share_with=? AND share_type = 0)"
}
filterQuery, filterParams, err := translateFilters(filters)
if err != nil {
return nil, err
}
params = append(params, filterParams...)
if filterQuery != "" {
query = fmt.Sprintf("%s AND (%s)", query, filterQuery)
}
rows, err := m.db.Query(query, params...)
if err != nil {
return nil, err
}
defer rows.Close()
var s conversions.DBShare
shares := []*collaboration.ReceivedShare{}
for rows.Next() {
if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.FileTarget, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil {
continue
}
shares = append(shares, conversions.ConvertToCS3ReceivedShare(s))
}
if err = rows.Err(); err != nil {
return nil, err
}
return shares, nil
}
func (m *mgr) getReceivedByID(ctx context.Context, id *collaboration.ShareId) (*collaboration.ReceivedShare, error) {
user := ctxpkg.ContextMustGetUser(ctx)
uid := conversions.FormatUserID(user.Id)
params := []interface{}{uid, id.OpaqueId, uid}
for _, v := range user.Groups {
params = append(params, v)
}
s := conversions.DBShare{ID: id.OpaqueId}
query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with,
coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(file_target, '') as file_target,
stime, permissions, share_type, coalesce(tr.state, 0) as state
FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?)
WHERE (orphan = 0 or orphan IS NULL) AND ts.id=?`
if len(user.Groups) > 0 {
query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))"
} else {
query += " AND (share_with=? AND share_type = 0)"
}
if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.FileTarget, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil {
if err == sql.ErrNoRows {
return nil, errtypes.NotFound(id.OpaqueId)
}
return nil, err
}
return conversions.ConvertToCS3ReceivedShare(s), nil
}
func (m *mgr) getReceivedByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.ReceivedShare, error) {
user := ctxpkg.ContextMustGetUser(ctx)
uid := conversions.FormatUserID(user.Id)
shareType, shareWith := conversions.FormatGrantee(key.Grantee)
params := []interface{}{uid, conversions.FormatUserID(key.Owner), key.GetResourceId().SpaceId, key.ResourceId.OpaqueId, shareType, shareWith, shareWith}
for _, v := range user.Groups {
params = append(params, v)
}
s := conversions.DBShare{}
query := `select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with,
coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(file_target, '') as file_target,
ts.id, stime, permissions, share_type, coalesce(tr.state, 0) as state
FROM oc_share ts LEFT JOIN oc_share_status tr ON (ts.id = tr.id AND tr.recipient = ?)
WHERE (orphan = 0 or orphan IS NULL) AND uid_owner=? AND fileid_prefix=? AND item_source=? AND share_type=? AND share_with=?`
if len(user.Groups) > 0 {
query += " AND ((share_with=? AND share_type = 0) OR (share_type = 1 AND share_with in (?" + strings.Repeat(",?", len(user.Groups)-1) + ")))"
} else {
query += " AND (share_with=? AND share_type = 0)"
}
if err := m.db.QueryRow(query, params...).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.FileTarget, &s.ID, &s.STime, &s.Permissions, &s.ShareType, &s.State); err != nil {
if err == sql.ErrNoRows {
return nil, errtypes.NotFound(key.String())
}
return nil, err
}
return conversions.ConvertToCS3ReceivedShare(s), nil
}
func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) {
var s *collaboration.ReceivedShare
var err error
switch {
case ref.GetId() != nil:
s, err = m.getReceivedByID(ctx, ref.GetId())
case ref.GetKey() != nil:
s, err = m.getReceivedByKey(ctx, ref.GetKey())
default:
err = errtypes.NotFound(ref.String())
}
if err != nil {
return nil, err
}
return s, nil
}
func (m *mgr) UpdateReceivedShare(ctx context.Context, share *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) {
user := ctxpkg.ContextMustGetUser(ctx)
rs, err := m.GetReceivedShare(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Share.Id}})
if err != nil {
return nil, err
}
for i := range fieldMask.Paths {
switch fieldMask.Paths[i] {
case "state":
rs.State = share.State
case "mount_point":
rs.MountPoint = share.MountPoint
default:
return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported")
}
}
state := 0
switch rs.GetState() {
case collaboration.ShareState_SHARE_STATE_REJECTED:
state = -1
case collaboration.ShareState_SHARE_STATE_ACCEPTED:
state = 1
}
params := []interface{}{rs.Share.Id.OpaqueId, conversions.FormatUserID(user.Id), state, state}
query := "insert into oc_share_status(id, recipient, state) values(?, ?, ?) ON DUPLICATE KEY UPDATE state = ?"
stmt, err := m.db.Prepare(query)
if err != nil {
return nil, err
}
_, err = stmt.Exec(params...)
if err != nil {
return nil, err
}
return rs, nil
}
func granteeTypeToShareType(granteeType provider.GranteeType) int {
switch granteeType {
case provider.GranteeType_GRANTEE_TYPE_USER:
return shareTypeUser
case provider.GranteeType_GRANTEE_TYPE_GROUP:
return shareTypeGroup
}
return -1
}
// translateFilters translates the filters to sql queries
func translateFilters(filters []*collaboration.Filter) (string, []interface{}, error) {
var (
filterQuery string
params []interface{}
)
groupedFilters := share.GroupFiltersByType(filters)
// If multiple filters of the same type are passed to this function, they need to be combined with the `OR` operator.
// That is why the filters got grouped by type.
// For every given filter type, iterate over the filters and if there are more than one combine them.
// Combine the different filter types using `AND`
var filterCounter = 0
for filterType, filters := range groupedFilters {
switch filterType {
case collaboration.Filter_TYPE_RESOURCE_ID:
filterQuery += "("
for i, f := range filters {
filterQuery += "(fileid_prefix =? AND item_source=?)"
params = append(params, f.GetResourceId().SpaceId, f.GetResourceId().OpaqueId)
if i != len(filters)-1 {
filterQuery += " OR "
}
}
filterQuery += ")"
case collaboration.Filter_TYPE_GRANTEE_TYPE:
filterQuery += "("
for i, f := range filters {
filterQuery += "share_type=?"
params = append(params, granteeTypeToShareType(f.GetGranteeType()))
if i != len(filters)-1 {
filterQuery += " OR "
}
}
filterQuery += ")"
case collaboration.Filter_TYPE_EXCLUDE_DENIALS:
// TODO this may change once the mapping of permission to share types is completed (cf. pkg/cbox/utils/conversions.go)
filterQuery += "(permissions > 0)"
default:
return "", nil, fmt.Errorf("filter type is not supported")
}
if filterCounter != len(groupedFilters)-1 {
filterQuery += " AND "
}
filterCounter++
}
return filterQuery, params, nil
}
@@ -0,0 +1,128 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eoshomewrapper
import (
"bytes"
"context"
"text/template"
"github.com/Masterminds/sprig"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
"github.com/cs3org/reva/v2/pkg/storage/utils/eosfs"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("eoshomewrapper", New)
}
type wrapper struct {
storage.FS
mountIDTemplate *template.Template
}
func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) {
c := &eosfs.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, "", err
}
// default to version invariance if not configured
if _, ok := m["version_invariant"]; !ok {
c.VersionInvariant = true
}
t, ok := m["mount_id_template"].(string)
if !ok || t == "" {
t = "eoshome-{{substr 0 1 .Username}}"
}
return c, t, nil
}
// New returns an implementation of the storage.FS interface that forms a wrapper
// around separate connections to EOS.
func New(m map[string]interface{}, _ events.Stream) (storage.FS, error) {
c, t, err := parseConfig(m)
if err != nil {
return nil, err
}
c.EnableHome = true
eos, err := eosfs.NewEOSFS(c)
if err != nil {
return nil, err
}
mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t)
if err != nil {
return nil, err
}
return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil
}
// We need to override the two methods, GetMD and ListFolder to fill the
// StorageId in the ResourceInfo objects.
func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string, fieldMask []string) (*provider.ResourceInfo, error) {
res, err := w.FS.GetMD(ctx, ref, mdKeys, fieldMask)
if err != nil {
return nil, err
}
// We need to extract the mount ID based on the mapping template.
//
// Take the first letter of the username of the logged-in user, as the home
// storage provider restricts requests only to the home namespace.
res.Id.StorageId = w.getMountID(ctx, res)
return res, nil
}
func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys, fieldMask []string) ([]*provider.ResourceInfo, error) {
res, err := w.FS.ListFolder(ctx, ref, mdKeys, fieldMask)
if err != nil {
return nil, err
}
for _, r := range res {
r.Id.StorageId = w.getMountID(ctx, r)
}
return res, nil
}
func (w *wrapper) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error {
return errtypes.NotSupported("eos: deny grant is only enabled for project spaces")
}
func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string {
u := ctxpkg.ContextMustGetUser(ctx)
b := bytes.Buffer{}
if err := w.mountIDTemplate.Execute(&b, u); err != nil {
return ""
}
return b.String()
}
@@ -0,0 +1,294 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eoswrapper
import (
"bytes"
"context"
"io"
"path"
"strings"
"text/template"
"github.com/Masterminds/sprig"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
"github.com/cs3org/reva/v2/pkg/storage/utils/eosfs"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("eoswrapper", New)
}
const (
eosProjectsNamespace = "/eos/project"
// We can use a regex for these, but that might have inferior performance
projectSpaceGroupsPrefix = "cernbox-project-"
projectSpaceAdminGroupsSuffix = "-admins"
)
type wrapper struct {
storage.FS
conf *eosfs.Config
mountIDTemplate *template.Template
}
func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) {
c := &eosfs.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, "", err
}
// default to version invariance if not configured
if _, ok := m["version_invariant"]; !ok {
c.VersionInvariant = true
}
// allow recycle operations for project spaces
if !c.EnableHome && strings.HasPrefix(c.Namespace, eosProjectsNamespace) {
c.AllowPathRecycleOperations = true
c.ImpersonateOwnerforRevisions = true
}
t, ok := m["mount_id_template"].(string)
if !ok || t == "" {
t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}"
}
return c, t, nil
}
// New returns an implementation of the storage.FS interface that forms a wrapper
// around separate connections to EOS.
func New(m map[string]interface{}, _ events.Stream) (storage.FS, error) {
c, t, err := parseConfig(m)
if err != nil {
return nil, err
}
eos, err := eosfs.NewEOSFS(c)
if err != nil {
return nil, err
}
mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t)
if err != nil {
return nil, err
}
return &wrapper{FS: eos, conf: c, mountIDTemplate: mountIDTemplate}, nil
}
// We need to override the methods, GetMD, GetPathByID and ListFolder to fill the
// StorageId in the ResourceInfo objects.
func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string, fieldMask []string) (*provider.ResourceInfo, error) {
res, err := w.FS.GetMD(ctx, ref, mdKeys, fieldMask)
if err != nil {
return nil, err
}
// We need to extract the mount ID based on the mapping template.
//
// Take the first letter of the resource path after the namespace has been removed.
// If it's empty, leave it empty to be filled by storageprovider.
res.Id.StorageId = w.getMountID(ctx, res)
if err = w.setProjectSharingPermissions(ctx, res); err != nil {
return nil, err
}
// If the request contains a relative reference, we also need to return the base path instead of the full one
if utils.IsRelativeReference(ref) {
res.Path = path.Base(res.Path)
}
return res, nil
}
func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys, fieldMask []string) ([]*provider.ResourceInfo, error) {
res, err := w.FS.ListFolder(ctx, ref, mdKeys, fieldMask)
if err != nil {
return nil, err
}
for _, r := range res {
r.Id.StorageId = w.getMountID(ctx, r)
// If the request contains a relative reference, we also need to return the base path instead of the full one
if utils.IsRelativeReference(ref) {
r.Path = path.Base(r.Path)
}
if err = w.setProjectSharingPermissions(ctx, r); err != nil {
continue
}
}
return res, nil
}
func (w *wrapper) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) {
res, err := w.FS.ListRecycle(ctx, ref, key, relativePath)
if err != nil {
return nil, err
}
// If the request contains a relative reference, we also need to return the base path instead of the full one
if utils.IsRelativeReference(ref) {
for _, info := range res {
info.Ref.Path = path.Base(info.Ref.Path)
}
}
return res, nil
}
func (w *wrapper) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, unrestricted bool) ([]*provider.StorageSpace, error) {
res, err := w.FS.ListStorageSpaces(ctx, filter, unrestricted)
if err != nil {
return nil, err
}
for _, r := range res {
if mountID, _, _, _ := storagespace.SplitID(r.Id.OpaqueId); mountID == "" {
mountID = w.getMountID(ctx, &provider.ResourceInfo{Path: r.Name})
r.Root.StorageId = mountID
}
}
return res, nil
}
func (w *wrapper) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) {
if err := w.userIsProjectAdmin(ctx, ref); err != nil {
return nil, err
}
return w.FS.ListRevisions(ctx, ref)
}
func (w *wrapper) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) {
if err := w.userIsProjectAdmin(ctx, ref); err != nil {
return nil, err
}
return w.FS.DownloadRevision(ctx, ref, revisionKey)
}
func (w *wrapper) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error {
if err := w.userIsProjectAdmin(ctx, ref); err != nil {
return err
}
return w.FS.RestoreRevision(ctx, ref, revisionKey)
}
func (w *wrapper) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error {
// This is only allowed for project space admins
if strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) {
if err := w.userIsProjectAdmin(ctx, ref); err != nil {
return err
}
return w.FS.DenyGrant(ctx, ref, g)
}
return errtypes.NotSupported("eos: deny grant is only enabled for project spaces")
}
func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string {
if r == nil {
return ""
}
r.Path = strings.TrimPrefix(r.Path, w.conf.MountPath)
b := bytes.Buffer{}
if err := w.mountIDTemplate.Execute(&b, r); err != nil {
return ""
}
r.Path = path.Join(w.conf.MountPath, r.Path)
return b.String()
}
func (w *wrapper) setProjectSharingPermissions(ctx context.Context, r *provider.ResourceInfo) error {
// Check if this storage provider corresponds to a project spaces instance
if strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) {
// Extract project name from the path resembling /c/cernbox or /c/cernbox/minutes/..
parts := strings.SplitN(r.Path, "/", 4)
if len(parts) != 4 && len(parts) != 3 {
// The request might be for / or /$letter
// Nothing to do in that case
return nil
}
adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix
user := ctxpkg.ContextMustGetUser(ctx)
for _, g := range user.Groups {
if g == adminGroup {
r.PermissionSet.AddGrant = true
r.PermissionSet.RemoveGrant = true
r.PermissionSet.UpdateGrant = true
r.PermissionSet.ListGrants = true
r.PermissionSet.GetQuota = true
r.PermissionSet.DenyGrant = true
return nil
}
}
}
return nil
}
func (w *wrapper) userIsProjectAdmin(ctx context.Context, ref *provider.Reference) error {
// Check if this storage provider corresponds to a project spaces instance
if !strings.HasPrefix(w.conf.Namespace, eosProjectsNamespace) {
return nil
}
res, err := w.FS.GetMD(ctx, ref, nil, nil)
if err != nil {
return err
}
// Extract project name from the path resembling /c/cernbox or /c/cernbox/minutes/..
parts := strings.SplitN(res.Path, "/", 4)
if len(parts) != 4 && len(parts) != 3 {
// The request might be for / or /$letter
// Nothing to do in that case
return nil
}
adminGroup := projectSpaceGroupsPrefix + parts[2] + projectSpaceAdminGroupsSuffix
user := ctxpkg.ContextMustGetUser(ctx)
for _, g := range user.Groups {
if g == adminGroup {
return nil
}
}
return errtypes.PermissionDenied("eosfs: project spaces revisions can only be accessed by admins")
}
+214
View File
@@ -0,0 +1,214 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rest
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/gomodule/redigo/redis"
)
const (
userPrefix = "user:"
usernamePrefix = "username:"
userIDPrefix = "userid:"
namePrefix = "name:"
mailPrefix = "mail:"
uidPrefix = "uid:"
userGroupsPrefix = "groups:"
)
func initRedisPool(address, username, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 50,
MaxActive: 1000,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
var opts []redis.DialOption
if username != "" {
opts = append(opts, redis.DialUsername(username))
}
if password != "" {
opts = append(opts, redis.DialPassword(password))
}
c, err := redis.Dial("tcp", address, opts...)
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
func (m *manager) setVal(key, val string, expiration int) error {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
args := []interface{}{key, val}
if expiration != -1 {
args = append(args, "EX", expiration)
}
if _, err := conn.Do("SET", args...); err != nil {
return err
}
return nil
}
return errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) getVal(key string) (string, error) {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
val, err := redis.String(conn.Do("GET", key))
if err != nil {
return "", err
}
return val, nil
}
return "", errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) findCachedUsers(query string) ([]*userpb.User, error) {
conn := m.redisPool.Get()
defer conn.Close()
if conn != nil {
query = fmt.Sprintf("%s*%s*", userPrefix, strings.ReplaceAll(strings.ToLower(query), " ", "_"))
keys, err := redis.Strings(conn.Do("KEYS", query))
if err != nil {
return nil, err
}
var args []interface{}
for _, k := range keys {
args = append(args, k)
}
// Fetch the users for all these keys
userStrings, err := redis.Strings(conn.Do("MGET", args...))
if err != nil {
return nil, err
}
userMap := make(map[string]*userpb.User)
for _, user := range userStrings {
u := userpb.User{}
if err = json.Unmarshal([]byte(user), &u); err == nil {
userMap[u.Id.OpaqueId] = &u
}
}
var users []*userpb.User
for _, u := range userMap {
users = append(users, u)
}
return users, nil
}
return nil, errors.New("rest: unable to get connection from redis pool")
}
func (m *manager) fetchCachedUserDetails(uid *userpb.UserId) (*userpb.User, error) {
user, err := m.getVal(userPrefix + usernamePrefix + strings.ToLower(uid.OpaqueId))
if err != nil {
return nil, err
}
u := userpb.User{}
if err = json.Unmarshal([]byte(user), &u); err != nil {
return nil, err
}
return &u, nil
}
func (m *manager) cacheUserDetails(u *userpb.User) error {
encodedUser, err := json.Marshal(&u)
if err != nil {
return err
}
if err = m.setVal(userPrefix+usernamePrefix+strings.ToLower(u.Id.OpaqueId), string(encodedUser), -1); err != nil {
return err
}
if err = m.setVal(userPrefix+userIDPrefix+strings.ToLower(u.Id.OpaqueId), string(encodedUser), -1); err != nil {
return err
}
if u.Mail != "" {
if err = m.setVal(userPrefix+mailPrefix+strings.ToLower(u.Mail), string(encodedUser), -1); err != nil {
return err
}
}
if u.DisplayName != "" {
if err = m.setVal(userPrefix+namePrefix+u.Id.OpaqueId+"_"+strings.ReplaceAll(strings.ToLower(u.DisplayName), " ", "_"), string(encodedUser), -1); err != nil {
return err
}
}
if u.UidNumber != 0 {
if err = m.setVal(userPrefix+uidPrefix+strconv.FormatInt(u.UidNumber, 10), string(encodedUser), -1); err != nil {
return err
}
}
return nil
}
func (m *manager) fetchCachedUserByParam(field, claim string) (*userpb.User, error) {
user, err := m.getVal(userPrefix + field + ":" + strings.ToLower(claim))
if err != nil {
return nil, err
}
u := userpb.User{}
if err = json.Unmarshal([]byte(user), &u); err != nil {
return nil, err
}
return &u, nil
}
func (m *manager) fetchCachedUserGroups(uid *userpb.UserId) ([]string, error) {
groups, err := m.getVal(userPrefix + userGroupsPrefix + strings.ToLower(uid.OpaqueId))
if err != nil {
return nil, err
}
g := []string{}
if err = json.Unmarshal([]byte(groups), &g); err != nil {
return nil, err
}
return g, nil
}
func (m *manager) cacheUserGroups(uid *userpb.UserId, groups []string) error {
g, err := json.Marshal(&groups)
if err != nil {
return err
}
return m.setVal(userPrefix+userGroupsPrefix+strings.ToLower(uid.OpaqueId), string(g), m.conf.UserGroupsCacheExpiration*60)
}
+381
View File
@@ -0,0 +1,381 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rest
import (
"context"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
utils "github.com/cs3org/reva/v2/pkg/cbox/utils"
"github.com/cs3org/reva/v2/pkg/user"
"github.com/cs3org/reva/v2/pkg/user/manager/registry"
"github.com/gomodule/redigo/redis"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
func init() {
registry.Register("rest", New)
}
type manager struct {
conf *config
redisPool *redis.Pool
apiTokenManager *utils.APITokenManager
}
type config struct {
// The address at which the redis server is running
RedisAddress string `mapstructure:"redis_address" docs:"localhost:6379"`
// The username for connecting to the redis server
RedisUsername string `mapstructure:"redis_username" docs:""`
// The password for connecting to the redis server
RedisPassword string `mapstructure:"redis_password" docs:""`
// The time in minutes for which the groups to which a user belongs would be cached
UserGroupsCacheExpiration int `mapstructure:"user_groups_cache_expiration" docs:"5"`
// The OIDC Provider
IDProvider string `mapstructure:"id_provider" docs:"http://cernbox.cern.ch"`
// Base API Endpoint
APIBaseURL string `mapstructure:"api_base_url" docs:"https://authorization-service-api-dev.web.cern.ch"`
// Client ID needed to authenticate
ClientID string `mapstructure:"client_id" docs:"-"`
// Client Secret
ClientSecret string `mapstructure:"client_secret" docs:"-"`
// Endpoint to generate token to access the API
OIDCTokenEndpoint string `mapstructure:"oidc_token_endpoint" docs:"https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token"`
// The target application for which token needs to be generated
TargetAPI string `mapstructure:"target_api" docs:"authorization-service-api"`
// The time in seconds between bulk fetch of user accounts
UserFetchInterval int `mapstructure:"user_fetch_interval" docs:"3600"`
}
func (c *config) init() {
if c.UserGroupsCacheExpiration == 0 {
c.UserGroupsCacheExpiration = 5
}
if c.RedisAddress == "" {
c.RedisAddress = ":6379"
}
if c.APIBaseURL == "" {
c.APIBaseURL = "https://authorization-service-api-dev.web.cern.ch"
}
if c.TargetAPI == "" {
c.TargetAPI = "authorization-service-api"
}
if c.OIDCTokenEndpoint == "" {
c.OIDCTokenEndpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token"
}
if c.IDProvider == "" {
c.IDProvider = "http://cernbox.cern.ch"
}
if c.UserFetchInterval == 0 {
c.UserFetchInterval = 3600
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
return nil, err
}
return c, nil
}
// New returns a user manager implementation that makes calls to the GRAPPA API.
func New(m map[string]interface{}) (user.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
return nil, err
}
return mgr, err
}
func (m *manager) Configure(ml map[string]interface{}) error {
c, err := parseConfig(ml)
if err != nil {
return err
}
c.init()
redisPool := initRedisPool(c.RedisAddress, c.RedisUsername, c.RedisPassword)
apiTokenManager := utils.InitAPITokenManager(c.TargetAPI, c.OIDCTokenEndpoint, c.ClientID, c.ClientSecret)
m.conf = c
m.redisPool = redisPool
m.apiTokenManager = apiTokenManager
// Since we're starting a subroutine which would take some time to execute,
// we can't wait to see if it works before returning the user.Manager object
// TODO: return err if the fetch fails
go m.fetchAllUsers()
return nil
}
func (m *manager) fetchAllUsers() {
_ = m.fetchAllUserAccounts()
ticker := time.NewTicker(time.Duration(m.conf.UserFetchInterval) * time.Second)
work := make(chan os.Signal, 1)
signal.Notify(work, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT)
for {
select {
case <-work:
return
case <-ticker.C:
_ = m.fetchAllUserAccounts()
}
}
}
func (m *manager) fetchAllUserAccounts() error {
ctx := context.Background()
url := fmt.Sprintf("%s/api/v1.0/Identity?field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type", m.conf.APIBaseURL)
for url != "" {
result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false)
if err != nil {
return err
}
responseData, ok := result["data"].([]interface{})
if !ok {
return errors.New("rest: error in type assertion")
}
for _, usr := range responseData {
userData, ok := usr.(map[string]interface{})
if !ok {
continue
}
_, err = m.parseAndCacheUser(ctx, userData)
if err != nil {
continue
}
}
url = ""
if pagination, ok := result["pagination"].(map[string]interface{}); ok {
if links, ok := pagination["links"].(map[string]interface{}); ok {
if next, ok := links["next"].(string); ok {
url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, next)
}
}
}
}
return nil
}
func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) (*userpb.User, error) {
upn, ok := userData["upn"].(string)
if !ok {
return nil, errors.New("rest: missing upn in user data")
}
mail, _ := userData["primaryAccountEmail"].(string)
name, _ := userData["displayName"].(string)
uidNumber, _ := userData["uid"].(float64)
gidNumber, _ := userData["gid"].(float64)
t, _ := userData["type"].(string)
userType := getUserType(t, upn)
userID := &userpb.UserId{
OpaqueId: upn,
Idp: m.conf.IDProvider,
Type: userType,
}
u := &userpb.User{
Id: userID,
Username: upn,
Mail: mail,
DisplayName: name,
UidNumber: int64(uidNumber),
GidNumber: int64(gidNumber),
}
if err := m.cacheUserDetails(u); err != nil {
log.Error().Err(err).Msg("rest: error caching user details")
}
return u, nil
}
func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) {
u, err := m.fetchCachedUserDetails(uid)
if err != nil {
return nil, err
}
if !skipFetchingGroups {
userGroups, err := m.GetUserGroups(ctx, uid)
if err != nil {
return nil, err
}
u.Groups = userGroups
}
return u, nil
}
func (m *manager) GetUserByClaim(ctx context.Context, claim, value string, skipFetchingGroups bool) (*userpb.User, error) {
u, err := m.fetchCachedUserByParam(claim, value)
if err != nil {
return nil, err
}
if !skipFetchingGroups {
userGroups, err := m.GetUserGroups(ctx, u.Id)
if err != nil {
return nil, err
}
u.Groups = userGroups
}
return u, nil
}
func (m *manager) FindUsers(ctx context.Context, query string, skipFetchingGroups bool) ([]*userpb.User, error) {
// Look at namespaces filters. If the query starts with:
// "a" => look into primary/secondary/service accounts
// "l" => look into lightweight/federated accounts
// none => look into primary
parts := strings.SplitN(query, ":", 2)
var namespace string
if len(parts) == 2 {
// the query contains a namespace filter
namespace, query = parts[0], parts[1]
}
users, err := m.findCachedUsers(query)
if err != nil {
return nil, err
}
userSlice := []*userpb.User{}
var accountsFilters []userpb.UserType
switch namespace {
case "":
accountsFilters = []userpb.UserType{userpb.UserType_USER_TYPE_PRIMARY}
case "a":
accountsFilters = []userpb.UserType{userpb.UserType_USER_TYPE_PRIMARY, userpb.UserType_USER_TYPE_SECONDARY, userpb.UserType_USER_TYPE_SERVICE}
case "l":
accountsFilters = []userpb.UserType{userpb.UserType_USER_TYPE_LIGHTWEIGHT, userpb.UserType_USER_TYPE_FEDERATED}
}
for _, u := range users {
if isUserAnyType(u, accountsFilters) {
userSlice = append(userSlice, u)
}
}
return userSlice, nil
}
// isUserAnyType returns true if the user's type is one of types list
func isUserAnyType(user *userpb.User, types []userpb.UserType) bool {
for _, t := range types {
if user.GetId().Type == t {
return true
}
}
return false
}
func (m *manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]string, error) {
groups, err := m.fetchCachedUserGroups(uid)
if err == nil {
return groups, nil
}
url := fmt.Sprintf("%s/api/v1.0/Identity/%s/groups?recursive=true", m.conf.APIBaseURL, uid.OpaqueId)
result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false)
if err != nil {
return nil, err
}
groupData := result["data"].([]interface{})
groups = []string{}
for _, g := range groupData {
groupInfo, ok := g.(map[string]interface{})
if !ok {
return nil, errors.New("rest: error in type assertion")
}
name, ok := groupInfo["displayName"].(string)
if ok {
groups = append(groups, name)
}
}
if err = m.cacheUserGroups(uid, groups); err != nil {
log := appctx.GetLogger(ctx)
log.Error().Err(err).Msg("rest: error caching user groups")
}
return groups, nil
}
func (m *manager) IsInGroup(ctx context.Context, uid *userpb.UserId, group string) (bool, error) {
userGroups, err := m.GetUserGroups(ctx, uid)
if err != nil {
return false, err
}
for _, g := range userGroups {
if group == g {
return true, nil
}
}
return false, nil
}
func getUserType(userType, upn string) userpb.UserType {
var t userpb.UserType
switch userType {
case "Application":
t = userpb.UserType_USER_TYPE_APPLICATION
case "Service":
t = userpb.UserType_USER_TYPE_SERVICE
case "Secondary":
t = userpb.UserType_USER_TYPE_SECONDARY
case "Person":
switch {
case strings.HasPrefix(upn, "guest"):
t = userpb.UserType_USER_TYPE_LIGHTWEIGHT
case strings.Contains(upn, "@"):
t = userpb.UserType_USER_TYPE_FEDERATED
default:
t = userpb.UserType_USER_TYPE_PRIMARY
}
default:
t = userpb.UserType_USER_TYPE_INVALID
}
return t
}
+255
View File
@@ -0,0 +1,255 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package utils
import (
"strings"
"time"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/conversions"
)
// DBShare stores information about user and public shares.
type DBShare struct {
ID string
UIDOwner string
UIDInitiator string
Prefix string
ItemSource string
ItemType string
ShareWith string
Token string
Expiration string
Permissions int
ShareType int
ShareName string
STime int
FileTarget string
State int
}
// FormatGrantee formats a CS3API grantee to a string
func FormatGrantee(g *provider.Grantee) (int, string) {
var granteeType int
var formattedID string
switch g.Type {
case provider.GranteeType_GRANTEE_TYPE_USER:
granteeType = 0
formattedID = FormatUserID(g.GetUserId())
case provider.GranteeType_GRANTEE_TYPE_GROUP:
granteeType = 1
formattedID = FormatGroupID(g.GetGroupId())
default:
granteeType = -1
}
return granteeType, formattedID
}
// ExtractGrantee retrieves the CS3API grantee from a formatted string
func ExtractGrantee(t int, g string) *provider.Grantee {
var grantee provider.Grantee
switch t {
case 0:
grantee.Type = provider.GranteeType_GRANTEE_TYPE_USER
grantee.Id = &provider.Grantee_UserId{UserId: ExtractUserID(g)}
case 1:
grantee.Type = provider.GranteeType_GRANTEE_TYPE_GROUP
grantee.Id = &provider.Grantee_GroupId{GroupId: ExtractGroupID(g)}
default:
grantee.Type = provider.GranteeType_GRANTEE_TYPE_INVALID
}
return &grantee
}
// ResourceTypeToItem maps a resource type to a string
func ResourceTypeToItem(r provider.ResourceType) string {
switch r {
case provider.ResourceType_RESOURCE_TYPE_FILE:
return "file"
case provider.ResourceType_RESOURCE_TYPE_CONTAINER:
return "folder"
case provider.ResourceType_RESOURCE_TYPE_REFERENCE:
return "reference"
case provider.ResourceType_RESOURCE_TYPE_SYMLINK:
return "symlink"
default:
return ""
}
}
// ResourceTypeToItemInt maps a resource type to an integer
func ResourceTypeToItemInt(r provider.ResourceType) int {
switch r {
case provider.ResourceType_RESOURCE_TYPE_CONTAINER:
return 0
case provider.ResourceType_RESOURCE_TYPE_FILE:
return 1
default:
return -1
}
}
// SharePermToInt maps read/write permissions to an integer
func SharePermToInt(p *provider.ResourcePermissions) int {
var perm int
switch {
case p.InitiateFileUpload && !p.InitiateFileDownload:
perm = 4
case p.InitiateFileUpload:
perm = 15
case p.InitiateFileDownload:
perm = 1
}
// TODO map denials and resharing; currently, denials are mapped to 0
return perm
}
// IntTosharePerm retrieves read/write permissions from an integer
func IntTosharePerm(p int, itemType string) *provider.ResourcePermissions {
switch p {
case 1:
return conversions.NewViewerRole().CS3ResourcePermissions()
case 15:
if itemType == "folder" {
return conversions.NewEditorRole().CS3ResourcePermissions()
}
return conversions.NewFileEditorRole().CS3ResourcePermissions()
case 4:
return conversions.NewUploaderRole().CS3ResourcePermissions()
default:
// TODO we may have other options, for now this is a denial
return &provider.ResourcePermissions{}
}
}
// IntToShareState retrieves the received share state from an integer
func IntToShareState(g int) collaboration.ShareState {
switch g {
case 0:
return collaboration.ShareState_SHARE_STATE_PENDING
case 1:
return collaboration.ShareState_SHARE_STATE_ACCEPTED
case -1:
return collaboration.ShareState_SHARE_STATE_REJECTED
default:
return collaboration.ShareState_SHARE_STATE_INVALID
}
}
// FormatUserID formats a CS3API user ID to a string
func FormatUserID(u *userpb.UserId) string {
return u.OpaqueId
}
// ExtractUserID retrieves a CS3API user ID from a string
func ExtractUserID(u string) *userpb.UserId {
t := userpb.UserType_USER_TYPE_PRIMARY
if strings.HasPrefix(u, "guest:") {
t = userpb.UserType_USER_TYPE_LIGHTWEIGHT
} else if strings.Contains(u, "@") {
t = userpb.UserType_USER_TYPE_FEDERATED
}
return &userpb.UserId{OpaqueId: u, Type: t}
}
// FormatGroupID formats a CS3API group ID to a string
func FormatGroupID(u *grouppb.GroupId) string {
return u.OpaqueId
}
// ExtractGroupID retrieves a CS3API group ID from a string
func ExtractGroupID(u string) *grouppb.GroupId {
return &grouppb.GroupId{OpaqueId: u}
}
// ConvertToCS3Share converts a DBShare to a CS3API collaboration share
func ConvertToCS3Share(s DBShare) *collaboration.Share {
ts := &typespb.Timestamp{
Seconds: uint64(s.STime),
}
return &collaboration.Share{
Id: &collaboration.ShareId{
OpaqueId: s.ID,
},
//ResourceId: &provider.Reference{StorageId: s.Prefix, NodeId: s.ItemSource},
ResourceId: &provider.ResourceId{
SpaceId: s.Prefix,
OpaqueId: s.ItemSource,
},
Permissions: &collaboration.SharePermissions{Permissions: IntTosharePerm(s.Permissions, s.ItemType)},
Grantee: ExtractGrantee(s.ShareType, s.ShareWith),
Owner: ExtractUserID(s.UIDOwner),
Creator: ExtractUserID(s.UIDInitiator),
Ctime: ts,
Mtime: ts,
}
}
// ConvertToCS3ReceivedShare converts a DBShare to a CS3API collaboration received share
func ConvertToCS3ReceivedShare(s DBShare) *collaboration.ReceivedShare {
return &collaboration.ReceivedShare{
Share: ConvertToCS3Share(s),
State: IntToShareState(s.State),
MountPoint: &provider.Reference{Path: strings.TrimLeft(s.FileTarget, "/")},
}
}
// ConvertToCS3PublicShare converts a DBShare to a CS3API public share
func ConvertToCS3PublicShare(s DBShare) *link.PublicShare {
ts := &typespb.Timestamp{
Seconds: uint64(s.STime),
}
pwd := false
if s.ShareWith != "" {
pwd = true
}
var expires *typespb.Timestamp
if s.Expiration != "" {
t, err := time.Parse("2006-01-02 15:04:05", s.Expiration)
if err == nil {
expires = &typespb.Timestamp{
Seconds: uint64(t.Unix()),
}
}
}
return &link.PublicShare{
Id: &link.PublicShareId{
OpaqueId: s.ID,
},
ResourceId: &provider.ResourceId{
SpaceId: s.Prefix,
OpaqueId: s.ItemSource,
},
Permissions: &link.PublicSharePermissions{Permissions: IntTosharePerm(s.Permissions, s.ItemType)},
Owner: ExtractUserID(s.UIDOwner),
Creator: ExtractUserID(s.UIDInitiator),
Token: s.Token,
DisplayName: s.ShareName,
PasswordProtected: pwd,
Expiration: expires,
Ctime: ts,
Mtime: ts,
}
}
+172
View File
@@ -0,0 +1,172 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package utils
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/cs3org/reva/v2/pkg/rhttp"
)
// APITokenManager stores config related to api management
type APITokenManager struct {
oidcToken OIDCToken
conf *config
client *http.Client
}
// OIDCToken stores the OIDC token used to authenticate requests to the REST API service
type OIDCToken struct {
sync.Mutex // concurrent access to apiToken and tokenExpirationTime
apiToken string
tokenExpirationTime time.Time
}
type config struct {
TargetAPI string
OIDCTokenEndpoint string
ClientID string
ClientSecret string
}
// InitAPITokenManager initializes a new APITokenManager
func InitAPITokenManager(targetAPI, oidcTokenEndpoint, clientID, clientSecret string) *APITokenManager {
return &APITokenManager{
conf: &config{
TargetAPI: targetAPI,
OIDCTokenEndpoint: oidcTokenEndpoint,
ClientID: clientID,
ClientSecret: clientSecret,
},
client: rhttp.GetHTTPClient(
rhttp.Timeout(10*time.Second),
rhttp.Insecure(true),
),
}
}
func (a *APITokenManager) renewAPIToken(ctx context.Context, forceRenewal bool) error {
// Received tokens have an expiration time of 20 minutes.
// Take a couple of seconds as buffer time for the API call to complete
if forceRenewal || a.oidcToken.tokenExpirationTime.Before(time.Now().Add(time.Second*time.Duration(2))) {
token, expiration, err := a.getAPIToken(ctx)
if err != nil {
return err
}
a.oidcToken.Lock()
defer a.oidcToken.Unlock()
a.oidcToken.apiToken = token
a.oidcToken.tokenExpirationTime = expiration
}
return nil
}
func (a *APITokenManager) getAPIToken(ctx context.Context) (string, time.Time, error) {
params := url.Values{
"grant_type": {"client_credentials"},
"audience": {a.conf.TargetAPI},
}
httpReq, err := http.NewRequest("POST", a.conf.OIDCTokenEndpoint, strings.NewReader(params.Encode()))
if err != nil {
return "", time.Time{}, err
}
httpReq.SetBasicAuth(a.conf.ClientID, a.conf.ClientSecret)
httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
httpRes, err := a.client.Do(httpReq)
if err != nil {
return "", time.Time{}, err
}
defer httpRes.Body.Close()
body, err := io.ReadAll(httpRes.Body)
if err != nil {
return "", time.Time{}, err
}
if httpRes.StatusCode < 200 || httpRes.StatusCode > 299 {
return "", time.Time{}, errors.New("rest: get token endpoint returned " + httpRes.Status)
}
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return "", time.Time{}, err
}
expirationSecs := result["expires_in"].(float64)
expirationTime := time.Now().Add(time.Second * time.Duration(expirationSecs))
return result["access_token"].(string), expirationTime, nil
}
// SendAPIGetRequest makes an API GET Request to the passed URL
func (a *APITokenManager) SendAPIGetRequest(ctx context.Context, url string, forceRenewal bool) (map[string]interface{}, error) {
err := a.renewAPIToken(ctx, forceRenewal)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
// We don't need to take the lock when reading apiToken, because if we reach here,
// the token is valid at least for a couple of seconds. Even if another request modifies
// the token and expiration time while this request is in progress, the current token will still be valid.
httpReq.Header.Set("Authorization", "Bearer "+a.oidcToken.apiToken)
httpRes, err := a.client.Do(httpReq)
if err != nil {
return nil, err
}
defer httpRes.Body.Close()
if httpRes.StatusCode == http.StatusUnauthorized {
// The token is no longer valid, try renewing it
return a.SendAPIGetRequest(ctx, url, true)
}
if httpRes.StatusCode < 200 || httpRes.StatusCode > 299 {
return nil, errors.New("rest: API request returned " + httpRes.Status)
}
body, err := io.ReadAll(httpRes.Body)
if err != nil {
return nil, err
}
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, err
}
return result, nil
}
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ctx
import (
"context"
"strings"
ua "github.com/mileusna/useragent"
"google.golang.org/grpc/metadata"
)
// UserAgentHeader is the header used for the user agent
const (
UserAgentHeader = "x-user-agent"
WebUserAgent = "web"
GrpcUserAgent = "grpc"
MobileUserAgent = "mobile"
DesktopUserAgent = "desktop"
)
// ContextGetUserAgent returns the user agent if set in the given context.
// see https://github.com/grpc/grpc-go/issues/1100
func ContextGetUserAgent(ctx context.Context) (*ua.UserAgent, bool) {
if userAgentStr, ok := ContextGetUserAgentString(ctx); ok {
userAgent := ua.Parse(userAgentStr)
return &userAgent, true
}
return nil, false
}
// ContextGetUserAgentString returns the user agent string if set in the given context.
func ContextGetUserAgentString(ctx context.Context) (string, bool) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return "", false
}
userAgentLst, ok := md[UserAgentHeader]
if !ok {
userAgentLst, ok = md["user-agent"]
if !ok {
return "", false
}
}
if len(userAgentLst) == 0 {
return "", false
}
return userAgentLst[0], true
}
// ContextGetUserAgentCategory returns the category of the user agent
// (i.e. if it is a web, mobile, desktop or grpc user agent)
func ContextGetUserAgentCategory(ctx context.Context) (string, bool) {
agent, ok := ContextGetUserAgent(ctx)
if !ok {
return "", false
}
switch {
case isWeb(agent):
return WebUserAgent, true
case isMobile(agent):
return MobileUserAgent, true
case isDesktop(agent):
return DesktopUserAgent, true
case isGRPC(agent):
return GrpcUserAgent, true
default:
return "", false
}
}
func isWeb(ua *ua.UserAgent) bool {
return ua.IsChrome() || ua.IsEdge() || ua.IsFirefox() || ua.IsSafari() ||
ua.IsInternetExplorer() || ua.IsOpera() || ua.IsOperaMini()
}
// isMobile returns true if the useragent is generated by the mobile
func isMobile(ua *ua.UserAgent) bool {
// workaround as the library does not recognise iOS string inside the user agent
isIOS := ua.IsIOS() || strings.Contains(ua.String, "iOS")
return !isWeb(ua) && (ua.IsAndroid() || isIOS)
}
// isDesktop returns true if the useragent is generated by a desktop application
func isDesktop(ua *ua.UserAgent) bool {
return ua.Desktop && !isWeb(ua)
}
// isGRPC returns true if the useragent is generated by a grpc client
func isGRPC(ua *ua.UserAgent) bool {
return strings.HasPrefix(ua.Name, "grpc")
}
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ctx
import (
"context"
)
// ContextGetLockID returns the lock id if set in the given context.
func ContextGetLockID(ctx context.Context) (string, bool) {
u, ok := ctx.Value(lockIDKey).(string)
return u, ok
}
// ContextSetLockID stores the lock id in the context.
func ContextSetLockID(ctx context.Context, t string) context.Context {
return context.WithValue(ctx, lockIDKey, t)
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ctx
import (
"context"
auth "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1"
)
// ContextGetScopes returns the scopes if set in the given context.
func ContextGetScopes(ctx context.Context) (map[string]*auth.Scope, bool) {
s, ok := ctx.Value(scopeKey).(map[string]*auth.Scope)
return s, ok
}
// ContextSetScopes stores the scopes in the context.
func ContextSetScopes(ctx context.Context, s map[string]*auth.Scope) context.Context {
return context.WithValue(ctx, scopeKey, s)
}
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ctx
import (
"context"
)
// TokenHeader is the header to be used across grpc and http services
// to forward the access token.
const TokenHeader = "x-access-token"
// ContextGetToken returns the token if set in the given context.
func ContextGetToken(ctx context.Context) (string, bool) {
u, ok := ctx.Value(tokenKey).(string)
return u, ok
}
// ContextMustGetToken panics if token is not in context.
func ContextMustGetToken(ctx context.Context) string {
u, ok := ContextGetToken(ctx)
if !ok {
panic("token not found in context")
}
return u
}
// ContextSetToken stores the token in the context.
func ContextSetToken(ctx context.Context, t string) context.Context {
return context.WithValue(ctx, tokenKey, t)
}
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ctx
import (
"context"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
)
type key int
const (
userKey key = iota
tokenKey
idKey
lockIDKey
scopeKey
)
// ContextGetUser returns the user if set in the given context.
func ContextGetUser(ctx context.Context) (*userpb.User, bool) {
u, ok := ctx.Value(userKey).(*userpb.User)
return u, ok
}
// ContextMustGetUser panics if user is not in context.
func ContextMustGetUser(ctx context.Context) *userpb.User {
u, ok := ContextGetUser(ctx)
if !ok {
panic("user not found in context")
}
return u
}
// ContextSetUser stores the user in the context.
func ContextSetUser(ctx context.Context, u *userpb.User) context.Context {
return context.WithValue(ctx, userKey, u)
}
// ContextGetUserID returns the user if set in the given context.
func ContextGetUserID(ctx context.Context) (*userpb.UserId, bool) {
u, ok := ctx.Value(idKey).(*userpb.UserId)
return u, ok
}
// ContextSetUserID stores the userid in the context.
func ContextSetUserID(ctx context.Context, id *userpb.UserId) context.Context {
return context.WithValue(ctx, idKey, id)
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package datatx
import (
"context"
datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1"
)
// Manager the interface any transfer driver should implement
type Manager interface {
// StartTransfer initiates a transfer job and returns a TxInfo object including a unique transfer id, and error if any.
StartTransfer(ctx context.Context, srcRemote string, srcPath string, srcToken string, destRemote string, destPath string, destToken string) (*datatx.TxInfo, error)
// GetTransferStatus returns a TxInfo object including the current status, and error if any.
GetTransferStatus(ctx context.Context, transferID string) (*datatx.TxInfo, error)
// CancelTransfer cancels the transfer and returns a TxInfo object and error if any.
CancelTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error)
// RetryTransfer retries the transfer and returns a TxInfo object and error if any.
// Note that tokens must still be valid.
RetryTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error)
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load datatx drivers.
_ "github.com/cs3org/reva/v2/pkg/datatx/manager/rclone"
// Add your own here
)
+831
View File
@@ -0,0 +1,831 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rclone
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strconv"
"sync"
"time"
datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1"
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
txdriver "github.com/cs3org/reva/v2/pkg/datatx"
registry "github.com/cs3org/reva/v2/pkg/datatx/manager/registry"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/google/uuid"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("rclone", New)
}
func (c *config) init(m map[string]interface{}) {
// set sane defaults
if c.File == "" {
c.File = "/var/tmp/reva/datatx-transfers.json"
}
if c.JobStatusCheckInterval == 0 {
c.JobStatusCheckInterval = 2000
}
if c.JobTimeout == 0 {
c.JobTimeout = 50000
}
}
type config struct {
Endpoint string `mapstructure:"endpoint"`
AuthUser string `mapstructure:"auth_user"` // rclone basicauth user
AuthPass string `mapstructure:"auth_pass"` // rclone basicauth pass
File string `mapstructure:"file"`
JobStatusCheckInterval int `mapstructure:"job_status_check_interval"`
JobTimeout int `mapstructure:"job_timeout"`
}
type rclone struct {
config *config
client *http.Client
pDriver *pDriver
}
type rcloneHTTPErrorRes struct {
Error string `json:"error"`
Input map[string]interface{} `json:"input"`
Path string `json:"path"`
Status int `json:"status"`
}
type transferModel struct {
File string
Transfers map[string]*transfer `json:"transfers"`
}
// persistency driver
type pDriver struct {
sync.Mutex // concurrent access to the file
model *transferModel
}
type transfer struct {
TransferID string
JobID int64
TransferStatus datatx.Status
SrcToken string
SrcRemote string
SrcPath string
DestToken string
DestRemote string
DestPath string
Ctime string
}
// txEndStatuses final statuses that cannot be changed anymore
var txEndStatuses = map[string]int32{
"STATUS_INVALID": 0,
"STATUS_DESTINATION_NOT_FOUND": 1,
"STATUS_TRANSFER_COMPLETE": 6,
"STATUS_TRANSFER_FAILED": 7,
"STATUS_TRANSFER_CANCELLED": 8,
"STATUS_TRANSFER_CANCEL_FAILED": 9,
"STATUS_TRANSFER_EXPIRED": 10,
}
// New returns a new rclone driver
func New(m map[string]interface{}) (txdriver.Manager, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
c.init(m)
// TODO insecure should be configurable
client := rhttp.GetHTTPClient(rhttp.Insecure(true))
// The persistency driver
// Load or create 'db'
model, err := loadOrCreate(c.File)
if err != nil {
err = errors.Wrap(err, "error loading the file containing the transfers")
return nil, err
}
pDriver := &pDriver{
model: model,
}
return &rclone{
config: c,
client: client,
pDriver: pDriver,
}, nil
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
func loadOrCreate(file string) (*transferModel, error) {
_, err := os.Stat(file)
if os.IsNotExist(err) {
if err := os.WriteFile(file, []byte("{}"), 0700); err != nil {
err = errors.Wrap(err, "error creating the transfers storage file: "+file)
return nil, err
}
}
fd, err := os.OpenFile(file, os.O_CREATE, 0644)
if err != nil {
err = errors.Wrap(err, "error opening the transfers storage file: "+file)
return nil, err
}
defer fd.Close()
data, err := io.ReadAll(fd)
if err != nil {
err = errors.Wrap(err, "error reading the data")
return nil, err
}
model := &transferModel{}
if err := json.Unmarshal(data, model); err != nil {
err = errors.Wrap(err, "error decoding transfers data to json")
return nil, err
}
if model.Transfers == nil {
model.Transfers = make(map[string]*transfer)
}
model.File = file
return model, nil
}
// saveTransfer saves the transfer. If an error is specified than that error will be returned, possibly wrapped with additional errors.
func (m *transferModel) saveTransfer(e error) error {
data, err := json.Marshal(m)
if err != nil {
e = errors.Wrap(err, "error encoding transfer data to json")
return e
}
if err := os.WriteFile(m.File, data, 0644); err != nil {
e = errors.Wrap(err, "error writing transfer data to file: "+m.File)
return e
}
return e
}
// StartTransfer initiates a transfer job and returns a TxInfo object that includes a unique transfer id.
func (driver *rclone) StartTransfer(ctx context.Context, srcRemote string, srcPath string, srcToken string, destRemote string, destPath string, destToken string) (*datatx.TxInfo, error) {
return driver.startJob(ctx, "", srcRemote, srcPath, srcToken, destRemote, destPath, destToken)
}
// startJob starts a transfer job. Retries a previous job if transferID is specified.
func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote string, srcPath string, srcToken string, destRemote string, destPath string, destToken string) (*datatx.TxInfo, error) {
logger := appctx.GetLogger(ctx)
driver.pDriver.Lock()
defer driver.pDriver.Unlock()
var txID string
var cTime *typespb.Timestamp
if transferID == "" {
txID = uuid.New().String()
cTime = &typespb.Timestamp{Seconds: uint64(time.Now().Unix())}
} else { // restart existing transfer if transferID is specified
logger.Debug().Msgf("Restarting transfer (txID: %s)", transferID)
txID = transferID
transfer, err := driver.pDriver.model.getTransfer(txID)
if err != nil {
err = errors.Wrap(err, "rclone: error retrying transfer (transferID: "+txID+")")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: nil,
}, err
}
seconds, _ := strconv.ParseInt(transfer.Ctime, 10, 64)
cTime = &typespb.Timestamp{Seconds: uint64(seconds)}
_, endStatusFound := txEndStatuses[transfer.TransferStatus.String()]
if !endStatusFound {
err := errors.New("rclone: transfer still running, unable to restart")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, err
}
srcToken = transfer.SrcToken
srcRemote = transfer.SrcRemote
srcPath = transfer.SrcPath
destToken = transfer.DestToken
destRemote = transfer.DestRemote
destPath = transfer.DestPath
delete(driver.pDriver.model.Transfers, txID)
}
transferStatus := datatx.Status_STATUS_TRANSFER_NEW
transfer := &transfer{
TransferID: txID,
JobID: int64(-1),
TransferStatus: transferStatus,
SrcToken: srcToken,
SrcRemote: srcRemote,
SrcPath: srcPath,
DestToken: destToken,
DestRemote: destRemote,
DestPath: destPath,
Ctime: fmt.Sprint(cTime.Seconds), // TODO do we need nanos here?
}
driver.pDriver.model.Transfers[txID] = transfer
type rcloneAsyncReqJSON struct {
SrcFs string `json:"srcFs"`
// SrcToken string `json:"srcToken"`
DstFs string `json:"dstFs"`
// DstToken string `json:"destToken"`
Async bool `json:"_async"`
}
srcFs := fmt.Sprintf(":webdav,headers=\"x-access-token,%v\",url=\"%v\":%v", srcToken, srcRemote, srcPath)
dstFs := fmt.Sprintf(":webdav,headers=\"x-access-token,%v\",url=\"%v\":%v", destToken, destRemote, destPath)
rcloneReq := &rcloneAsyncReqJSON{
SrcFs: srcFs,
DstFs: dstFs,
Async: true,
}
data, err := json.Marshal(rcloneReq)
if err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer: error marshalling rclone req data")
transfer.TransferStatus = datatx.Status_STATUS_INVALID
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
transferFileMethod := "/sync/copy"
remotePathIsFolder, err := driver.remotePathIsFolder(srcRemote, srcPath, srcToken)
if err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer: error stating src path")
transfer.TransferStatus = datatx.Status_STATUS_INVALID
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
if !remotePathIsFolder {
err = errors.Wrap(err, "rclone: error pulling transfer: path is a file, only folder transfer is implemented")
transfer.TransferStatus = datatx.Status_STATUS_INVALID
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
u, err := url.Parse(driver.config.Endpoint)
if err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer: error parsing driver endpoint")
transfer.TransferStatus = datatx.Status_STATUS_INVALID
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
u.Path = path.Join(u.Path, transferFileMethod)
requestURL := u.String()
req, err := http.NewRequest("POST", requestURL, bytes.NewReader(data))
if err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer: error framing post request")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
req.Header.Set("Content-Type", "application/json")
req.SetBasicAuth(driver.config.AuthUser, driver.config.AuthPass)
res, err := driver.client.Do(req)
if err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer: error sending post request")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
var errorResData rcloneHTTPErrorRes
if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil {
err = errors.Wrap(err, "rclone driver: error decoding response data")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
e := errors.New("rclone: rclone request responded with error, " + fmt.Sprintf(" status: %v, error: %v", errorResData.Status, errorResData.Error))
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(e)
}
type rcloneAsyncResJSON struct {
JobID int64 `json:"jobid"`
}
var resData rcloneAsyncResJSON
if err = json.NewDecoder(res.Body).Decode(&resData); err != nil {
err = errors.Wrap(err, "rclone: error decoding response data")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transfer.TransferStatus,
Ctime: cTime,
}, driver.pDriver.model.saveTransfer(err)
}
transfer.JobID = resData.JobID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
err = errors.Wrap(err, "rclone: error pulling transfer")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: datatx.Status_STATUS_INVALID,
Ctime: cTime,
}, err
}
// start separate dedicated process to periodically check the transfer progress
go func() {
// runs for as long as no end state or time out has been reached
startTimeMs := time.Now().Nanosecond() / 1000
timeout := driver.config.JobTimeout
driver.pDriver.Lock()
defer driver.pDriver.Unlock()
for {
transfer, err := driver.pDriver.model.getTransfer(txID)
if err != nil {
transfer.TransferStatus = datatx.Status_STATUS_INVALID
err = driver.pDriver.model.saveTransfer(err)
logger.Error().Err(err).Msgf("rclone driver: unable to retrieve transfer with id: %v", txID)
break
}
// check for end status first
_, endStatusFound := txEndStatuses[transfer.TransferStatus.String()]
if endStatusFound {
logger.Info().Msgf("rclone driver: transfer endstatus reached: %v", transfer.TransferStatus)
break
}
// check for possible timeout and if true were done
currentTimeMs := time.Now().Nanosecond() / 1000
timePastMs := currentTimeMs - startTimeMs
if timePastMs > timeout {
logger.Info().Msgf("rclone driver: transfer timed out: %vms (timeout = %v)", timePastMs, timeout)
// set status to EXPIRED and save
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_EXPIRED
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
jobID := transfer.JobID
type rcloneStatusReqJSON struct {
JobID int64 `json:"jobid"`
}
rcloneStatusReq := &rcloneStatusReqJSON{
JobID: jobID,
}
data, err := json.Marshal(rcloneStatusReq)
if err != nil {
logger.Error().Err(err).Msgf("rclone driver: marshalling request failed: %v", err)
transfer.TransferStatus = datatx.Status_STATUS_INVALID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
transferFileMethod := "/job/status"
u, err := url.Parse(driver.config.Endpoint)
if err != nil {
logger.Error().Err(err).Msgf("rclone driver: could not parse driver endpoint: %v", err)
transfer.TransferStatus = datatx.Status_STATUS_INVALID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
u.Path = path.Join(u.Path, transferFileMethod)
requestURL := u.String()
req, err := http.NewRequest("POST", requestURL, bytes.NewReader(data))
if err != nil {
logger.Error().Err(err).Msgf("rclone driver: error framing post request: %v", err)
transfer.TransferStatus = datatx.Status_STATUS_INVALID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
req.Header.Set("Content-Type", "application/json")
req.SetBasicAuth(driver.config.AuthUser, driver.config.AuthPass)
res, err := driver.client.Do(req)
if err != nil {
logger.Error().Err(err).Msgf("rclone driver: error sending post request: %v", err)
transfer.TransferStatus = datatx.Status_STATUS_INVALID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
var errorResData rcloneHTTPErrorRes
if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil {
err = errors.Wrap(err, "rclone driver: error decoding response data")
logger.Error().Err(err).Msgf("rclone driver: error reading response body: %v", err)
}
logger.Error().Err(err).Msgf("rclone driver: rclone request responded with error, status: %v, error: %v", errorResData.Status, errorResData.Error)
transfer.TransferStatus = datatx.Status_STATUS_INVALID
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err)
}
break
}
type rcloneStatusResJSON struct {
Finished bool `json:"finished"`
Success bool `json:"success"`
ID int64 `json:"id"`
Error string `json:"error"`
Group string `json:"group"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
Duration float64 `json:"duration"`
// think we don't need this
// "output": {} // output of the job as would have been returned if called synchronously
}
var resData rcloneStatusResJSON
if err = json.NewDecoder(res.Body).Decode(&resData); err != nil {
logger.Error().Err(err).Msgf("rclone driver: error decoding response data: %v", err)
break
}
if resData.Error != "" {
logger.Error().Err(err).Msgf("rclone driver: rclone responded with error: %v", resData.Error)
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err)
break
}
break
}
// transfer complete
if resData.Finished && resData.Success {
logger.Info().Msg("rclone driver: transfer job finished")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_COMPLETE
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err)
break
}
break
}
// transfer completed unsuccessfully without error
if resData.Finished && !resData.Success {
logger.Info().Msgf("rclone driver: transfer job failed")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err)
break
}
break
}
// transfer not yet finished: continue
if !resData.Finished {
logger.Info().Msgf("rclone driver: transfer job in progress")
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_IN_PROGRESS
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err)
break
}
}
<-time.After(time.Millisecond * time.Duration(driver.config.JobStatusCheckInterval))
}
}()
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: txID},
Status: transferStatus,
Ctime: cTime,
}, nil
}
// GetTransferStatus returns the status of the transfer with the specified job id
func (driver *rclone) GetTransferStatus(ctx context.Context, transferID string) (*datatx.TxInfo, error) {
transfer, err := driver.pDriver.model.getTransfer(transferID)
if err != nil {
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: nil,
}, err
}
cTime, _ := strconv.ParseInt(transfer.Ctime, 10, 64)
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: transfer.TransferStatus,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, nil
}
// CancelTransfer cancels the transfer with the specified transfer id
func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error) {
transfer, err := driver.pDriver.model.getTransfer(transferID)
if err != nil {
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: nil,
}, err
}
cTime, _ := strconv.ParseInt(transfer.Ctime, 10, 64)
_, endStatusFound := txEndStatuses[transfer.TransferStatus.String()]
if endStatusFound {
err := errors.New("rclone driver: transfer already in end state")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
// rcloneStop the rclone job/stop method json request
type rcloneStopRequest struct {
JobID int64 `json:"jobid"`
}
rcloneCancelTransferReq := &rcloneStopRequest{
JobID: transfer.JobID,
}
data, err := json.Marshal(rcloneCancelTransferReq)
if err != nil {
err = errors.Wrap(err, "rclone driver: error marshalling rclone req data")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
transferFileMethod := "/job/stop"
u, err := url.Parse(driver.config.Endpoint)
if err != nil {
err = errors.Wrap(err, "rclone driver: error parsing driver endpoint")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
u.Path = path.Join(u.Path, transferFileMethod)
requestURL := u.String()
req, err := http.NewRequest("POST", requestURL, bytes.NewReader(data))
if err != nil {
err = errors.Wrap(err, "rclone driver: error framing post request")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
req.Header.Set("Content-Type", "application/json")
req.SetBasicAuth(driver.config.AuthUser, driver.config.AuthPass)
res, err := driver.client.Do(req)
if err != nil {
err = errors.Wrap(err, "rclone driver: error sending post request")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
var errorResData rcloneHTTPErrorRes
if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil {
err = errors.Wrap(err, "rclone driver: error decoding response data")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
err = errors.Wrap(errors.Errorf("status: %v, error: %v", errorResData.Status, errorResData.Error), "rclone driver: rclone request responded with error")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
type rcloneCancelTransferResJSON struct {
Finished bool `json:"finished"`
Success bool `json:"success"`
ID int64 `json:"id"`
Error string `json:"error"`
Group string `json:"group"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
Duration float64 `json:"duration"`
// think we don't need this
// "output": {} // output of the job as would have been returned if called synchronously
}
var resData rcloneCancelTransferResJSON
if err = json.NewDecoder(res.Body).Decode(&resData); err != nil {
err = errors.Wrap(err, "rclone driver: error decoding response data")
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
if resData.Error != "" {
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_TRANSFER_CANCEL_FAILED,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, errors.New(resData.Error)
}
transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_CANCELLED
if err := driver.pDriver.model.saveTransfer(nil); err != nil {
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_INVALID,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, err
}
return &datatx.TxInfo{
Id: &datatx.TxId{OpaqueId: transferID},
Status: datatx.Status_STATUS_TRANSFER_CANCELLED,
Ctime: &typespb.Timestamp{Seconds: uint64(cTime)},
}, nil
}
// RetryTransfer retries the transfer with the specified transfer ID.
// Note that tokens must still be valid.
func (driver *rclone) RetryTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error) {
return driver.startJob(ctx, transferID, "", "", "", "", "", "")
}
// getTransfer returns the transfer with the specified transfer ID
func (m *transferModel) getTransfer(transferID string) (*transfer, error) {
transfer, ok := m.Transfers[transferID]
if !ok {
return nil, errors.New("rclone driver: invalid transfer ID")
}
return transfer, nil
}
func (driver *rclone) remotePathIsFolder(remote string, remotePath string, remoteToken string) (bool, error) {
type rcloneListReqJSON struct {
Fs string `json:"fs"`
Remote string `json:"remote"`
}
fs := fmt.Sprintf(":webdav,headers=\"x-access-token,%v\",url=\"%v\":", remoteToken, remote)
rcloneReq := &rcloneListReqJSON{
Fs: fs,
Remote: remotePath,
}
data, err := json.Marshal(rcloneReq)
if err != nil {
return false, errors.Wrap(err, "rclone: error marshalling rclone req data")
}
listMethod := "/operations/list"
u, err := url.Parse(driver.config.Endpoint)
if err != nil {
return false, errors.Wrap(err, "rclone driver: error parsing driver endpoint")
}
u.Path = path.Join(u.Path, listMethod)
requestURL := u.String()
req, err := http.NewRequest("POST", requestURL, bytes.NewReader(data))
if err != nil {
return false, errors.Wrap(err, "rclone driver: error framing post request")
}
req.Header.Set("Content-Type", "application/json")
req.SetBasicAuth(driver.config.AuthUser, driver.config.AuthPass)
res, err := driver.client.Do(req)
if err != nil {
return false, errors.Wrap(err, "rclone driver: error sending post request")
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
var errorResData rcloneHTTPErrorRes
if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil {
return false, errors.Wrap(err, "rclone driver: error decoding response data")
}
return false, errors.Wrap(errors.Errorf("status: %v, error: %v", errorResData.Status, errorResData.Error), "rclone driver: rclone request responded with error")
}
type item struct {
Path string `json:"Path"`
Name string `json:"Name"`
Size int64 `json:"Size"`
MimeType string `json:"MimeType"`
ModTime string `json:"ModTime"`
IsDir bool `json:"IsDir"`
}
type rcloneListResJSON struct {
List []*item `json:"list"`
}
var resData rcloneListResJSON
if err = json.NewDecoder(res.Body).Decode(&resData); err != nil {
return false, errors.Wrap(err, "rclone driver: error decoding response data")
}
// a file will return one single item, the file, with path being the remote path and IsDir will be false
if len(resData.List) == 1 && resData.List[0].Path == remotePath && !resData.List[0].IsDir {
return false, nil
}
// in all other cases the remote path is a directory
return true, nil
}
@@ -0,0 +1,36 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import (
"github.com/cs3org/reva/v2/pkg/datatx"
)
// NewFunc is the function that datatx implementations
// should register at init time.
type NewFunc func(map[string]interface{}) (datatx.Manager, error)
// NewFuncs is a map containing all the registered datatx backends.
var NewFuncs = map[string]NewFunc{}
// Register registers a new datatx backend new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
File diff suppressed because it is too large Load Diff
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosclient
import (
"context"
"io"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/storage/utils/acl"
)
// EOSClient is the interface which enables access to EOS instances through various interfaces.
type EOSClient interface {
AddACL(ctx context.Context, auth, rootAuth Authorization, path string, position uint, a *acl.Entry) error
RemoveACL(ctx context.Context, auth, rootAuth Authorization, path string, a *acl.Entry) error
UpdateACL(ctx context.Context, auth, rootAuth Authorization, path string, position uint, a *acl.Entry) error
GetACL(ctx context.Context, auth Authorization, path, aclType, target string) (*acl.Entry, error)
ListACLs(ctx context.Context, auth Authorization, path string) ([]*acl.Entry, error)
GetFileInfoByInode(ctx context.Context, auth Authorization, inode uint64) (*FileInfo, error)
GetFileInfoByFXID(ctx context.Context, auth Authorization, fxid string) (*FileInfo, error)
GetFileInfoByPath(ctx context.Context, auth Authorization, path string) (*FileInfo, error)
SetAttr(ctx context.Context, auth Authorization, attr *Attribute, errorIfExists, recursive bool, path string) error
UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path string) error
GetAttr(ctx context.Context, auth Authorization, key, path string) (*Attribute, error)
GetQuota(ctx context.Context, username string, rootAuth Authorization, path string) (*QuotaInfo, error)
SetQuota(ctx context.Context, rooAuth Authorization, info *SetQuotaInfo) error
Touch(ctx context.Context, auth Authorization, path string) error
Chown(ctx context.Context, auth, chownauth Authorization, path string) error
Chmod(ctx context.Context, auth Authorization, mode, path string) error
CreateDir(ctx context.Context, auth Authorization, path string) error
Remove(ctx context.Context, auth Authorization, path string, noRecycle bool) error
Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error
List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error)
Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error)
Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser) error
WriteFile(ctx context.Context, auth Authorization, path, source string) error
ListDeletedEntries(ctx context.Context, auth Authorization) ([]*DeletedEntry, error)
RestoreDeletedEntry(ctx context.Context, auth Authorization, key string) error
PurgeDeletedEntries(ctx context.Context, auth Authorization) error
ListVersions(ctx context.Context, auth Authorization, p string) ([]*FileInfo, error)
RollbackToVersion(ctx context.Context, auth Authorization, path, version string) error
ReadVersion(ctx context.Context, auth Authorization, p, version string) (io.ReadCloser, error)
GenerateToken(ctx context.Context, auth Authorization, path string, a *acl.Entry) (string, error)
}
// AttrType is the type of extended attribute,
// either system (sys) or user (user).
type AttrType uint32
// Attribute represents an EOS extended attribute.
type Attribute struct {
Type AttrType
Key, Val string
}
// FileInfo represents the metadata information returned by querying the EOS namespace.
type FileInfo struct {
IsDir bool
MTimeNanos uint32
Inode uint64 `json:"inode"`
FID uint64 `json:"fid"`
UID uint64 `json:"uid"`
GID uint64 `json:"gid"`
TreeSize uint64 `json:"tree_size"`
MTimeSec uint64 `json:"mtime_sec"`
Size uint64 `json:"size"`
TreeCount uint64 `json:"tree_count"`
File string `json:"eos_file"`
ETag string `json:"etag"`
Instance string `json:"instance"`
XS *Checksum `json:"xs"`
SysACL *acl.ACLs `json:"sys_acl"`
Attrs map[string]string `json:"attrs"`
}
// DeletedEntry represents an entry from the trashbin.
type DeletedEntry struct {
RestorePath string
RestoreKey string
Size uint64
DeletionMTime uint64
IsDir bool
}
// Checksum represents a cheksum entry for a file returned by EOS.
type Checksum struct {
XSSum string
XSType string
}
// QuotaInfo reports the available bytes and inodes for a particular user.
// eos reports all quota values are unsigned long, see https://github.com/cern-eos/eos/blob/93515df8c0d5a858982853d960bec98f983c1285/mgm/Quota.hh#L135
type QuotaInfo struct {
AvailableBytes, UsedBytes uint64
AvailableInodes, UsedInodes uint64
}
// SetQuotaInfo encapsulates the information needed to
// create a quota space in EOS for a user
type SetQuotaInfo struct {
Username string
UID string
GID string
QuotaNode string
MaxBytes uint64
MaxFiles uint64
}
// Constants for ACL position
const (
EndPosition uint = 0
StartPosition uint = 1
)
// Role holds the attributes required to authenticate to EOS via role-based access.
type Role struct {
UID, GID string
}
// Authorization specifies the mechanisms through which EOS can be accessed.
// One of the data members must be set.
type Authorization struct {
Role Role
Token string
}
// AttrAlreadyExistsError is the error raised when setting
// an already existing attr on a resource
const AttrAlreadyExistsError = errtypes.BadRequest("attr already exists")
// AttrNotExistsError is the error raised when removing
// an attribute that does not exist
const AttrNotExistsError = errtypes.BadRequest("attr not exists")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,604 @@
// @project The CERN Tape Archive (CTA)
// @brief CTA-EOS gRPC API for CASTOR-EOS migration
// @copyright Copyright 2019 CERN
// @license This program is free software: you can redistribute it and/or
// modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the GNU General Public License for more
// details.
//
// You should have received a copy of the GNU General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
// NOTE: Compile for Go with:
// protoc ./eos_grpc.proto --go_out=plugins=grpc:.
syntax = "proto3";
package eos.rpc;
option java_multiple_files = true;
option java_package = "io.grpc.eos.rpc";
option java_outer_classname = "EosProto";
option objc_class_prefix = "EOS";
option go_package = "github.com/cern-eos/grpc-proto/protobuf;eos_grpc";
service Eos {
// Replies to a ping
rpc Ping(PingRequest) returns (PingReply) {}
// ---------------------------------------------------------------------
// NAMESPACE
// ---------------------------------------------------------------------
// Replies to MD requests with a stream
rpc MD(MDRequest) returns (stream MDResponse) {}
// Replies to Find requests with a stream
rpc Find(FindRequest) returns (stream MDResponse) {}
// Replies to a NsStat operation
rpc NsStat(NsStatRequest) returns (NsStatResponse) {}
// Replies to an insert
rpc ContainerInsert(ContainerInsertRequest) returns (InsertReply) {}
rpc FileInsert(FileInsertRequest) returns (InsertReply) {}
// Replies to a NsRequest operation
rpc Exec(NSRequest) returns (NSResponse) {}
// ---------------------------------------------------------------------
// OPENSTACK
// ---------------------------------------------------------------------
// Manila Driver
rpc ManilaServerRequest(ManilaRequest) returns (ManilaResponse) {}
}
message PingRequest {
string authkey = 1;
bytes message = 2;
}
message PingReply { bytes message = 1; }
// ---------------------------------------------------------------------
// NAMESPACE
// ---------------------------------------------------------------------
message ContainerInsertRequest {
repeated ContainerMdProto container = 1;
string authkey = 2;
bool inherit_md = 3;
}
message FileInsertRequest {
repeated FileMdProto files = 1;
string authkey = 2;
}
message InsertReply {
repeated string message = 1;
repeated uint32 retc = 2;
}
message Time {
uint64 sec = 1;
uint64 n_sec = 2;
}
message Checksum {
bytes value = 1;
string type = 2;
}
message FileMdProto {
uint64 id = 1;
uint64 cont_id = 2;
uint64 uid = 3;
uint64 gid = 4;
uint64 size = 5;
uint32 layout_id = 6;
uint32 flags = 7;
bytes name = 8;
bytes link_name = 9;
Time ctime = 10; // change time
Time mtime = 11; // modification time
Checksum checksum = 12;
repeated uint32 locations = 13;
repeated uint32 unlink_locations = 14;
map<string, bytes> xattrs = 15;
bytes path = 16;
string etag = 17;
uint64 inode = 18;
}
message ContainerMdProto {
uint64 id = 1;
uint64 parent_id = 2;
uint64 uid = 3;
uint64 gid = 4;
int64 tree_size = 6;
uint32 mode = 5;
uint32 flags = 7;
bytes name = 8;
Time ctime = 9; // change time
Time mtime = 10; // modification time
Time stime = 11; // sync time
map<string, bytes> xattrs = 12;
bytes path = 13;
string etag = 14;
uint64 inode = 15;
}
enum TYPE {
FILE = 0;
CONTAINER = 1;
LISTING = 2;
STAT = 3;
}
enum QUOTATYPE {
USER = 0;
GROUP = 2;
PROJECT = 3;
}
enum QUOTAOP {
GET = 0;
SET = 1;
RM = 2;
RMNODE = 3;
}
enum QUOTAENTRY {
NONE = 0;
VOLUME = 1;
INODE = 2;
}
message QuotaProto {
bytes path = 1; // quota node path
string name = 2; // associated name for the given type
QUOTATYPE type = 3; // user,group,project or all quota
uint64 usedbytes = 4; // bytes used physical
uint64 usedlogicalbytes = 5; // bytes used logical
uint64 usedfiles = 6; // number of files used
uint64 maxbytes = 7; // maximum number of bytes (volume quota)
uint64 maxlogicalbytes =
8; // maximum number of logical bytes (logical volume quota)
uint64 maxfiles = 9; // maximum number of files (inode quota)
float percentageusedbytes =
10; // percentage of volume quota used from 0 to 100
float percentageusedfiles = 11; // percentag of inode quota used from 0 to 100
string statusbytes = 12; // status string for volume quota ok,warning,exceeded
string statusfiles = 13; // status string for inode quota ok,warning,exceeded
}
message RoleId {
uint64 uid = 1;
uint64 gid = 2;
string username = 3;
string groupname = 4;
}
message MDId {
bytes path = 1;
fixed64 id = 2;
fixed64 ino = 3;
TYPE type = 4;
}
message Limit {
bool zero = 1;
uint64 min = 2;
uint64 max = 3;
}
message MDSelection {
bool select = 1;
Limit ctime = 2;
Limit mtime = 3;
Limit stime = 4;
Limit size = 5;
Limit treesize = 6;
Limit children = 7;
Limit locations = 8;
Limit unlinked_locations = 9;
uint64 layoutid = 10;
uint64 flags = 11;
bool symlink = 12;
Checksum checksum = 13;
uint32 owner = 14;
uint32 group = 15;
bool owner_root = 16;
bool group_root = 17;
bytes regexp_filename = 18;
bytes regexp_dirname = 19;
map<string, bytes> xattr = 20;
}
message MDRequest {
TYPE type = 1;
MDId id = 2;
string authkey = 3;
RoleId role = 4;
MDSelection selection = 5;
}
message MDResponse {
TYPE type = 1;
FileMdProto fmd = 2;
ContainerMdProto cmd = 3;
}
message FindRequest {
TYPE type = 1;
MDId id = 2;
RoleId role = 3;
string authkey = 4;
uint64 maxdepth = 5;
MDSelection selection = 6;
}
message ShareAuth {
string prot = 1;
string name = 2;
string host = 3;
}
message ShareProto {
string permission = 1;
uint64 expires = 2;
string owner = 3;
string group = 4;
uint64 generation = 5;
string path = 6;
bool allowtree = 7;
string vtoken = 8;
repeated ShareAuth origins = 9;
}
message ShareToken {
ShareProto token = 1;
bytes signature = 2;
bytes serialized = 3;
int32 seed = 4;
}
message NSRequest {
message MkdirRequest {
MDId id = 1;
bool recursive = 2;
int64 mode = 3;
}
message RmdirRequest { MDId id = 1; }
message TouchRequest { MDId id = 1; }
message UnlinkRequest {
MDId id = 1;
bool norecycle = 3;
}
message RmRequest {
MDId id = 1;
bool recursive = 2;
bool norecycle = 3;
}
message RenameRequest {
MDId id = 1;
bytes target = 2;
}
message SymlinkRequest {
MDId id = 1;
bytes target = 2;
}
message VersionRequest {
enum VERSION_CMD {
CREATE = 0;
PURGE = 1;
LIST = 2;
GRAB = 3;
}
MDId id = 1;
VERSION_CMD cmd = 2;
int32 maxversion = 3;
string grabversion = 4;
}
message RecycleRequest {
string key = 1;
enum RECYCLE_CMD {
RESTORE = 0;
PURGE = 1;
LIST = 2;
}
RECYCLE_CMD cmd = 2;
message RestoreFlags {
bool force = 1;
bool mkpath = 2;
bool versions = 3;
}
message PurgeDate {
int32 year = 1;
int32 month = 2;
int32 day = 3;
}
RestoreFlags restoreflag = 3;
PurgeDate purgedate = 4;
}
message SetXAttrRequest {
MDId id = 1;
map<string, bytes> xattrs = 2;
bool recursive = 3;
repeated string keystodelete = 4;
bool create = 5;
}
message ChownRequest {
MDId id = 1;
RoleId owner = 2;
}
message ChmodRequest {
MDId id = 1;
int64 mode = 2;
}
message AclRequest {
enum ACL_COMMAND {
NONE = 0;
MODIFY = 1;
LIST = 2;
}
enum ACL_TYPE {
USER_ACL = 0;
SYS_ACL = 1;
}
MDId id = 1;
ACL_COMMAND cmd = 2;
bool recursive = 3;
ACL_TYPE type = 4;
string rule = 5;
uint32 position = 6;
}
message TokenRequest { ShareToken token = 1; }
message QuotaRequest {
bytes path = 1;
RoleId id = 2;
QUOTAOP op = 3; // get or set, rm or rmnode
uint64 maxfiles = 4; // maximum number of bytes (volume quota) for setting
uint64 maxbytes = 5; // maximum number of bytes (volume quota) for setting
QUOTAENTRY entry = 6; // select volume or inode entry for deletion
}
message ShareRequest {
message LsShare {
enum OutFormat {
NONE = 0; //
MONITORING = 1; // [-m]
LISTING = 2; // [-l]
JSON = 3; // [grpc]
}
OutFormat outformat = 1; //
string selection = 2; //
}
message OperateShare {
enum Op {
CREATE = 0;
REMOVE = 1;
SHARE = 2;
UNSHARE = 3;
ACCESS = 4;
MODIFY = 5;
}
Op op = 1;
string share = 2;
string acl = 3;
string path = 4;
string user = 5;
string group = 6;
}
oneof subcmd {
LsShare ls = 1;
OperateShare op = 2;
}
}
string authkey = 1;
RoleId role = 2;
// Actual request data object
oneof command {
MkdirRequest mkdir = 21;
RmdirRequest rmdir = 22;
TouchRequest touch = 23;
UnlinkRequest unlink = 24;
RmRequest rm = 25;
RenameRequest rename = 26;
SymlinkRequest symlink = 27;
VersionRequest version = 28;
RecycleRequest recycle = 29;
SetXAttrRequest xattr = 30;
ChownRequest chown = 31;
ChmodRequest chmod = 32;
AclRequest acl = 33;
TokenRequest token = 34;
QuotaRequest quota = 35;
ShareRequest share = 36;
}
}
message NSResponse {
message ErrorResponse {
int64 code = 1;
string msg = 2;
}
message VersionResponse {
message VersionInfo {
MDId id = 1;
Time mtime = 2;
}
int64 code = 1;
string msg = 2;
repeated VersionInfo versions = 3;
}
message RecycleResponse {
int64 code = 1;
string msg = 2;
message RecycleInfo {
enum DELETIONTYPE {
FILE = 0;
TREE = 1;
}
MDId id = 1;
RoleId owner = 2;
Time dtime = 3;
uint64 size = 4;
DELETIONTYPE type = 5;
string key = 6;
}
repeated RecycleInfo recycles = 3;
}
message AclResponse {
int64 code = 1;
string msg = 2;
string rule = 3;
}
message QuotaResponse {
int64 code = 1;
string msg = 2;
repeated QuotaProto quotanode = 3;
}
message ShareInfo {
string name = 1;
string root = 2;
string rule = 3;
uint64 uid = 4;
uint64 nshared = 5;
}
message ShareAccess {
string name = 1;
bool granted = 2;
}
message ShareResponse {
int64 code = 1;
string msg = 2;
repeated ShareInfo shares = 3;
repeated ShareAccess access = 4;
}
ErrorResponse error = 1;
VersionResponse version = 2;
RecycleResponse recycle = 3;
AclResponse acl = 4;
QuotaResponse quota = 5;
ShareResponse share = 6;
}
message NsStatRequest { string authkey = 1; }
message NsStatResponse {
int64 code = 1;
string emsg = 2;
string state = 3;
uint64 nfiles = 4;
uint64 ncontainers = 5;
uint64 boot_time = 6;
uint64 current_fid = 7;
uint64 current_cid = 8;
uint64 mem_virtual = 9;
uint64 mem_resident = 10;
uint64 mem_share = 11;
uint64 mem_growth = 12;
uint64 threads = 13;
uint64 fds = 14;
uint64 uptime = 15;
}
// ---------------------------------------------------------------------
// OPENSTACK
// ---------------------------------------------------------------------
enum MANILA_REQUEST_TYPE {
CREATE_SHARE = 0;
DELETE_SHARE = 1;
EXTEND_SHARE = 2;
SHRINK_SHARE = 3;
MANAGE_EXISTING = 4;
UNMANAGE = 5;
GET_CAPACITIES = 6;
/* EXTRA FUNCTIONS NOT IMPLEMENTED */
/*
CREATE_SNAPSHOT = 7;
DELETE_SNAPSHOT = 8;
CREATE_SHARE_FROM_SNAPSHOT = 9;
ENSURE_SHARE = 10;
ALLOW_ACCESS = 11;
DENY_ACCESS = 12;
GET_SHARE_STATS = 13;
DO_SETUP = 14;
SETUP_SERVER = 15;
TEARDOWN_SERVER = 16;
GET_NETWORK_ALLOCATIONS_NUMBER = 17;
VERIFY_SHARE_SERVER_HANDLING = 18;
CREATE_SHARE_GROUP = 19;
DELETE_SHARE_GROUP = 20;
*/
}
message ManilaRequest {
MANILA_REQUEST_TYPE request_type = 1;
string auth_key = 2;
string protocol = 3;
string share_name = 4;
string description = 5;
string share_id = 6;
string share_group_id = 7;
int32 quota = 8;
string creator = 9;
string egroup = 10;
string admin_egroup = 11;
string share_host = 12;
string share_location = 13;
}
message ManilaResponse {
string msg = 1; // for generic messages
int32 code = 2; // < 1 is an error -- > 1 is OK
int64 total_used = 3;
int64 total_capacity = 4;
int64 new_share_quota = 5;
string new_share_path = 6;
}
@@ -0,0 +1,445 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.19.1
// source: Rpc.proto
package eos_grpc
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
// EosClient is the client API for Eos service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type EosClient interface {
// Replies to a ping
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingReply, error)
// Replies to MD requests with a stream
MD(ctx context.Context, in *MDRequest, opts ...grpc.CallOption) (Eos_MDClient, error)
// Replies to Find requests with a stream
Find(ctx context.Context, in *FindRequest, opts ...grpc.CallOption) (Eos_FindClient, error)
// Replies to a NsStat operation
NsStat(ctx context.Context, in *NsStatRequest, opts ...grpc.CallOption) (*NsStatResponse, error)
// Replies to an insert
ContainerInsert(ctx context.Context, in *ContainerInsertRequest, opts ...grpc.CallOption) (*InsertReply, error)
FileInsert(ctx context.Context, in *FileInsertRequest, opts ...grpc.CallOption) (*InsertReply, error)
// Replies to a NsRequest operation
Exec(ctx context.Context, in *NSRequest, opts ...grpc.CallOption) (*NSResponse, error)
// Manila Driver
ManilaServerRequest(ctx context.Context, in *ManilaRequest, opts ...grpc.CallOption) (*ManilaResponse, error)
}
type eosClient struct {
cc grpc.ClientConnInterface
}
func NewEosClient(cc grpc.ClientConnInterface) EosClient {
return &eosClient{cc}
}
func (c *eosClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingReply, error) {
out := new(PingReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/Ping", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) MD(ctx context.Context, in *MDRequest, opts ...grpc.CallOption) (Eos_MDClient, error) {
stream, err := c.cc.NewStream(ctx, &Eos_ServiceDesc.Streams[0], "/eos.rpc.Eos/MD", opts...)
if err != nil {
return nil, err
}
x := &eosMDClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Eos_MDClient interface {
Recv() (*MDResponse, error)
grpc.ClientStream
}
type eosMDClient struct {
grpc.ClientStream
}
func (x *eosMDClient) Recv() (*MDResponse, error) {
m := new(MDResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *eosClient) Find(ctx context.Context, in *FindRequest, opts ...grpc.CallOption) (Eos_FindClient, error) {
stream, err := c.cc.NewStream(ctx, &Eos_ServiceDesc.Streams[1], "/eos.rpc.Eos/Find", opts...)
if err != nil {
return nil, err
}
x := &eosFindClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Eos_FindClient interface {
Recv() (*MDResponse, error)
grpc.ClientStream
}
type eosFindClient struct {
grpc.ClientStream
}
func (x *eosFindClient) Recv() (*MDResponse, error) {
m := new(MDResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *eosClient) NsStat(ctx context.Context, in *NsStatRequest, opts ...grpc.CallOption) (*NsStatResponse, error) {
out := new(NsStatResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/NsStat", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) ContainerInsert(ctx context.Context, in *ContainerInsertRequest, opts ...grpc.CallOption) (*InsertReply, error) {
out := new(InsertReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/ContainerInsert", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) FileInsert(ctx context.Context, in *FileInsertRequest, opts ...grpc.CallOption) (*InsertReply, error) {
out := new(InsertReply)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/FileInsert", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) Exec(ctx context.Context, in *NSRequest, opts ...grpc.CallOption) (*NSResponse, error) {
out := new(NSResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/Exec", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *eosClient) ManilaServerRequest(ctx context.Context, in *ManilaRequest, opts ...grpc.CallOption) (*ManilaResponse, error) {
out := new(ManilaResponse)
err := c.cc.Invoke(ctx, "/eos.rpc.Eos/ManilaServerRequest", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// EosServer is the server API for Eos service.
// All implementations must embed UnimplementedEosServer
// for forward compatibility
type EosServer interface {
// Replies to a ping
Ping(context.Context, *PingRequest) (*PingReply, error)
// Replies to MD requests with a stream
MD(*MDRequest, Eos_MDServer) error
// Replies to Find requests with a stream
Find(*FindRequest, Eos_FindServer) error
// Replies to a NsStat operation
NsStat(context.Context, *NsStatRequest) (*NsStatResponse, error)
// Replies to an insert
ContainerInsert(context.Context, *ContainerInsertRequest) (*InsertReply, error)
FileInsert(context.Context, *FileInsertRequest) (*InsertReply, error)
// Replies to a NsRequest operation
Exec(context.Context, *NSRequest) (*NSResponse, error)
// Manila Driver
ManilaServerRequest(context.Context, *ManilaRequest) (*ManilaResponse, error)
mustEmbedUnimplementedEosServer()
}
// UnimplementedEosServer must be embedded to have forward compatible implementations.
type UnimplementedEosServer struct {
}
func (UnimplementedEosServer) Ping(context.Context, *PingRequest) (*PingReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
}
func (UnimplementedEosServer) MD(*MDRequest, Eos_MDServer) error {
return status.Errorf(codes.Unimplemented, "method MD not implemented")
}
func (UnimplementedEosServer) Find(*FindRequest, Eos_FindServer) error {
return status.Errorf(codes.Unimplemented, "method Find not implemented")
}
func (UnimplementedEosServer) NsStat(context.Context, *NsStatRequest) (*NsStatResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method NsStat not implemented")
}
func (UnimplementedEosServer) ContainerInsert(context.Context, *ContainerInsertRequest) (*InsertReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method ContainerInsert not implemented")
}
func (UnimplementedEosServer) FileInsert(context.Context, *FileInsertRequest) (*InsertReply, error) {
return nil, status.Errorf(codes.Unimplemented, "method FileInsert not implemented")
}
func (UnimplementedEosServer) Exec(context.Context, *NSRequest) (*NSResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Exec not implemented")
}
func (UnimplementedEosServer) ManilaServerRequest(context.Context, *ManilaRequest) (*ManilaResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ManilaServerRequest not implemented")
}
func (UnimplementedEosServer) mustEmbedUnimplementedEosServer() {}
// UnsafeEosServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to EosServer will
// result in compilation errors.
type UnsafeEosServer interface {
mustEmbedUnimplementedEosServer()
}
func RegisterEosServer(s grpc.ServiceRegistrar, srv EosServer) {
s.RegisterService(&Eos_ServiceDesc, srv)
}
func _Eos_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PingRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).Ping(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/Ping",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).Ping(ctx, req.(*PingRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_MD_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(MDRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(EosServer).MD(m, &eosMDServer{stream})
}
type Eos_MDServer interface {
Send(*MDResponse) error
grpc.ServerStream
}
type eosMDServer struct {
grpc.ServerStream
}
func (x *eosMDServer) Send(m *MDResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Eos_Find_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(FindRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(EosServer).Find(m, &eosFindServer{stream})
}
type Eos_FindServer interface {
Send(*MDResponse) error
grpc.ServerStream
}
type eosFindServer struct {
grpc.ServerStream
}
func (x *eosFindServer) Send(m *MDResponse) error {
return x.ServerStream.SendMsg(m)
}
func _Eos_NsStat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(NsStatRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).NsStat(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/NsStat",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).NsStat(ctx, req.(*NsStatRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_ContainerInsert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ContainerInsertRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).ContainerInsert(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/ContainerInsert",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).ContainerInsert(ctx, req.(*ContainerInsertRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_FileInsert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(FileInsertRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).FileInsert(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/FileInsert",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).FileInsert(ctx, req.(*FileInsertRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_Exec_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(NSRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).Exec(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/Exec",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).Exec(ctx, req.(*NSRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Eos_ManilaServerRequest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ManilaRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EosServer).ManilaServerRequest(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/eos.rpc.Eos/ManilaServerRequest",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EosServer).ManilaServerRequest(ctx, req.(*ManilaRequest))
}
return interceptor(ctx, in, info, handler)
}
// Eos_ServiceDesc is the grpc.ServiceDesc for Eos service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var Eos_ServiceDesc = grpc.ServiceDesc{
ServiceName: "eos.rpc.Eos",
HandlerType: (*EosServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Ping",
Handler: _Eos_Ping_Handler,
},
{
MethodName: "NsStat",
Handler: _Eos_NsStat_Handler,
},
{
MethodName: "ContainerInsert",
Handler: _Eos_ContainerInsert_Handler,
},
{
MethodName: "FileInsert",
Handler: _Eos_FileInsert_Handler,
},
{
MethodName: "Exec",
Handler: _Eos_Exec_Handler,
},
{
MethodName: "ManilaServerRequest",
Handler: _Eos_ManilaServerRequest_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "MD",
Handler: _Eos_MD_Handler,
ServerStreams: true,
},
{
StreamName: "Find",
Handler: _Eos_Find_Handler,
ServerStreams: true,
},
},
Metadata: "Rpc.proto",
}
File diff suppressed because it is too large Load Diff
+494
View File
@@ -0,0 +1,494 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosgrpc
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"time"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/eosclient"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/logger"
)
// HTTPOptions to configure the Client.
type HTTPOptions struct {
// HTTP URL of the EOS MGM.
// Default is https://eos-example.org
BaseURL string
// Timeout in seconds for connecting to the service
ConnectTimeout int
// Timeout in seconds for sending a request to the service and getting a response
// Does not include redirections
RWTimeout int
// Timeout in seconds for performing an operation. Includes every redirection, retry, etc
OpTimeout int
// Max idle conns per Transport
MaxIdleConns int
// Max conns per transport per destination host
MaxConnsPerHost int
// Max idle conns per transport per destination host
MaxIdleConnsPerHost int
// TTL for an idle conn per transport
IdleConnTimeout int
// If the URL is https, then we need to configure this client
// with the usual TLS stuff
// Defaults are /etc/grid-security/hostcert.pem and /etc/grid-security/hostkey.pem
ClientCertFile string
ClientKeyFile string
// These will override the defaults, which are common system paths hardcoded
// in the go x509 implementation (why did they do that?!?!?)
// of course /etc/grid-security/certificates is NOT in those defaults!
ClientCADirs string
ClientCAFiles string
}
// Init fills the basic fields
func (opt *HTTPOptions) init() {
if opt.BaseURL == "" {
opt.BaseURL = "https://eos-example.org"
}
if opt.ConnectTimeout == 0 {
opt.ConnectTimeout = 30
}
if opt.RWTimeout == 0 {
opt.RWTimeout = 180
}
if opt.OpTimeout == 0 {
opt.OpTimeout = 360
}
if opt.MaxIdleConns == 0 {
opt.MaxIdleConns = 100
}
if opt.MaxConnsPerHost == 0 {
opt.MaxConnsPerHost = 64
}
if opt.MaxIdleConnsPerHost == 0 {
opt.MaxIdleConnsPerHost = 8
}
if opt.IdleConnTimeout == 0 {
opt.IdleConnTimeout = 30
}
if opt.ClientCertFile == "" {
opt.ClientCertFile = "/etc/grid-security/hostcert.pem"
}
if opt.ClientKeyFile == "" {
opt.ClientKeyFile = "/etc/grid-security/hostkey.pem"
}
if opt.ClientCAFiles != "" {
os.Setenv("SSL_CERT_FILE", opt.ClientCAFiles)
}
if opt.ClientCADirs != "" {
os.Setenv("SSL_CERT_DIR", opt.ClientCADirs)
} else {
os.Setenv("SSL_CERT_DIR", "/etc/grid-security/certificates")
}
}
// EOSHTTPClient performs HTTP-based tasks (e.g. upload, download)
// against a EOS management node (MGM)
// using the EOS XrdHTTP interface.
// In this module we wrap eos-related behaviour, e.g. headers or r/w retries
type EOSHTTPClient struct {
opt *HTTPOptions
cl *http.Client
}
// NewEOSHTTPClient creates a new client with the given options.
func NewEOSHTTPClient(opt *HTTPOptions) (*EOSHTTPClient, error) {
log := logger.New().With().Int("pid", os.Getpid()).Logger()
log.Debug().Str("func", "New").Str("Creating new eoshttp client. opt: ", "'"+fmt.Sprintf("%#v", opt)+"' ").Msg("")
if opt == nil {
log.Debug().Str("opt is nil, error creating http client ", "").Msg("")
return nil, errtypes.InternalError("HTTPOptions is nil")
}
opt.init()
cert, err := tls.LoadX509KeyPair(opt.ClientCertFile, opt.ClientKeyFile)
if err != nil {
return nil, err
}
// TODO: the error reporting of http.transport is insufficient
// we may want to check manually at least the existence of the certfiles
// The point is that also the error reporting of the context that calls this function
// is weak
t := &http.Transport{
TLSClientConfig: &tls.Config{
Certificates: []tls.Certificate{cert},
},
MaxIdleConns: opt.MaxIdleConns,
MaxConnsPerHost: opt.MaxConnsPerHost,
MaxIdleConnsPerHost: opt.MaxIdleConnsPerHost,
IdleConnTimeout: time.Duration(opt.IdleConnTimeout) * time.Second,
DisableCompression: true,
}
cl := &http.Client{
Transport: t,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
return &EOSHTTPClient{
opt: opt,
cl: cl,
}, nil
}
// Format a human readable line that describes a response
func rspdesc(rsp *http.Response) string {
desc := "'" + fmt.Sprintf("%d", rsp.StatusCode) + "'" + ": '" + rsp.Status + "'"
buf := new(bytes.Buffer)
r := "<none>"
n, e := buf.ReadFrom(rsp.Body)
if e != nil {
r = "Error reading body: '" + e.Error() + "'"
} else if n > 0 {
r = buf.String()
}
desc += " - '" + r + "'"
return desc
}
// If the error is not nil, take that
// If there is an error coming from EOS, erturn a descriptive error
func (c *EOSHTTPClient) getRespError(rsp *http.Response, err error) error {
if err != nil {
return err
}
if rsp.StatusCode == 0 {
return nil
}
switch rsp.StatusCode {
case 0, 200, 201:
return nil
case 403:
return errtypes.PermissionDenied(rspdesc(rsp))
case 404:
return errtypes.NotFound(rspdesc(rsp))
}
err2 := errtypes.InternalError("Err from EOS: " + rspdesc(rsp))
return err2
}
// From the basepath and the file path... build an url
func (c *EOSHTTPClient) buildFullURL(urlpath string, auth eosclient.Authorization) (string, error) {
u, err := url.Parse(c.opt.BaseURL)
if err != nil {
return "", err
}
u, err = u.Parse(url.PathEscape(urlpath))
if err != nil {
return "", err
}
// Prohibit malicious users from injecting a false uid/gid into the url
v := u.Query()
if v.Get("eos.ruid") != "" || v.Get("eos.rgid") != "" {
return "", errtypes.PermissionDenied("Illegal malicious url " + urlpath)
}
if len(auth.Role.UID) > 0 {
v.Set("eos.ruid", auth.Role.UID)
}
if len(auth.Role.GID) > 0 {
v.Set("eos.rgid", auth.Role.GID)
}
u.RawQuery = v.Encode()
return u.String(), nil
}
// GETFile does an entire GET to download a full file. Returns a stream to read the content from
func (c *EOSHTTPClient) GETFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.WriteCloser) (io.ReadCloser, error) {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "GETFile").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "GET", finalurl, nil)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return nil, err
}
ntries := 0
nredirs := 0
timebegin := time.Now().Unix()
for {
// Check for a max count of redirections or retries
// Check for a global timeout in any case
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "GETFile").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return nil, errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
log.Debug().Str("func", "GETFile").Msg("sending req")
resp, err := c.cl.Do(req)
// Let's support redirections... and if we retry we have to retry at the same FST, avoid going back to the MGM
if resp != nil && (resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect) {
// io.Copy(io.Discard, resp.Body)
// resp.Body.Close()
loc, err := resp.Location()
if err != nil {
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't get a new location for a redirection")
return nil, err
}
req, err = http.NewRequestWithContext(ctx, "GET", loc.String(), nil)
if err != nil {
log.Error().Str("func", "GETFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return nil, err
}
req.Close = true
log.Debug().Str("func", "GETFile").Str("location", loc.String()).Msg("redirection")
nredirs++
resp = nil
err = nil
continue
}
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "GETFile").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "GETFile").Str("url", finalurl).Str("err", e.Error()).Msg("")
return nil, e
}
log.Debug().Str("func", "GETFile").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return nil, errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
if stream != nil {
// Streaming versus localfile. If we have bene given a dest stream then copy the body into it
_, err = io.Copy(stream, resp.Body)
return nil, err
}
// If we have not been given a stream to write into then return our stream to read from
return resp.Body, nil
}
}
// PUTFile does an entire PUT to upload a full file, taking the data from a stream
func (c *EOSHTTPClient) PUTFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.ReadCloser, length int64) error {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "PUTFile").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Int64("length", length).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req, err := http.NewRequestWithContext(ctx, "PUT", finalurl, nil)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req.Close = true
ntries := 0
nredirs := 0
timebegin := time.Now().Unix()
for {
// Check for a max count of redirections or retries
// Check for a global timeout in any case
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
log.Debug().Str("func", "PUTFile").Msg("sending req")
resp, err := c.cl.Do(req)
// Let's support redirections... and if we retry we retry at the same FST
if resp != nil && resp.StatusCode == 307 {
// io.Copy(io.Discard, resp.Body)
// resp.Body.Close()
loc, err := resp.Location()
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't get a new location for a redirection")
return err
}
req, err = http.NewRequestWithContext(ctx, "PUT", loc.String(), stream)
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return err
}
if length >= 0 {
log.Debug().Str("func", "PUTFile").Int64("Content-Length", length).Msg("setting header")
req.Header.Set("Content-Length", strconv.FormatInt(length, 10))
}
if err != nil {
log.Error().Str("func", "PUTFile").Str("url", loc.String()).Str("err", err.Error()).Msg("can't create redirected request")
return err
}
if length >= 0 {
log.Debug().Str("func", "PUTFile").Int64("Content-Length", length).Msg("setting header")
req.Header.Set("Content-Length", strconv.FormatInt(length, 10))
}
log.Debug().Str("func", "PUTFile").Str("location", loc.String()).Msg("redirection")
nredirs++
resp = nil
err = nil
continue
}
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "PUTFile").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", e.Error()).Msg("")
return e
}
log.Debug().Str("func", "PUTFile").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
return nil
}
}
// Head performs a HEAD req. Useful to check the server
func (c *EOSHTTPClient) Head(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string) error {
log := appctx.GetLogger(ctx)
log.Info().Str("func", "Head").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Msg("")
// Now send the req and see what happens
finalurl, err := c.buildFullURL(urlpath, auth)
if err != nil {
log.Error().Str("func", "Head").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
req, err := http.NewRequestWithContext(ctx, "HEAD", finalurl, nil)
if err != nil {
log.Error().Str("func", "Head").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
return err
}
ntries := 0
timebegin := time.Now().Unix()
for {
tdiff := time.Now().Unix() - timebegin
if tdiff > int64(c.opt.OpTimeout) {
log.Error().Str("func", "Head").Str("url", finalurl).Int64("timeout", tdiff).Int("ntries", ntries).Msg("")
return errtypes.InternalError("Timeout with url" + finalurl)
}
// Execute the request. I don't like that there is no explicit timeout or buffer control on the input stream
resp, err := c.cl.Do(req)
// And get an error code (if error) that is worth propagating
e := c.getRespError(resp, err)
if e != nil {
if os.IsTimeout(e) {
ntries++
log.Warn().Str("func", "Head").Str("url", finalurl).Str("err", e.Error()).Int("try", ntries).Msg("recoverable network timeout")
continue
}
log.Error().Str("func", "Head").Str("url", finalurl).Str("err", e.Error()).Msg("")
return e
}
log.Debug().Str("func", "Head").Str("url", finalurl).Str("resp:", fmt.Sprintf("%#v", resp)).Msg("")
if resp == nil {
return errtypes.NotFound(fmt.Sprintf("url: %s", finalurl))
}
}
// return nil
}
+61
View File
@@ -0,0 +1,61 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package eosclient
import (
"fmt"
"github.com/cs3org/reva/v2/pkg/errtypes"
)
const (
// SystemAttr is the system extended attribute.
SystemAttr AttrType = iota
// UserAttr is the user extended attribute.
UserAttr
)
// AttrStringToType converts a string to an AttrType
func AttrStringToType(t string) (AttrType, error) {
switch t {
case "sys":
return SystemAttr, nil
case "user":
return UserAttr, nil
default:
return 0, errtypes.InternalError("attr type not existing")
}
}
// AttrTypeToString converts a type to a string representation.
func AttrTypeToString(at AttrType) string {
switch at {
case SystemAttr:
return "sys"
case UserAttr:
return "user"
default:
return "invalid"
}
}
// GetKey returns the key considering the type of attribute.
func (a *Attribute) GetKey() string {
return fmt.Sprintf("%s.%s", AttrTypeToString(a.Type), a.Key)
}
+304
View File
@@ -0,0 +1,304 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package errtypes contains definitions for common errors.
// It would have nice to call this package errors, err or error
// but errors clashes with github.com/pkg/errors, err is used for any error variable
// and error is a reserved word :)
package errtypes
import (
"strings"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
)
// NotFound is the error to use when a something is not found.
type NotFound string
func (e NotFound) Error() string { return "error: not found: " + string(e) }
// IsNotFound implements the IsNotFound interface.
func (e NotFound) IsNotFound() {}
// InternalError is the error to use when we really don't know what happened. Use with care
type InternalError string
func (e InternalError) Error() string { return "internal error: " + string(e) }
// IsInternalError implements the IsInternalError interface.
func (e InternalError) IsInternalError() {}
// PermissionDenied is the error to use when a resource cannot be access because of missing permissions.
type PermissionDenied string
func (e PermissionDenied) Error() string { return "error: permission denied: " + string(e) }
// IsPermissionDenied implements the IsPermissionDenied interface.
func (e PermissionDenied) IsPermissionDenied() {}
// Locked is the error to use when a resource cannot be modified because of a lock.
type Locked string
func (e Locked) Error() string { return "error: locked by " + string(e) }
// LockID returns the lock ID that caused this error
func (e Locked) LockID() string {
return string(e)
}
// IsLocked implements the IsLocked interface.
func (e Locked) IsLocked() {}
// Aborted is the error to use when a client should retry at a higher level
// (e.g., when a client-specified test-and-set fails, indicating the
// client should restart a read-modify-write sequence) request fails
// because a requested etag or lock ID mismatches.
//
// HTTP Mapping: 412 Precondition Failed
type Aborted string
func (e Aborted) Error() string { return "error: aborted: " + string(e) }
// IsAborted implements the IsAborted interface.
func (e Aborted) IsAborted() {}
// PreconditionFailed is the error to use when a client should not retry until
// the system state has been explicitly fixed. E.g., if an "rmdir"
// fails because the directory is non-empty, PreconditionFailed
// should be returned since the client should not retry unless
// the files are deleted from the directory. PreconditionFailed should also be
// returned when an intermediate directory for an MKCOL or PUT is missing.
//
// # FIXME rename to FailedPrecondition to make it less confusable with the http status Precondition Failed
//
// HTTP Mapping: 400 Bad Request, 405 Method Not Allowed, 409 Conflict
type PreconditionFailed string
func (e PreconditionFailed) Error() string { return "error: precondition failed: " + string(e) }
// IsPreconditionFailed implements the IsPreconditionFailed interface.
func (e PreconditionFailed) IsPreconditionFailed() {}
// AlreadyExists is the error to use when a resource something is not found.
type AlreadyExists string
func (e AlreadyExists) Error() string { return "error: already exists: " + string(e) }
// IsAlreadyExists implements the IsAlreadyExists interface.
func (e AlreadyExists) IsAlreadyExists() {}
// UserRequired represents an error when a resource is not found.
type UserRequired string
func (e UserRequired) Error() string { return "error: user required: " + string(e) }
// IsUserRequired implements the IsUserRequired interface.
func (e UserRequired) IsUserRequired() {}
// InvalidCredentials is the error to use when receiving invalid credentials.
type InvalidCredentials string
func (e InvalidCredentials) Error() string { return "error: invalid credentials: " + string(e) }
// IsInvalidCredentials implements the IsInvalidCredentials interface.
func (e InvalidCredentials) IsInvalidCredentials() {}
// NotSupported is the error to use when an action is not supported.
type NotSupported string
func (e NotSupported) Error() string { return "error: not supported: " + string(e) }
// IsNotSupported implements the IsNotSupported interface.
func (e NotSupported) IsNotSupported() {}
// PartialContent is the error to use when the client request has partial data.
type PartialContent string
func (e PartialContent) Error() string { return "error: partial content: " + string(e) }
// IsPartialContent implements the IsPartialContent interface.
func (e PartialContent) IsPartialContent() {}
// BadRequest is the error to use when the server cannot or will not process the request (due to a client error). Reauthenticating won't help.
type BadRequest string
func (e BadRequest) Error() string { return "error: bad request: " + string(e) }
// IsBadRequest implements the IsBadRequest interface.
func (e BadRequest) IsBadRequest() {}
// ChecksumMismatch is the error to use when the sent hash does not match the calculated hash.
type ChecksumMismatch string
func (e ChecksumMismatch) Error() string { return "error: checksum mismatch: " + string(e) }
// IsChecksumMismatch implements the IsChecksumMismatch interface.
func (e ChecksumMismatch) IsChecksumMismatch() {}
// StatusChecksumMismatch 419 is an unofficial http status code in an unassigned range that is used for checksum mismatches
// Proposed by https://stackoverflow.com/a/35665694
// Official HTTP status code registry: https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
// Note: TUS uses unassigned 460 Checksum-Mismatch
// RFC proposal for checksum digest uses a `Want-Digest` header: https://tools.ietf.org/html/rfc3230
// oc clienst issue: https://github.com/owncloud/core/issues/22711
const StatusChecksumMismatch = 419
// InsufficientStorage is the error to use when there is insufficient storage.
type InsufficientStorage string
func (e InsufficientStorage) Error() string { return "error: insufficient storage: " + string(e) }
// IsInsufficientStorage implements the IsInsufficientStorage interface.
func (e InsufficientStorage) IsInsufficientStorage() {}
// StatusCode returns StatusInsufficientStorage, this implementation is needed to allow TUS to cast the correct http errors.
func (e InsufficientStorage) StatusCode() int {
return StatusInsufficientStorage
}
// Body returns the error body. This implementation is needed to allow TUS to cast the correct http errors
func (e InsufficientStorage) Body() []byte {
return []byte(e.Error())
}
// StatusInsufficientStorage 507 is an official HTTP status code to indicate that there is insufficient storage
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/507
const StatusInsufficientStorage = 507
// IsNotFound is the interface to implement
// to specify that an a resource is not found.
type IsNotFound interface {
IsNotFound()
}
// IsAlreadyExists is the interface to implement
// to specify that a resource already exists.
type IsAlreadyExists interface {
IsAlreadyExists()
}
// IsInternalError is the interface to implement
// to specify that there was some internal error
type IsInternalError interface {
IsInternalError()
}
// IsUserRequired is the interface to implement
// to specify that a user is required.
type IsUserRequired interface {
IsUserRequired()
}
// IsInvalidCredentials is the interface to implement
// to specify that credentials were wrong.
type IsInvalidCredentials interface {
IsInvalidCredentials()
}
// IsNotSupported is the interface to implement
// to specify that an action is not supported.
type IsNotSupported interface {
IsNotSupported()
}
// IsPermissionDenied is the interface to implement
// to specify that an action is denied.
type IsPermissionDenied interface {
IsPermissionDenied()
}
// IsLocked is the interface to implement
// to specify that an resource is locked.
type IsLocked interface {
IsLocked()
}
// IsAborted is the interface to implement
// to specify that a request was aborted.
type IsAborted interface {
IsAborted()
}
// IsPreconditionFailed is the interface to implement
// to specify that a precondition failed.
type IsPreconditionFailed interface {
IsPreconditionFailed()
}
// IsPartialContent is the interface to implement
// to specify that the client request has partial data.
type IsPartialContent interface {
IsPartialContent()
}
// IsBadRequest is the interface to implement
// to specify that the server cannot or will not process the request.
type IsBadRequest interface {
IsBadRequest()
}
// IsChecksumMismatch is the interface to implement
// to specify that a checksum does not match.
type IsChecksumMismatch interface {
IsChecksumMismatch()
}
// IsInsufficientStorage is the interface to implement
// to specify that there is insufficient storage.
type IsInsufficientStorage interface {
IsInsufficientStorage()
}
// NewErrtypeFromStatus maps an rpc status to an errtype
func NewErrtypeFromStatus(status *rpc.Status) error {
switch status.Code {
case rpc.Code_CODE_OK:
return nil
case rpc.Code_CODE_NOT_FOUND:
return NotFound(status.Message)
case rpc.Code_CODE_ALREADY_EXISTS:
return AlreadyExists(status.Message)
// case rpc.Code_CODE_FAILED_PRECONDITION: ?
// return UserRequired(status.Message)
// case rpc.Code_CODE_PERMISSION_DENIED: ?
// IsInvalidCredentials
case rpc.Code_CODE_UNIMPLEMENTED:
return NotSupported(status.Message)
case rpc.Code_CODE_PERMISSION_DENIED:
// FIXME add locked status!
if strings.HasPrefix(status.Message, "set lock: error: locked by ") {
return Locked(strings.TrimPrefix(status.Message, "set lock: error: locked by "))
}
return PermissionDenied(status.Message)
// case rpc.Code_CODE_LOCKED:
// return Locked(status.Message)
// case rpc.Code_CODE_DATA_LOSS: ?
// IsPartialContent
case rpc.Code_CODE_ABORTED:
return Aborted(status.Message)
case rpc.Code_CODE_FAILED_PRECONDITION:
return PreconditionFailed(status.Message)
case rpc.Code_CODE_INSUFFICIENT_STORAGE:
return InsufficientStorage(status.Message)
case rpc.Code_CODE_INVALID_ARGUMENT, rpc.Code_CODE_OUT_OF_RANGE:
return BadRequest(status.Message)
default:
return InternalError(status.Message)
}
}
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"log"
"reflect"
"github.com/google/uuid"
"go-micro.dev/v4/events"
)
var (
// MainQueueName is the name of the main queue
// All events will go through here as they are forwarded to the consumer via the
// group name
// TODO: "fan-out" so not all events go through the same queue? requires investigation
MainQueueName = "main-queue"
// MetadatakeyEventType is the key used for the eventtype in the metadata map of the event
MetadatakeyEventType = "eventtype"
// MetadatakeyEventID is the key used for the eventID in the metadata map of the event
MetadatakeyEventID = "eventid"
)
type (
// Unmarshaller is the interface events need to fulfill
Unmarshaller interface {
Unmarshal([]byte) (interface{}, error)
}
// Publisher is the interface publishers need to fulfill
Publisher interface {
Publish(string, interface{}, ...events.PublishOption) error
}
// Consumer is the interface consumer need to fulfill
Consumer interface {
Consume(string, ...events.ConsumeOption) (<-chan events.Event, error)
}
// Stream is the interface common to Publisher and Consumer
Stream interface {
Publish(string, interface{}, ...events.PublishOption) error
Consume(string, ...events.ConsumeOption) (<-chan events.Event, error)
}
// Event is the envelope for events
Event struct {
Type string
ID string
Event interface{}
}
)
// Consume returns a channel that will get all events that match the given evs
// group defines the service type: One group will get exactly one copy of a event that is emitted
// NOTE: uses reflect on initialization
func Consume(s Consumer, group string, evs ...Unmarshaller) (<-chan Event, error) {
c, err := s.Consume(MainQueueName, events.WithGroup(group))
if err != nil {
return nil, err
}
registeredEvents := map[string]Unmarshaller{}
for _, e := range evs {
typ := reflect.TypeOf(e)
registeredEvents[typ.String()] = e
}
outchan := make(chan Event)
go func() {
for {
e := <-c
et := e.Metadata[MetadatakeyEventType]
ev, ok := registeredEvents[et]
if !ok {
continue
}
event, err := ev.Unmarshal(e.Payload)
if err != nil {
log.Printf("can't unmarshal event %v", err)
continue
}
outchan <- Event{
Type: et,
ID: e.Metadata[MetadatakeyEventID],
Event: event,
}
}
}()
return outchan, nil
}
// ConsumeAll allows consuming all events. Note that unmarshalling must be done manually in this case, therefore Event.Event will always be of type []byte
func ConsumeAll(s Consumer, group string) (<-chan Event, error) {
c, err := s.Consume(MainQueueName, events.WithGroup(group))
if err != nil {
return nil, err
}
outchan := make(chan Event)
go func() {
for {
e := <-c
outchan <- Event{
Type: e.Metadata[MetadatakeyEventType],
ID: e.Metadata[MetadatakeyEventID],
Event: e.Payload,
}
}
}()
return outchan, nil
}
// Publish publishes the ev to the MainQueue from where it is distributed to all subscribers
// NOTE: needs to use reflect on runtime
func Publish(s Publisher, ev interface{}) error {
evName := reflect.TypeOf(ev).String()
return s.Publish(MainQueueName, ev, events.WithMetadata(map[string]string{
MetadatakeyEventType: evName,
MetadatakeyEventID: uuid.New().String(),
}))
}
+165
View File
@@ -0,0 +1,165 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
// ContainerCreated is emitted when a directory has been created
type ContainerCreated struct {
SpaceOwner *user.UserId
Executant *user.UserId
Ref *provider.Reference
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (ContainerCreated) Unmarshal(v []byte) (interface{}, error) {
e := ContainerCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// FileUploaded is emitted when a file is uploaded
type FileUploaded struct {
SpaceOwner *user.UserId
Executant *user.UserId
Ref *provider.Reference
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (FileUploaded) Unmarshal(v []byte) (interface{}, error) {
e := FileUploaded{}
err := json.Unmarshal(v, &e)
return e, err
}
// FileTouched is emitted when a file is uploaded
type FileTouched struct {
SpaceOwner *user.UserId
Executant *user.UserId
Ref *provider.Reference
}
// Unmarshal to fulfill umarshaller interface
func (FileTouched) Unmarshal(v []byte) (interface{}, error) {
e := FileTouched{}
err := json.Unmarshal(v, &e)
return e, err
}
// FileDownloaded is emitted when a file is downloaded
type FileDownloaded struct {
Executant *user.UserId
Ref *provider.Reference
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (FileDownloaded) Unmarshal(v []byte) (interface{}, error) {
e := FileDownloaded{}
err := json.Unmarshal(v, &e)
return e, err
}
// ItemTrashed is emitted when a file or folder is trashed
type ItemTrashed struct {
SpaceOwner *user.UserId
Executant *user.UserId
ID *provider.ResourceId
Ref *provider.Reference
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (ItemTrashed) Unmarshal(v []byte) (interface{}, error) {
e := ItemTrashed{}
err := json.Unmarshal(v, &e)
return e, err
}
// ItemMoved is emitted when a file or folder is moved
type ItemMoved struct {
SpaceOwner *user.UserId
Executant *user.UserId
Ref *provider.Reference
Owner *user.UserId
OldReference *provider.Reference
}
// Unmarshal to fulfill umarshaller interface
func (ItemMoved) Unmarshal(v []byte) (interface{}, error) {
e := ItemMoved{}
err := json.Unmarshal(v, &e)
return e, err
}
// ItemPurged is emitted when a file or folder is removed from trashbin
type ItemPurged struct {
Executant *user.UserId
ID *provider.ResourceId
Ref *provider.Reference
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (ItemPurged) Unmarshal(v []byte) (interface{}, error) {
e := ItemPurged{}
err := json.Unmarshal(v, &e)
return e, err
}
// ItemRestored is emitted when a file or folder is restored from trashbin
type ItemRestored struct {
SpaceOwner *user.UserId
Executant *user.UserId
ID *provider.ResourceId
Ref *provider.Reference
Owner *user.UserId
OldReference *provider.Reference
Key string
}
// Unmarshal to fulfill umarshaller interface
func (ItemRestored) Unmarshal(v []byte) (interface{}, error) {
e := ItemRestored{}
err := json.Unmarshal(v, &e)
return e, err
}
// FileVersionRestored is emitted when a file version is restored
type FileVersionRestored struct {
SpaceOwner *user.UserId
Executant *user.UserId
Ref *provider.Reference
Owner *user.UserId
Key string
}
// Unmarshal to fulfill umarshaller interface
func (FileVersionRestored) Unmarshal(v []byte) (interface{}, error) {
e := FileVersionRestored{}
err := json.Unmarshal(v, &e)
return e, err
}
+99
View File
@@ -0,0 +1,99 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
)
// GroupCreated is emitted when a group was created
type GroupCreated struct {
Executant *user.UserId
GroupID string
}
// Unmarshal to fulfill umarshaller interface
func (GroupCreated) Unmarshal(v []byte) (interface{}, error) {
e := GroupCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// GroupDeleted is emitted when a group was deleted
type GroupDeleted struct {
Executant *user.UserId
GroupID string
}
// Unmarshal to fulfill umarshaller interface
func (GroupDeleted) Unmarshal(v []byte) (interface{}, error) {
e := GroupDeleted{}
err := json.Unmarshal(v, &e)
return e, err
}
// GroupMemberAdded is emitted when a user was added to a group
type GroupMemberAdded struct {
Executant *user.UserId
GroupID string
UserID string
}
// Unmarshal to fulfill umarshaller interface
func (GroupMemberAdded) Unmarshal(v []byte) (interface{}, error) {
e := GroupMemberAdded{}
err := json.Unmarshal(v, &e)
return e, err
}
// GroupMemberRemoved is emitted when a user was removed from a group
type GroupMemberRemoved struct {
Executant *user.UserId
GroupID string
UserID string
}
// Unmarshal to fulfill umarshaller interface
func (GroupMemberRemoved) Unmarshal(v []byte) (interface{}, error) {
e := GroupMemberRemoved{}
err := json.Unmarshal(v, &e)
return e, err
}
// GroupFeature represents a group feature
type GroupFeature struct {
Name string
Value string
}
// GroupFeatureChanged is emitted when a group feature was changed
type GroupFeatureChanged struct {
Executant *user.UserId
GroupID string
Features []GroupFeature
}
// Unmarshal to fulfill unmarshaller interface
func (GroupFeatureChanged) Unmarshal(v []byte) (interface{}, error) {
e := GroupFeatureChanged{}
err := json.Unmarshal(v, &e)
return e, err
}
+167
View File
@@ -0,0 +1,167 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
"time"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
type (
// Postprocessingstep are the available postprocessingsteps
Postprocessingstep string
// PostprocessingOutcome defines the result of the postprocessing
PostprocessingOutcome string
)
var (
// PPStepAntivirus is the step that scans for viruses
PPStepAntivirus Postprocessingstep = "virusscan"
// PPStepPolicies is the step the step that enforces policies
PPStepPolicies Postprocessingstep = "policies"
// PPStepDelay is the step that processing. Useful for testing or user annoyment
PPStepDelay Postprocessingstep = "delay"
// PPOutcomeDelete means that the file and the upload should be deleted
PPOutcomeDelete PostprocessingOutcome = "delete"
// PPOutcomeAbort means that the upload is cancelled but the bytes are being kept in the upload folder
PPOutcomeAbort PostprocessingOutcome = "abort"
// PPOutcomeContinue means that the upload is moved to its final destination (eventually being marked with pp results)
PPOutcomeContinue PostprocessingOutcome = "continue"
)
// BytesReceived is emitted by the server when it received all bytes of an upload
type BytesReceived struct {
UploadID string
SpaceOwner *user.UserId
ExecutingUser *user.User
ResourceID *provider.ResourceId
Filename string
Filesize uint64
URL string
}
// Unmarshal to fulfill umarshaller interface
func (BytesReceived) Unmarshal(v []byte) (interface{}, error) {
e := BytesReceived{}
err := json.Unmarshal(v, &e)
return e, err
}
// StartPostprocessingStep can be issued by the server to start a postprocessing step
type StartPostprocessingStep struct {
UploadID string
URL string
ExecutingUser *user.User
Filename string
Filesize uint64
Token string // for file retrieval in after upload case
ResourceID *provider.ResourceId // for file retrieval in after upload case
RevaToken string // for file retrieval in after upload case
StepToStart Postprocessingstep
}
// Unmarshal to fulfill umarshaller interface
func (StartPostprocessingStep) Unmarshal(v []byte) (interface{}, error) {
e := StartPostprocessingStep{}
err := json.Unmarshal(v, &e)
return e, err
}
// PostprocessingStepFinished can be issued by the server when a postprocessing step is finished
type PostprocessingStepFinished struct {
UploadID string
ExecutingUser *user.User
Filename string
FinishedStep Postprocessingstep // name of the step
Result interface{} // result information see VirusscanResult for example
Error error // possible error of the step
Outcome PostprocessingOutcome // some services may cause postprocessing to stop
}
// Unmarshal to fulfill umarshaller interface
func (PostprocessingStepFinished) Unmarshal(v []byte) (interface{}, error) {
e := PostprocessingStepFinished{}
err := json.Unmarshal(v, &e)
if err != nil {
return nil, err
}
switch e.FinishedStep {
case PPStepAntivirus:
var res VirusscanResult
b, _ := json.Marshal(e.Result)
err = json.Unmarshal(b, &res)
e.Result = res
case PPStepPolicies:
// nothing to do, but this makes the linter happy
}
return e, err
}
// VirusscanResult is the Result of a PostprocessingStepFinished event from the antivirus
type VirusscanResult struct {
Infected bool
Description string
Scandate time.Time
ResourceID *provider.ResourceId
ErrorMsg string // empty when no error
}
// PostprocessingFinished is emitted by *some* service which can decide that
type PostprocessingFinished struct {
UploadID string
Filename string
SpaceOwner *user.UserId
ExecutingUser *user.User
Result map[Postprocessingstep]interface{} // it is a map[step]Event
Outcome PostprocessingOutcome
}
// Unmarshal to fulfill umarshaller interface
func (PostprocessingFinished) Unmarshal(v []byte) (interface{}, error) {
e := PostprocessingFinished{}
err := json.Unmarshal(v, &e)
return e, err
}
// UploadReady is emitted by the storage provider when postprocessing is finished
type UploadReady struct {
UploadID string
Filename string
SpaceOwner *user.UserId
ExecutingUser *user.User
FileRef *provider.Reference
Failed bool
Timestamp time.Time
// add reference here? We could use it to inform client pp is finished
}
// Unmarshal to fulfill umarshaller interface
func (UploadReady) Unmarshal(v []byte) (interface{}, error) {
e := UploadReady{}
err := json.Unmarshal(v, &e)
return e, err
}
+231
View File
@@ -0,0 +1,231 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
"time"
group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
)
// ShareCreated is emitted when a share is created
type ShareCreated struct {
ShareID *collaboration.ShareId
Executant *user.UserId
Sharer *user.UserId
// split the protobuf Grantee oneof so we can use stdlib encoding/json
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
Sharee *provider.Grantee
ItemID *provider.ResourceId
Permissions *collaboration.SharePermissions
CTime *types.Timestamp
}
// Unmarshal to fulfill umarshaller interface
func (ShareCreated) Unmarshal(v []byte) (interface{}, error) {
e := ShareCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// ShareRemoved is emitted when a share is removed
type ShareRemoved struct {
Executant *user.UserId
// split protobuf Spec
ShareID *collaboration.ShareId
ShareKey *collaboration.ShareKey
// split the protobuf Grantee oneof so we can use stdlib encoding/json
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
ItemID *provider.ResourceId
Timestamp time.Time
}
// Unmarshal to fulfill umarshaller interface
func (ShareRemoved) Unmarshal(v []byte) (interface{}, error) {
e := ShareRemoved{}
err := json.Unmarshal(v, &e)
return e, err
}
// ShareUpdated is emitted when a share is updated
type ShareUpdated struct {
Executant *user.UserId
ShareID *collaboration.ShareId
ItemID *provider.ResourceId
Permissions *collaboration.SharePermissions
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
Sharer *user.UserId
MTime *types.Timestamp
// indicates what was updated - one of "displayname", "permissions"
Updated string
}
// Unmarshal to fulfill umarshaller interface
func (ShareUpdated) Unmarshal(v []byte) (interface{}, error) {
e := ShareUpdated{}
err := json.Unmarshal(v, &e)
return e, err
}
// ShareExpired is emitted when a share expires
type ShareExpired struct {
ShareID *collaboration.ShareId
ShareOwner *user.UserId
ItemID *provider.ResourceId
ExpiredAt time.Time
// split the protobuf Grantee oneof so we can use stdlib encoding/json
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
}
// Unmarshal to fulfill umarshaller interface
func (ShareExpired) Unmarshal(v []byte) (interface{}, error) {
e := ShareExpired{}
err := json.Unmarshal(v, &e)
return e, err
}
// ReceivedShareUpdated is emitted when a received share is accepted or declined
type ReceivedShareUpdated struct {
Executant *user.UserId
ShareID *collaboration.ShareId
ItemID *provider.ResourceId
Permissions *collaboration.SharePermissions
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
Sharer *user.UserId
MTime *types.Timestamp
State string
}
// Unmarshal to fulfill umarshaller interface
func (ReceivedShareUpdated) Unmarshal(v []byte) (interface{}, error) {
e := ReceivedShareUpdated{}
err := json.Unmarshal(v, &e)
return e, err
}
// LinkCreated is emitted when a public link is created
type LinkCreated struct {
Executant *user.UserId
ShareID *link.PublicShareId
Sharer *user.UserId
ItemID *provider.ResourceId
Permissions *link.PublicSharePermissions
DisplayName string
Expiration *types.Timestamp
PasswordProtected bool
CTime *types.Timestamp
Token string
}
// Unmarshal to fulfill umarshaller interface
func (LinkCreated) Unmarshal(v []byte) (interface{}, error) {
e := LinkCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// LinkUpdated is emitted when a public link is updated
type LinkUpdated struct {
Executant *user.UserId
ShareID *link.PublicShareId
Sharer *user.UserId
ItemID *provider.ResourceId
Permissions *link.PublicSharePermissions
DisplayName string
Expiration *types.Timestamp
PasswordProtected bool
CTime *types.Timestamp
Token string
FieldUpdated string
}
// Unmarshal to fulfill umarshaller interface
func (LinkUpdated) Unmarshal(v []byte) (interface{}, error) {
e := LinkUpdated{}
err := json.Unmarshal(v, &e)
return e, err
}
// LinkAccessed is emitted when a public link is accessed successfully (by token)
type LinkAccessed struct {
Executant *user.UserId
ShareID *link.PublicShareId
Sharer *user.UserId
ItemID *provider.ResourceId
Permissions *link.PublicSharePermissions
DisplayName string
Expiration *types.Timestamp
PasswordProtected bool
CTime *types.Timestamp
Token string
}
// Unmarshal to fulfill umarshaller interface
func (LinkAccessed) Unmarshal(v []byte) (interface{}, error) {
e := LinkAccessed{}
err := json.Unmarshal(v, &e)
return e, err
}
// LinkAccessFailed is emitted when an access to a public link has resulted in an error (by token)
type LinkAccessFailed struct {
Executant *user.UserId
ShareID *link.PublicShareId
Token string
Status rpc.Code
Message string
}
// Unmarshal to fulfill umarshaller interface
func (LinkAccessFailed) Unmarshal(v []byte) (interface{}, error) {
e := LinkAccessFailed{}
err := json.Unmarshal(v, &e)
return e, err
}
// LinkRemoved is emitted when a share is removed
type LinkRemoved struct {
Executant *user.UserId
// split protobuf Ref
ShareID *link.PublicShareId
ShareToken string
}
// Unmarshal to fulfill umarshaller interface
func (LinkRemoved) Unmarshal(v []byte) (interface{}, error) {
e := LinkRemoved{}
err := json.Unmarshal(v, &e)
return e, err
}
+172
View File
@@ -0,0 +1,172 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
"time"
group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
)
// SpaceCreated is emitted when a space is created
type SpaceCreated struct {
Executant *user.UserId
ID *provider.StorageSpaceId
Owner *user.UserId
Root *provider.ResourceId
Name string
Type string
Quota *provider.Quota
MTime *types.Timestamp
}
// Unmarshal to fulfill umarshaller interface
func (SpaceCreated) Unmarshal(v []byte) (interface{}, error) {
e := SpaceCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceRenamed is emitted when a space is renamed
type SpaceRenamed struct {
Executant *user.UserId
ID *provider.StorageSpaceId
Owner *user.UserId
Name string
}
// Unmarshal to fulfill umarshaller interface
func (SpaceRenamed) Unmarshal(v []byte) (interface{}, error) {
e := SpaceRenamed{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceDisabled is emitted when a space is disabled
type SpaceDisabled struct {
Executant *user.UserId
ID *provider.StorageSpaceId
Timestamp time.Time
}
// Unmarshal to fulfill umarshaller interface
func (SpaceDisabled) Unmarshal(v []byte) (interface{}, error) {
e := SpaceDisabled{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceEnabled is emitted when a space is (re-)enabled
type SpaceEnabled struct {
Executant *user.UserId
ID *provider.StorageSpaceId
Owner *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (SpaceEnabled) Unmarshal(v []byte) (interface{}, error) {
e := SpaceEnabled{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceDeleted is emitted when a space is deleted
type SpaceDeleted struct {
Executant *user.UserId
ID *provider.StorageSpaceId
SpaceName string
FinalMembers map[string]provider.ResourcePermissions
Timestamp time.Time
}
// Unmarshal to fulfill umarshaller interface
func (SpaceDeleted) Unmarshal(v []byte) (interface{}, error) {
e := SpaceDeleted{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceShared is emitted when a space is shared
type SpaceShared struct {
Executant *user.UserId
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
Creator *user.UserId
ID *provider.StorageSpaceId
Timestamp time.Time
}
// Unmarshal to fulfill umarshaller interface
func (SpaceShared) Unmarshal(v []byte) (interface{}, error) {
e := SpaceShared{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceUnshared is emitted when a space is unshared
type SpaceUnshared struct {
Executant *user.UserId
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
ID *provider.StorageSpaceId
Timestamp time.Time
}
// Unmarshal to fulfill umarshaller interface
func (SpaceUnshared) Unmarshal(v []byte) (interface{}, error) {
e := SpaceUnshared{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceUpdated is emitted when a space is updated
type SpaceUpdated struct {
Executant *user.UserId
ID *provider.StorageSpaceId
Space *provider.StorageSpace
}
// Unmarshal to fulfill umarshaller interface
func (SpaceUpdated) Unmarshal(v []byte) (interface{}, error) {
e := SpaceUpdated{}
err := json.Unmarshal(v, &e)
return e, err
}
// SpaceMembershipExpired is emitted when a space membership expires
type SpaceMembershipExpired struct {
SpaceOwner *user.UserId
SpaceID *provider.StorageSpaceId
SpaceName string
ExpiredAt time.Time
// split the protobuf Grantee oneof so we can use stdlib encoding/json
GranteeUserID *user.UserId
GranteeGroupID *group.GroupId
}
// Unmarshal to fulfill umarshaller interface
func (SpaceMembershipExpired) Unmarshal(v []byte) (interface{}, error) {
e := ShareExpired{}
err := json.Unmarshal(v, &e)
return e, err
}
+95
View File
@@ -0,0 +1,95 @@
package stream
import (
"bytes"
"crypto/tls"
"crypto/x509"
"errors"
"io"
"os"
"time"
"github.com/cenkalti/backoff"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/logger"
"github.com/go-micro/plugins/v4/events/natsjs"
)
// NatsConfig is the configuration needed for a NATS event stream
type NatsConfig struct {
Endpoint string // Endpoint of the nats server
Cluster string // CluserID of the nats cluster
TLSInsecure bool // Whether to verify TLS certificates
TLSRootCACertificate string // The root CA certificate used to validate the TLS certificate
EnableTLS bool // Enable TLS
}
// NatsFromConfig returns a nats stream from the given config
func NatsFromConfig(cfg NatsConfig) (events.Stream, error) {
var tlsConf *tls.Config
if cfg.EnableTLS {
var rootCAPool *x509.CertPool
if cfg.TLSRootCACertificate != "" {
rootCrtFile, err := os.Open(cfg.TLSRootCACertificate)
if err != nil {
return nil, err
}
rootCAPool, err = newCertPoolFromPEM(rootCrtFile)
if err != nil {
return nil, err
}
cfg.TLSInsecure = false
}
tlsConf = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: cfg.TLSInsecure, //nolint:gosec
RootCAs: rootCAPool,
}
}
return Nats(
natsjs.TLSConfig(tlsConf),
natsjs.Address(cfg.Endpoint),
natsjs.ClusterID(cfg.Cluster),
)
}
// Nats returns a nats streaming client
// retries exponentially to connect to a nats server
func Nats(opts ...natsjs.Option) (events.Stream, error) {
b := backoff.NewExponentialBackOff()
var stream events.Stream
o := func() error {
n := b.NextBackOff()
s, err := natsjs.NewStream(opts...)
if err != nil && n > time.Second {
logger.New().Error().Err(err).Msgf("can't connect to nats (jetstream) server, retrying in %s", n)
}
stream = s
return err
}
err := backoff.Retry(o, b)
return stream, err
}
// newCertPoolFromPEM reads certificates from io.Reader and returns a x509.CertPool
// containing those certificates.
func newCertPoolFromPEM(crts ...io.Reader) (*x509.CertPool, error) {
certPool := x509.NewCertPool()
var buf bytes.Buffer
for _, c := range crts {
if _, err := io.Copy(&buf, c); err != nil {
return nil, err
}
if !certPool.AppendCertsFromPEM(buf.Bytes()) {
return nil, errors.New("failed to append cert from PEM")
}
buf.Reset()
}
return certPool, nil
}
+42
View File
@@ -0,0 +1,42 @@
// Package stream provides streaming clients used by `Consume` and `Publish` methods
package stream
import (
"encoding/json"
"reflect"
"go-micro.dev/v4/events"
)
// Chan is a channel based streaming clients
// Useful for tests or in memory applications
type Chan [2]chan interface{}
// Publish implementation
func (ch Chan) Publish(_ string, msg interface{}, _ ...events.PublishOption) error {
go func() {
ch[0] <- msg
}()
return nil
}
// Consume implementation
func (ch Chan) Consume(_ string, _ ...events.ConsumeOption) (<-chan events.Event, error) {
evch := make(chan events.Event)
go func() {
for {
e := <-ch[1]
if e == nil {
// channel closed
return
}
b, _ := json.Marshal(e)
evname := reflect.TypeOf(e).String()
evch <- events.Event{
Payload: b,
Metadata: map[string]string{"eventtype": evname},
}
}
}()
return evch, nil
}
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
// TagsAdded is emitted when a Tag has been added
type TagsAdded struct {
SpaceOwner *user.UserId
Tags string
Ref *provider.Reference
Executant *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (TagsAdded) Unmarshal(v []byte) (interface{}, error) {
e := TagsAdded{}
err := json.Unmarshal(v, &e)
return e, err
}
// TagsRemoved is emitted when a Tag has been added
type TagsRemoved struct {
SpaceOwner *user.UserId
Tags string
Ref *provider.Reference
Executant *user.UserId
}
// Unmarshal to fulfill umarshaller interface
func (TagsRemoved) Unmarshal(v []byte) (interface{}, error) {
e := TagsRemoved{}
err := json.Unmarshal(v, &e)
return e, err
}
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package events
import (
"encoding/json"
"time"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
)
// UserCreated is emitted when a user was created
type UserCreated struct {
Executant *user.UserId
UserID string
}
// Unmarshal to fulfill umarshaller interface
func (UserCreated) Unmarshal(v []byte) (interface{}, error) {
e := UserCreated{}
err := json.Unmarshal(v, &e)
return e, err
}
// UserDeleted is emitted when a user was deleted
type UserDeleted struct {
Executant *user.UserId
UserID string
}
// Unmarshal to fulfill umarshaller interface
func (UserDeleted) Unmarshal(v []byte) (interface{}, error) {
e := UserDeleted{}
err := json.Unmarshal(v, &e)
return e, err
}
// UserFeature represents a user feature
type UserFeature struct {
Name string
Value string
}
// UserFeatureChanged is emitted when a user feature was changed
type UserFeatureChanged struct {
Executant *user.UserId
UserID string
Features []UserFeature
}
// Unmarshal to fulfill umarshaller interface
func (UserFeatureChanged) Unmarshal(v []byte) (interface{}, error) {
e := UserFeatureChanged{}
err := json.Unmarshal(v, &e)
return e, err
}
// PersonalDataExtracted is emitted when a user data extraction is finished
type PersonalDataExtracted struct {
Executant *user.UserId
Timestamp time.Time
ErrorMsg string
}
// Unmarshal to fulfill umarshaller interface
func (PersonalDataExtracted) Unmarshal(v []byte) (interface{}, error) {
e := PersonalDataExtracted{}
err := json.Unmarshal(v, &e)
return e, err
}
+35
View File
@@ -0,0 +1,35 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package group
import (
"context"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
)
// Manager is the interface to implement to manipulate groups.
type Manager interface {
GetGroup(ctx context.Context, gid *grouppb.GroupId, skipFetchingMembers bool) (*grouppb.Group, error)
GetGroupByClaim(ctx context.Context, claim, value string, skipFetchingMembers bool) (*grouppb.Group, error)
FindGroups(ctx context.Context, query string, skipFetchingMembers bool) ([]*grouppb.Group, error)
GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*userpb.UserId, error)
HasMember(ctx context.Context, gid *grouppb.GroupId, uid *userpb.UserId) (bool, error)
}
+171
View File
@@ -0,0 +1,171 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package json
import (
"context"
"encoding/json"
"os"
"strconv"
"strings"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/group"
"github.com/cs3org/reva/v2/pkg/group/manager/registry"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("json", New)
}
type manager struct {
groups []*grouppb.Group
}
type config struct {
// Groups holds a path to a file containing json conforming to the Groups struct
Groups string `mapstructure:"groups"`
}
func (c *config) init() {
if c.Groups == "" {
c.Groups = "/etc/revad/groups.json"
}
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := &config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
c.init()
return c, nil
}
// New returns a group manager implementation that reads a json file to provide group metadata.
func New(m map[string]interface{}) (group.Manager, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
f, err := os.ReadFile(c.Groups)
if err != nil {
return nil, err
}
groups := []*grouppb.Group{}
err = json.Unmarshal(f, &groups)
if err != nil {
return nil, err
}
return &manager{
groups: groups,
}, nil
}
func (m *manager) GetGroup(ctx context.Context, gid *grouppb.GroupId, skipFetchingMembers bool) (*grouppb.Group, error) {
for _, g := range m.groups {
if (g.Id.GetOpaqueId() == gid.OpaqueId || g.GroupName == gid.OpaqueId) && (gid.Idp == "" || gid.Idp == g.Id.GetIdp()) {
group := *g
if skipFetchingMembers {
group.Members = nil
}
return &group, nil
}
}
return nil, errtypes.NotFound(gid.OpaqueId)
}
func (m *manager) GetGroupByClaim(ctx context.Context, claim, value string, skipFetchingMembers bool) (*grouppb.Group, error) {
for _, g := range m.groups {
if groupClaim, err := extractClaim(g, claim); err == nil && value == groupClaim {
group := *g
if skipFetchingMembers {
group.Members = nil
}
return &group, nil
}
}
return nil, errtypes.NotFound(value)
}
func extractClaim(g *grouppb.Group, claim string) (string, error) {
switch claim {
case "group_name":
return g.GroupName, nil
case "gid_number":
return strconv.FormatInt(g.GidNumber, 10), nil
case "display_name":
return g.DisplayName, nil
case "mail":
return g.Mail, nil
}
return "", errors.New("json: invalid field")
}
func (m *manager) FindGroups(ctx context.Context, query string, skipFetchingMembers bool) ([]*grouppb.Group, error) {
groups := []*grouppb.Group{}
for _, g := range m.groups {
if groupContains(g, query) {
group := *g
if skipFetchingMembers {
group.Members = nil
}
groups = append(groups, &group)
}
}
return groups, nil
}
func groupContains(g *grouppb.Group, query string) bool {
query = strings.ToLower(query)
return strings.Contains(strings.ToLower(g.GroupName), query) || strings.Contains(strings.ToLower(g.DisplayName), query) ||
strings.Contains(strings.ToLower(g.Mail), query) || strings.Contains(strings.ToLower(g.Id.OpaqueId), query)
}
func (m *manager) GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*userpb.UserId, error) {
for _, g := range m.groups {
if g.Id.GetOpaqueId() == gid.OpaqueId || g.GroupName == gid.OpaqueId {
return g.Members, nil
}
}
return nil, errtypes.NotFound(gid.OpaqueId)
}
func (m *manager) HasMember(ctx context.Context, gid *grouppb.GroupId, uid *userpb.UserId) (bool, error) {
members, err := m.GetMembers(ctx, gid)
if err != nil {
return false, err
}
for _, u := range members {
if u.OpaqueId == uid.OpaqueId && u.Idp == uid.Idp {
return true, nil
}
}
return false, nil
}
+326
View File
@@ -0,0 +1,326 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ldap
import (
"context"
"fmt"
"strconv"
grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/group"
"github.com/cs3org/reva/v2/pkg/group/manager/registry"
"github.com/cs3org/reva/v2/pkg/utils"
ldapIdentity "github.com/cs3org/reva/v2/pkg/utils/ldap"
"github.com/go-ldap/ldap/v3"
"github.com/google/uuid"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
func init() {
registry.Register("ldap", New)
}
type manager struct {
c *config
ldapClient ldap.Client
}
type config struct {
utils.LDAPConn `mapstructure:",squash"`
LDAPIdentity ldapIdentity.Identity `mapstructure:",squash"`
Idp string `mapstructure:"idp"`
// Nobody specifies the fallback gid number for groups that don't have a gidNumber set in LDAP
Nobody int64 `mapstructure:"nobody"`
}
func parseConfig(m map[string]interface{}) (*config, error) {
c := config{
LDAPIdentity: ldapIdentity.New(),
}
if err := mapstructure.Decode(m, &c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return &c, nil
}
// New returns a group manager implementation that connects to a LDAP server to provide group metadata.
func New(m map[string]interface{}) (group.Manager, error) {
mgr := &manager{}
err := mgr.Configure(m)
if err != nil {
return nil, err
}
mgr.ldapClient, err = utils.GetLDAPClientWithReconnect(&mgr.c.LDAPConn)
if err != nil {
return nil, err
}
return mgr, nil
}
// Configure initializes the configuration of the group manager from the supplied config map
func (m *manager) Configure(ml map[string]interface{}) error {
c, err := parseConfig(ml)
if err != nil {
return err
}
if c.Nobody == 0 {
c.Nobody = 99
}
if err = c.LDAPIdentity.Setup(); err != nil {
return fmt.Errorf("error setting up Identity config: %w", err)
}
m.c = c
return nil
}
// GetGroup implements the group.Manager interface. Looks up a group by Id and return the group
func (m *manager) GetGroup(ctx context.Context, gid *grouppb.GroupId, skipFetchingMembers bool) (*grouppb.Group, error) {
log := appctx.GetLogger(ctx)
if gid.Idp != "" && gid.Idp != m.c.Idp {
return nil, errtypes.NotFound("idp mismatch")
}
groupEntry, err := m.c.LDAPIdentity.GetLDAPGroupByID(log, m.ldapClient, gid.OpaqueId)
if err != nil {
return nil, err
}
log.Debug().Interface("entry", groupEntry).Msg("entries")
g, err := m.ldapEntryToGroup(groupEntry)
if err != nil {
return nil, err
}
if skipFetchingMembers {
return g, nil
}
members, err := m.c.LDAPIdentity.GetLDAPGroupMembers(log, m.ldapClient, groupEntry)
if err != nil {
return nil, err
}
memberIDs := make([]*userpb.UserId, 0, len(members))
for _, member := range members {
userid, err := m.ldapEntryToUserID(member)
if err != nil {
log.Warn().Err(err).Interface("member", member).Msg("Failed convert member entry to userid")
continue
}
memberIDs = append(memberIDs, userid)
}
g.Members = memberIDs
return g, nil
}
// GetGroupByClaim implements the group.Manager interface. Looks up a group by
// claim ('group_name', 'group_id', 'display_name') and returns the group.
func (m *manager) GetGroupByClaim(ctx context.Context, claim, value string, skipFetchingMembers bool) (*grouppb.Group, error) {
log := appctx.GetLogger(ctx)
groupEntry, err := m.c.LDAPIdentity.GetLDAPGroupByAttribute(log, m.ldapClient, claim, value)
if err != nil {
log.Debug().Err(err).Msg("GetGroupByClaim")
return nil, err
}
log.Debug().Interface("entry", groupEntry).Msg("entries")
g, err := m.ldapEntryToGroup(groupEntry)
if err != nil {
return nil, err
}
if skipFetchingMembers {
return g, nil
}
members, err := m.c.LDAPIdentity.GetLDAPGroupMembers(log, m.ldapClient, groupEntry)
if err != nil {
return nil, err
}
memberIDs := make([]*userpb.UserId, 0, len(members))
for _, member := range members {
userid, err := m.ldapEntryToUserID(member)
if err != nil {
log.Warn().Err(err).Interface("member", member).Msg("Failed convert member entry to userid")
continue
}
memberIDs = append(memberIDs, userid)
}
g.Members = memberIDs
return g, nil
}
// FindGroups implements the group.Manager interface. Searches for groups using
// a prefix-substring search on the group attributes ('group_name',
// 'display_name', 'group_id') and returns the groups. FindGroups does NOT expand the
// members of the Groups.
func (m *manager) FindGroups(ctx context.Context, query string, skipFetchingMembers bool) ([]*grouppb.Group, error) {
log := appctx.GetLogger(ctx)
entries, err := m.c.LDAPIdentity.GetLDAPGroups(log, m.ldapClient, query)
if err != nil {
return nil, err
}
groups := make([]*grouppb.Group, 0, len(entries))
for _, entry := range entries {
g, err := m.ldapEntryToGroup(entry)
if err != nil {
return nil, err
}
groups = append(groups, g)
}
return groups, nil
}
// GetMembers implements the group.Manager interface. It returns all the userids of the members
// of the group identified by the supplied id.
func (m *manager) GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*userpb.UserId, error) {
log := appctx.GetLogger(ctx)
if gid.Idp != "" && gid.Idp != m.c.Idp {
return nil, errtypes.NotFound("idp mismatch")
}
groupEntry, err := m.c.LDAPIdentity.GetLDAPGroupByID(log, m.ldapClient, gid.OpaqueId)
if err != nil {
return nil, err
}
log.Debug().Interface("entry", groupEntry).Msg("entries")
members, err := m.c.LDAPIdentity.GetLDAPGroupMembers(log, m.ldapClient, groupEntry)
if err != nil {
return nil, err
}
memberIDs := make([]*userpb.UserId, 0, len(members))
for _, member := range members {
userid, err := m.ldapEntryToUserID(member)
if err != nil {
log.Warn().Err(err).Interface("member", member).Msg("Failed convert member entry to userid")
continue
}
memberIDs = append(memberIDs, userid)
}
return memberIDs, nil
}
// HasMember implements the group.Member interface. Checks whether the supplied userid is a member
// of the supplied groupid.
func (m *manager) HasMember(ctx context.Context, gid *grouppb.GroupId, uid *userpb.UserId) (bool, error) {
// It might be possible to do a somewhat more clever LDAP search here. (First lookup the user and then
// search for (&(objectclass=<groupoc>)(<groupid>=gid)(member=<username/userdn>)
// The GetMembers call used below can be quiet ineffecient for large groups
members, err := m.GetMembers(ctx, gid)
if err != nil {
return false, err
}
for _, u := range members {
if u.OpaqueId == uid.OpaqueId && u.Idp == uid.Idp {
return true, nil
}
}
return false, nil
}
func (m *manager) ldapEntryToGroup(entry *ldap.Entry) (*grouppb.Group, error) {
id, err := m.ldapEntryToGroupID(entry)
if err != nil {
return nil, err
}
gidNumber := m.c.Nobody
gidValue := entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.Group.Schema.GIDNumber)
if gidValue != "" {
gidNumber, err = strconv.ParseInt(gidValue, 10, 64)
if err != nil {
return nil, err
}
}
g := &grouppb.Group{
Id: id,
GroupName: entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.Group.Schema.Groupname),
Mail: entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.Group.Schema.Mail),
DisplayName: entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.Group.Schema.DisplayName),
GidNumber: gidNumber,
}
return g, nil
}
func (m *manager) ldapEntryToGroupID(entry *ldap.Entry) (*grouppb.GroupId, error) {
var id string
if m.c.LDAPIdentity.Group.Schema.IDIsOctetString {
rawValue := entry.GetEqualFoldRawAttributeValue(m.c.LDAPIdentity.Group.Schema.ID)
if value, err := uuid.FromBytes(rawValue); err == nil {
id = value.String()
} else {
return nil, err
}
} else {
id = entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.Group.Schema.ID)
}
return &grouppb.GroupId{
Idp: m.c.Idp,
OpaqueId: id,
}, nil
}
func (m *manager) ldapEntryToUserID(entry *ldap.Entry) (*userpb.UserId, error) {
var uid string
if m.c.LDAPIdentity.User.Schema.IDIsOctetString {
rawValue := entry.GetEqualFoldRawAttributeValue(m.c.LDAPIdentity.User.Schema.ID)
var value uuid.UUID
var err error
if value, err = uuid.FromBytes(rawValue); err != nil {
return nil, err
}
uid = value.String()
} else {
uid = entry.GetEqualFoldAttributeValue(m.c.LDAPIdentity.User.Schema.ID)
}
return &userpb.UserId{
Idp: m.c.Idp,
OpaqueId: uid,
Type: userpb.UserType_USER_TYPE_PRIMARY,
}, nil
}
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core group manager drivers.
_ "github.com/cs3org/reva/v2/pkg/group/manager/json"
_ "github.com/cs3org/reva/v2/pkg/group/manager/ldap"
// Add your own here
)
@@ -0,0 +1,34 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import "github.com/cs3org/reva/v2/pkg/group"
// NewFunc is the function that group managers
// should register at init time.
type NewFunc func(map[string]interface{}) (group.Manager, error)
// NewFuncs is a map containing all the registered group managers.
var NewFuncs = map[string]NewFunc{}
// Register registers a new group manager new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package logger
import (
"io"
"os"
"time"
"github.com/rs/zerolog"
)
func init() {
zerolog.CallerSkipFrameCount = 2
zerolog.TimeFieldFormat = time.RFC3339Nano
}
// Mode changes the logging format.
type Mode string
const (
// JSONMode outputs JSON.
JSONMode Mode = "json"
// ConsoleMode outputs human-readable logs.
ConsoleMode Mode = "console"
)
// Option is the option to use to configure the logger.
type Option func(l *zerolog.Logger)
// New creates a new logger.
func New(opts ...Option) *zerolog.Logger {
// create a default logger
zl := zerolog.New(os.Stderr).With().Timestamp().Caller().Logger()
for _, opt := range opts {
opt(&zl)
}
return &zl
}
// WithLevel is an option to configure the logging level.
func WithLevel(lvl string) Option {
return func(l *zerolog.Logger) {
zlvl := parseLevel(lvl)
*l = l.Level(zlvl)
}
}
// WithWriter is an option to configure the logging output.
func WithWriter(w io.Writer, m Mode) Option {
return func(l *zerolog.Logger) {
if m == ConsoleMode {
*l = l.Output(zerolog.ConsoleWriter{Out: w, TimeFormat: "2006-01-02 15:04:05.999"})
} else {
*l = l.Output(w)
}
}
}
func parseLevel(v string) zerolog.Level {
if v == "" {
return zerolog.InfoLevel
}
lvl, err := zerolog.ParseLevel(v)
if err != nil {
return zerolog.InfoLevel
}
return lvl
}
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package accservice
import (
"encoding/json"
"fmt"
"net/url"
"path"
"strings"
"github.com/pkg/errors"
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/cs3org/reva/v2/pkg/mentix/utils/network"
)
// RequestResponse holds the response of an accounts service query.
type RequestResponse struct {
Success bool
Error string
Data interface{}
}
type accountsServiceSettings struct {
URL *url.URL
User string
Password string
}
var settings accountsServiceSettings
// Query performs an account service query.
func Query(endpoint string, params network.URLParams) (*RequestResponse, error) {
fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", settings.URL.Scheme, settings.URL.Host), path.Join(settings.URL.Path, endpoint), params)
if err != nil {
return nil, errors.Wrap(err, "error while building the service accounts query URL")
}
data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: settings.User, Password: settings.Password}, false)
if err != nil {
return nil, errors.Wrap(err, "unable to query the service accounts endpoint")
}
resp := &RequestResponse{}
if err := json.Unmarshal(data, resp); err != nil {
return nil, errors.Wrap(err, "unable to unmarshal response data")
}
return resp, nil
}
// GetResponseValue gets a value from an account service query using a dotted path notation.
func GetResponseValue(resp *RequestResponse, path string) interface{} {
if data, ok := resp.Data.(map[string]interface{}); ok {
tokens := strings.Split(path, ".")
for i, name := range tokens {
if i == len(tokens)-1 {
if value, ok := data[name]; ok {
return value
}
}
if data, ok = data[name].(map[string]interface{}); !ok {
break
}
}
}
return nil
}
// InitAccountsService initializes the global accounts service.
func InitAccountsService(conf *config.Configuration) error {
URL, err := url.Parse(conf.AccountsService.URL)
if err != nil {
return errors.Wrap(err, "unable to parse the accounts service URL")
}
settings.URL = URL
settings.User = conf.AccountsService.User
settings.Password = conf.AccountsService.Password
return nil
}
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package config
// Configuration holds the general Mentix configuration.
type Configuration struct {
Prefix string `mapstructure:"prefix"`
Connectors struct {
GOCDB struct {
Address string `mapstructure:"address"`
Scope string `mapstructure:"scope"`
APIKey string `mapstructure:"apikey"`
} `mapstructure:"gocdb"`
} `mapstructure:"connectors"`
UpdateInterval string `mapstructure:"update_interval"`
Services struct {
CriticalTypes []string `mapstructure:"critical_types"`
} `mapstructure:"services"`
Exporters struct {
WebAPI struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
} `mapstructure:"webapi"`
CS3API struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
ElevatedServiceTypes []string `mapstructure:"elevated_service_types"`
} `mapstructure:"cs3api"`
SiteLocations struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
} `mapstructure:"siteloc"`
PrometheusSD struct {
OutputPath string `mapstructure:"output_path"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
} `mapstructure:"promsd"`
Metrics struct {
EnabledConnectors []string `mapstructure:"enabled_connectors"`
} `mapstructure:"metrics"`
} `mapstructure:"exporters"`
AccountsService struct {
URL string `mapstructure:"url"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
} `mapstructure:"accounts"`
// Internal settings
EnabledConnectors []string `mapstructure:"-"`
EnabledImporters []string `mapstructure:"-"`
EnabledExporters []string `mapstructure:"-"`
}
// Init sets sane defaults.
func (c *Configuration) Init() {
if c.Prefix == "" {
c.Prefix = "mentix"
}
// TODO(daniel): add default that works out of the box
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package config
const (
// ConnectorIDGOCDB is the connector identifier for GOCDB.
ConnectorIDGOCDB = "gocdb"
)
const (
// ExporterIDWebAPI is the identifier for the WebAPI exporter.
ExporterIDWebAPI = "webapi"
// ExporterIDCS3API is the identifier for the CS3API exporter.
ExporterIDCS3API = "cs3api"
// ExporterIDSiteLocations is the identifier for the Site Locations exporter.
ExporterIDSiteLocations = "siteloc"
// ExporterIDPrometheusSD is the identifier for the PrometheusSD exporter.
ExporterIDPrometheusSD = "promsd"
// ExporterIDMetrics is the identifier for the Metrics exporter.
ExporterIDMetrics = "metrics"
)
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"fmt"
"github.com/rs/zerolog"
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/cs3org/reva/v2/pkg/mentix/entity"
"github.com/cs3org/reva/v2/pkg/mentix/meshdata"
)
// Connector is the interface that all connectors must implement.
type Connector interface {
entity.Entity
// RetrieveMeshData fetches new mesh data.
RetrieveMeshData() (*meshdata.MeshData, error)
// UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that
// should be updated, not the entire data set.
UpdateMeshData(data *meshdata.MeshData) error
}
// BaseConnector implements basic connector functionality common to all connectors.
type BaseConnector struct {
conf *config.Configuration
log *zerolog.Logger
}
// Activate activates the connector.
func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if conf == nil {
return fmt.Errorf("no configuration provided")
}
connector.conf = conf
if log == nil {
return fmt.Errorf("no logger provided")
}
connector.log = log
return nil
}
// UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that
// should be updated, not the entire data set.
func (connector *BaseConnector) UpdateMeshData(data *meshdata.MeshData) error {
return fmt.Errorf("the connector doesn't support updating of mesh data")
}
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"github.com/rs/zerolog"
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/cs3org/reva/v2/pkg/mentix/entity"
)
var (
registeredConnectors = entity.NewRegistry()
)
// Collection represents a collection of connectors.
type Collection struct {
Connectors []Connector
}
// Entities gets the entities in this collection.
func (collection *Collection) Entities() []entity.Entity {
entities := make([]entity.Entity, 0, len(collection.Connectors))
for _, connector := range collection.Connectors {
entities = append(entities, connector)
}
return entities
}
// ActivateAll activates all entities in the collection.
func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error {
return entity.ActivateEntities(collection, conf, log)
}
// AvailableConnectors returns a collection of all connectors that are enabled in the configuration.
func AvailableConnectors(conf *config.Configuration) (*Collection, error) {
entities, err := registeredConnectors.FindEntities(conf.EnabledConnectors, true, true)
if err != nil {
return nil, err
}
connectors := make([]Connector, 0, len(entities))
for _, entry := range entities {
connectors = append(connectors, entry.(Connector))
}
return &Collection{Connectors: connectors}, nil
}
func registerConnector(connector Connector) {
registeredConnectors.Register(connector)
}
+317
View File
@@ -0,0 +1,317 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"encoding/xml"
"fmt"
"net/url"
"path"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/mentix/utils"
"github.com/rs/zerolog"
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/cs3org/reva/v2/pkg/mentix/connectors/gocdb"
"github.com/cs3org/reva/v2/pkg/mentix/meshdata"
"github.com/cs3org/reva/v2/pkg/mentix/utils/network"
)
// GOCDBConnector is used to read mesh data from a GOCDB instance.
type GOCDBConnector struct {
BaseConnector
gocdbAddress string
}
// Activate activates the connector.
func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := connector.BaseConnector.Activate(conf, log); err != nil {
return err
}
// Check and store GOCDB specific settings
connector.gocdbAddress = conf.Connectors.GOCDB.Address
if len(connector.gocdbAddress) == 0 {
return fmt.Errorf("no GOCDB address configured")
}
return nil
}
// RetrieveMeshData fetches new mesh data.
func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) {
meshData := new(meshdata.MeshData)
// Query all data from GOCDB
if err := connector.queryServiceTypes(meshData); err != nil {
return nil, fmt.Errorf("could not query service types: %v", err)
}
if err := connector.querySites(meshData); err != nil {
return nil, fmt.Errorf("could not query sites: %v", err)
}
for _, site := range meshData.Sites {
// Get services associated with the current site
if err := connector.queryServices(meshData, site); err != nil {
return nil, fmt.Errorf("could not query services of site '%v': %v", site.Name, err)
}
// Get downtimes scheduled for the current site
if err := connector.queryDowntimes(meshData, site); err != nil {
return nil, fmt.Errorf("could not query downtimes of site '%v': %v", site.Name, err)
}
}
meshData.InferMissingData()
return meshData, nil
}
func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error {
var scope string
if hasScope {
scope = connector.conf.Connectors.GOCDB.Scope
}
// Get the data from GOCDB
data, err := gocdb.QueryGOCDB(connector.gocdbAddress, method, isPrivate, scope, connector.conf.Connectors.GOCDB.APIKey, params)
if err != nil {
return err
}
// Unmarshal it
if err := xml.Unmarshal(data, v); err != nil {
return fmt.Errorf("unable to unmarshal data: %v", err)
}
return nil
}
func (connector *GOCDBConnector) queryServiceTypes(meshData *meshdata.MeshData) error {
var serviceTypes gocdb.ServiceTypes
if err := connector.query(&serviceTypes, "get_service_types", false, false, network.URLParams{}); err != nil {
return err
}
// Copy retrieved data into the mesh data
meshData.ServiceTypes = nil
for _, serviceType := range serviceTypes.Types {
meshData.ServiceTypes = append(meshData.ServiceTypes, &meshdata.ServiceType{
Name: serviceType.Name,
Description: serviceType.Description,
})
}
return nil
}
func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error {
var sites gocdb.Sites
if err := connector.query(&sites, "get_site", false, true, network.URLParams{}); err != nil {
return err
}
// Copy retrieved data into the mesh data
meshData.Sites = nil
for _, site := range sites.Sites {
properties := connector.extensionsToMap(&site.Extensions)
// The site ID can be set through a property; by default, the site short name will be used
siteID := meshdata.GetPropertyValue(properties, meshdata.PropertySiteID, site.ShortName)
// See if an organization has been defined using properties; otherwise, use the official name
organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName)
meshsite := &meshdata.Site{
ID: siteID,
Name: site.ShortName,
FullName: site.OfficialName,
Organization: organization,
Domain: site.Domain,
Homepage: site.Homepage,
Email: site.Email,
Description: site.Description,
Country: site.Country,
CountryCode: site.CountryCode,
Longitude: site.Longitude,
Latitude: site.Latitude,
Services: nil,
Properties: properties,
Downtimes: meshdata.Downtimes{},
}
meshData.Sites = append(meshData.Sites, meshsite)
}
return nil
}
func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site *meshdata.Site) error {
var services gocdb.Services
if err := connector.query(&services, "get_service", false, true, network.URLParams{"sitename": site.Name}); err != nil {
return err
}
getServiceURLString := func(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint, host string) string {
urlstr := "https://" + host // Fall back to the provided hostname
if svcURL, err := connector.getServiceURL(service, endpoint); err == nil {
urlstr = svcURL.String()
}
return urlstr
}
// Copy retrieved data into the mesh data
site.Services = nil
for _, service := range services.Services {
host := service.Host
// If a URL is provided, extract the port from it and append it to the host
if len(service.URL) > 0 {
if hostURL, err := url.Parse(service.URL); err == nil {
if port := hostURL.Port(); len(port) > 0 {
host += ":" + port
}
}
}
// Assemble additional endpoints
var endpoints []*meshdata.ServiceEndpoint
for _, endpoint := range service.Endpoints.Endpoints {
endpoints = append(endpoints, &meshdata.ServiceEndpoint{
Type: connector.findServiceType(meshData, endpoint.Type),
Name: endpoint.Name,
RawURL: endpoint.URL,
URL: getServiceURLString(service, endpoint, host),
IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"),
Properties: connector.extensionsToMap(&endpoint.Extensions),
})
}
// Add the service to the site
site.Services = append(site.Services, &meshdata.Service{
ServiceEndpoint: &meshdata.ServiceEndpoint{
Type: connector.findServiceType(meshData, service.Type),
Name: service.Type,
RawURL: service.URL,
URL: getServiceURLString(service, nil, host),
IsMonitored: strings.EqualFold(service.IsMonitored, "Y"),
Properties: connector.extensionsToMap(&service.Extensions),
},
Host: host,
AdditionalEndpoints: endpoints,
})
}
return nil
}
func (connector *GOCDBConnector) queryDowntimes(meshData *meshdata.MeshData, site *meshdata.Site) error {
var downtimes gocdb.Downtimes
if err := connector.query(&downtimes, "get_downtime_nested_services", false, true, network.URLParams{"topentity": site.Name, "ongoing_only": "yes"}); err != nil {
return err
}
// Copy retrieved data into the mesh data
site.Downtimes.Clear()
for _, dt := range downtimes.Downtimes {
if !strings.EqualFold(dt.Severity, "outage") { // Only take real outages into account
continue
}
services := make([]string, 0, len(dt.AffectedServices.Services))
for _, service := range dt.AffectedServices.Services {
// Only add critical services to the list of affected services
if utils.FindInStringArray(service.Type, connector.conf.Services.CriticalTypes, false) != -1 {
services = append(services, service.Type)
}
}
_, _ = site.Downtimes.ScheduleDowntime(time.Unix(dt.StartDate, 0), time.Unix(dt.EndDate, 0), services)
}
return nil
}
func (connector *GOCDBConnector) findServiceType(meshData *meshdata.MeshData, name string) *meshdata.ServiceType {
for _, serviceType := range meshData.ServiceTypes {
if strings.EqualFold(serviceType.Name, name) {
return serviceType
}
}
// If the service type doesn't exist, create a default one
return &meshdata.ServiceType{Name: name, Description: ""}
}
func (connector *GOCDBConnector) extensionsToMap(extensions *gocdb.Extensions) map[string]string {
properties := make(map[string]string)
for _, ext := range extensions.Extensions {
properties[ext.Key] = ext.Value
}
return properties
}
func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint) (*url.URL, error) {
urlstr := service.URL
if len(urlstr) == 0 {
// The URL defaults to the hostname using the HTTPS protocol
urlstr = "https://" + service.Host
}
svcURL, err := url.ParseRequestURI(urlstr)
if err != nil {
return nil, fmt.Errorf("unable to parse URL '%v': %v", urlstr, err)
}
// If an endpoint was provided, use its path
if endpoint != nil {
// If the endpoint URL is an absolute one, just use that; otherwise, make an absolute one out of it
if endpointURL, err := url.ParseRequestURI(endpoint.URL); err == nil && len(endpointURL.Scheme) > 0 {
svcURL = endpointURL
} else {
// Replace entire URL path if the relative path starts with a slash; otherwise, just append
if strings.HasPrefix(endpoint.URL, "/") {
svcURL.Path = endpoint.URL
} else {
svcURL.Path = path.Join(svcURL.Path, endpoint.URL)
if strings.HasSuffix(endpoint.URL, "/") { // Restore trailing slash if necessary
svcURL.Path += "/"
}
}
}
}
return svcURL, nil
}
// GetID returns the ID of the connector.
func (connector *GOCDBConnector) GetID() string {
return config.ConnectorIDGOCDB
}
// GetName returns the display name of the connector.
func (connector *GOCDBConnector) GetName() string {
return "GOCDB"
}
func init() {
registerConnector(&GOCDBConnector{})
}
+61
View File
@@ -0,0 +1,61 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package gocdb
import (
"fmt"
"github.com/cs3org/reva/v2/pkg/mentix/utils/network"
)
// QueryGOCDB retrieves data from one of GOCDB's endpoints.
func QueryGOCDB(address string, method string, isPrivate bool, scope string, apiKey string, params network.URLParams) ([]byte, error) {
// The method must always be specified
params["method"] = method
// If a scope or an API key were specified, pass them to the endpoint as well
if len(scope) > 0 {
params["scope"] = scope
}
if len(apiKey) > 0 {
params["apikey"] = apiKey
}
// GOCDB's public API is located at <gocdb-host>/gocdbpi/public, the private one at <gocdb-host>/gocdbpi/private
var path string
if isPrivate {
path = "/gocdbpi/private"
} else {
path = "/gocdbpi/public"
}
// Query the data from GOCDB
endpointURL, err := network.GenerateURL(address, path, params)
if err != nil {
return nil, fmt.Errorf("unable to generate the GOCDB URL: %v", err)
}
data, err := network.ReadEndpoint(endpointURL, nil, true)
if err != nil {
return nil, fmt.Errorf("unable to read GOCDB endpoint: %v", err)
}
return data, nil
}
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package gocdb
// Extension represents Key-Value pairs in GOCDB.
type Extension struct {
Key string `xml:"KEY"`
Value string `xml:"VALUE"`
}
// Extensions is a list of Extension objects.
type Extensions struct {
Extensions []*Extension `xml:"EXTENSION"`
}
// ServiceType represents a service type in GOCDB.
type ServiceType struct {
Name string `xml:"SERVICE_TYPE_NAME"`
Description string `xml:"SERVICE_TYPE_DESC"`
}
// ServiceTypes is a list of ServiceType objects.
type ServiceTypes struct {
Types []*ServiceType `xml:"SERVICE_TYPE"`
}
// Site represents a site in GOCDB.
type Site struct {
ShortName string `xml:"SHORT_NAME"`
OfficialName string `xml:"OFFICIAL_NAME"`
Description string `xml:"SITE_DESCRIPTION"`
Homepage string `xml:"HOME_URL"`
Email string `xml:"CONTACT_EMAIL"`
Domain string `xml:"DOMAIN>DOMAIN_NAME"`
Country string `xml:"COUNTRY"`
CountryCode string `xml:"COUNTRY_CODE"`
Latitude float32 `xml:"LATITUDE"`
Longitude float32 `xml:"LONGITUDE"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// Sites is a list of Site objects.
type Sites struct {
Sites []*Site `xml:"SITE"`
}
// ServiceEndpoint represents an additional service endpoint of a service in GOCDB.
type ServiceEndpoint struct {
Name string `xml:"NAME"`
URL string `xml:"URL"`
Type string `xml:"INTERFACENAME"`
IsMonitored string `xml:"ENDPOINT_MONITORED"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// ServiceEndpoints is a list of ServiceEndpoint objects.
type ServiceEndpoints struct {
Endpoints []*ServiceEndpoint `xml:"ENDPOINT"`
}
// Service represents a service in GOCDB.
type Service struct {
Host string `xml:"HOSTNAME"`
Type string `xml:"SERVICE_TYPE"`
IsMonitored string `xml:"NODE_MONITORED"`
URL string `xml:"URL"`
Endpoints ServiceEndpoints `xml:"ENDPOINTS"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// Services is a list of Service objects.
type Services struct {
Services []*Service `xml:"SERVICE_ENDPOINT"`
}
// DowntimeService represents a service scheduled for downtime.
type DowntimeService struct {
Type string `xml:"SERVICE_TYPE"`
}
// DowntimeServices represents a list of DowntimeService objects.
type DowntimeServices struct {
Services []*DowntimeService `xml:"SERVICE"`
}
// Downtime is a scheduled downtime for a site.
type Downtime struct {
Severity string `xml:"SEVERITY"`
StartDate int64 `xml:"START_DATE"`
EndDate int64 `xml:"END_DATE"`
AffectedServices DowntimeServices `xml:"SERVICES"`
}
// Downtimes represents a list of Downtime objects.
type Downtimes struct {
Downtimes []*Downtime `xml:"DOWNTIME"`
}
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package entity
import (
"fmt"
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/rs/zerolog"
)
// Collection is an interface for entity collections.
type Collection interface {
// Entities returns a vector of entities within the collection.
Entities() []Entity
}
// ActivateEntities activates the given entities.
func ActivateEntities(collection Collection, conf *config.Configuration, log *zerolog.Logger) error {
for _, exchanger := range collection.Entities() {
if err := exchanger.Activate(conf, log); err != nil {
return fmt.Errorf("unable to activate entity '%v': %v", exchanger.GetName(), err)
}
}
return nil
}
// GetIDs gets a list of entity IDs.
func GetIDs(collection Collection) []string {
entities := collection.Entities()
ids := make([]string, 0, len(entities))
for _, entity := range entities {
ids = append(ids, entity.GetID())
}
return ids
}
// GetNames gets a list of entity names.
func GetNames(collection Collection) []string {
entities := collection.Entities()
names := make([]string, 0, len(entities))
for _, entity := range entities {
names = append(names, entity.GetName())
}
return names
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package entity
import (
"github.com/cs3org/reva/v2/pkg/mentix/config"
"github.com/rs/zerolog"
)
// Entity is the base interface for all Mentix entities.
type Entity interface {
// GetID returns the ID of the entity.
GetID() string
// GetName returns the display name of the entity.
GetName() string
// Activate activates the entity.
Activate(conf *config.Configuration, log *zerolog.Logger) error
}

Some files were not shown because too many files have changed in this diff Show More