diff --git a/changelog/unreleased/ocm.md b/changelog/unreleased/ocm.md index c1bf6d459..d8e08f23f 100644 --- a/changelog/unreleased/ocm.md +++ b/changelog/unreleased/ocm.md @@ -2,6 +2,7 @@ Enhancement: Add ocm and sciencemesh services We added sciencemesh and ocm services to enable federation. +https://github.com/owncloud/ocis/pull/7998 https://github.com/owncloud/ocis/pull/7576 https://github.com/owncloud/ocis/pull/7464 https://github.com/owncloud/ocis/pull/7463 diff --git a/go.mod b/go.mod index 72b8c2abd..a0461f0ba 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d + github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e @@ -349,4 +349,4 @@ require ( replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 -replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 +replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f diff --git a/go.sum b/go.sum index b2dba7dca..e12647b94 100644 --- a/go.sum +++ b/go.sum @@ -827,8 +827,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 h1:m63hhLqbqmLGGPtyTtjTdxae61d9tMbRdKvMaDHWcDs= -github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= +github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f h1:rxzQfsnLmEm5YnAf0KDoTmswnnTX9whwAsFT7n1I1kk= +github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -1021,10 +1021,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47 h1:6DfMeFpCXoqlfm/+FJ/mFs8Ul5WCZNlorsbDM9Z/ATE= -github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47/go.mod h1:oX1YtLKGr7jatGk0CpPM4GKbSEIdHhmsQuSAYElnN1U= -github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d h1:OYkjbcOAntD5JBMAuyj+bR1bg5jM+BjRvFBiTlmnxWQ= -github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d/go.mod h1:JyvlRw2v8BTH5t+ISj1Yc+EMDBcyf8LMB5o98HufWis= +github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38 h1:vkA/Ty82yETTDrpLV/b5/9VXUJ/9o7vTRs7ampZC5LU= +github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38/go.mod h1:5Yxh1DneWZQvMBOiBVv3LJaSsSlRCoqCBa4Wws7PWHw= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/services/ocm/pkg/config/config.go b/services/ocm/pkg/config/config.go index 07c2ae7a3..d7fb74210 100644 --- a/services/ocm/pkg/config/config.go +++ b/services/ocm/pkg/config/config.go @@ -17,11 +17,12 @@ type Config struct { Log *Log `yaml:"log"` Debug Debug `yaml:"debug"` - HTTP HTTPConfig `yaml:"http"` - Middleware Middleware `yaml:"middleware"` - GRPC GRPCConfig `yaml:"grpc"` - GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` - GrpcClient client.Client `yaml:"-"` + HTTP HTTPConfig `yaml:"http"` + Middleware Middleware `yaml:"middleware"` + GRPC GRPCConfig `yaml:"grpc"` + GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` + GrpcClient client.Client `yaml:"-"` + ServiceAccount ServiceAccount `yaml:"service_account"` Reva *shared.Reva `yaml:"reva"` OCMD OCMD `yaml:"ocmd"` @@ -56,6 +57,12 @@ type Auth struct { CredentialsByUserAgent map[string]string `yaml:"credentials_by_user_agent"` } +// ServiceAccount is the configuration for the used service account +type ServiceAccount struct { + ID string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;OCM_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details."` + Secret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;OCM_SERVICE_ACCOUNT_SECRET" desc:"The service account secret."` +} + // CORS defines the available cors configuration. type CORS struct { AllowedOrigins []string `yaml:"allow_origins" env:"OCIS_CORS_ALLOW_ORIGINS;OCM_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details."` diff --git a/services/ocm/pkg/revaconfig/config.go b/services/ocm/pkg/revaconfig/config.go index d221501ed..7b87a60b1 100644 --- a/services/ocm/pkg/revaconfig/config.go +++ b/services/ocm/pkg/revaconfig/config.go @@ -56,7 +56,10 @@ func OCMConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]inter "driver": "ocmreceived", "drivers": map[string]interface{}{ "ocmreceived": map[string]interface{}{ - "insecure": cfg.OCMStorageProvider.Insecure, + "insecure": cfg.OCMStorageProvider.Insecure, + "storage_root": cfg.OCMStorageProvider.StorageRoot, + "service_account_id": cfg.ServiceAccount.ID, + "service_account_secret": cfg.ServiceAccount.Secret, }, }, "data_txs": map[string]interface{}{ diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 88d775eaf..0f68136fd 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -515,6 +515,14 @@ func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateRec func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMShareRequest) (*ocm.GetReceivedOCMShareResponse, error) { user := ctxpkg.ContextMustGetUser(ctx) + if user.Id.GetType() == userpb.UserType_USER_TYPE_SERVICE { + var uid userpb.UserId + _ = utils.ReadJSONFromOpaque(req.Opaque, "userid", &uid) + user = &userpb.User{ + Id: &uid, + } + } + ocmshare, err := s.repo.GetReceivedShare(ctx, user, req.Ref) if err != nil { if errors.Is(err, share.ErrShareNotFound) { @@ -523,7 +531,7 @@ func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedO }, nil } return &ocm.GetReceivedOCMShareResponse{ - Status: status.NewInternal(ctx, "error getting received share"), + Status: status.NewInternal(ctx, "error getting received share: "+err.Error()), }, nil } diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go index 38b64f49d..4cfd98cb2 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -68,6 +68,7 @@ type config struct { } type passwordPolicy struct { + Disabled bool `mapstructure:"disabled"` MinCharacters int `mapstructure:"min_characters"` MinLowerCaseCharacters int `mapstructure:"min_lowercase_characters"` MinUpperCaseCharacters int `mapstructure:"min_uppercase_characters"` @@ -173,9 +174,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { func newPasswordPolicy(c *passwordPolicy) password.Validator { if c == nil { - return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( + c.Disabled, c.MinCharacters, c.MinLowerCaseCharacters, c.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go index e648582c0..cb52ea695 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/put.go @@ -137,6 +137,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ length, err := getContentLength(w, r) if err != nil { + log.Error().Err(err).Msg("error getting the content length") w.WriteHeader(http.StatusBadRequest) return } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go index 369ca2317..f6abda35b 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go @@ -117,6 +117,7 @@ type CapabilitiesGraph struct { // CapabilitiesPasswordPolicy hold the password policy capabilities type CapabilitiesPasswordPolicy struct { + Disabled bool `json:"disabled" xml:"disabled" mapstructure:"disabled"` MinCharacters int `json:"min_characters" xml:"min_characters" mapstructure:"min_characters"` MaxCharacters int `json:"max_characters" xml:"max_characters" mapstructure:"max_characters"` MinLowerCaseCharacters int `json:"min_lowercase_characters" xml:"min_lowercase_characters" mapstructure:"min_lowercase_characters"` diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 22265a857..303fc95d4 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -1734,9 +1734,10 @@ func publicPwdEnforced(c *config.Config) passwordEnforced { func passwordPolicies(c *config.Config) password.Validator { if c.Capabilities.Capabilities == nil || c.Capabilities.Capabilities.PasswordPolicy == nil { - return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( + c.Capabilities.Capabilities.PasswordPolicy.Disabled, c.Capabilities.Capabilities.PasswordPolicy.MinCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinLowerCaseCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go index 95ab95492..28acb07a0 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/ocm.go @@ -30,6 +30,7 @@ import ( "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" @@ -59,8 +60,11 @@ type driver struct { } type config struct { - GatewaySVC string `mapstructure:"gatewaysvc"` - Insecure bool `mapstructure:"insecure"` + GatewaySVC string `mapstructure:"gatewaysvc"` + Insecure bool `mapstructure:"insecure"` + StorageRoot string `mapstructure:"storage_root"` + ServiceAccountID string `mapstructure:"service_account_id"` + ServiceAccountSecret string `mapstructure:"service_account_secret"` } func (c *config) ApplyDefaults() { @@ -136,15 +140,19 @@ func shareInfoFromReference(ref *provider.Reference) (*ocmpb.ShareId, string) { } -func (d *driver) getWebDAVFromShare(ctx context.Context, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) { +func (d *driver) getWebDAVFromShare(ctx context.Context, forUser *userpb.UserId, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) { // TODO: we may want to cache the share - res, err := d.gateway.GetReceivedOCMShare(ctx, &ocmpb.GetReceivedOCMShareRequest{ + req := &ocmpb.GetReceivedOCMShareRequest{ Ref: &ocmpb.ShareReference{ Spec: &ocmpb.ShareReference_Id{ Id: shareID, }, }, - }) + } + if forUser != nil { + req.Opaque = utils.AppendJSONToOpaque(nil, "userid", forUser) + } + res, err := d.gateway.GetReceivedOCMShare(ctx, req) if err != nil { return nil, "", "", err } @@ -173,10 +181,10 @@ func getWebDAVProtocol(protocols []*ocmpb.Protocol) (*ocmpb.WebDAVProtocol, bool return nil, false } -func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) { +func (d *driver) webdavClient(ctx context.Context, forUser *userpb.UserId, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) { id, rel := shareInfoFromReference(ref) - share, endpoint, secret, err := d.getWebDAVFromShare(ctx, id) + share, endpoint, secret, err := d.getWebDAVFromShare(ctx, forUser, id) if err != nil { return nil, nil, "", err } @@ -199,7 +207,7 @@ func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*go } func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -207,7 +215,7 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { } func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -215,7 +223,7 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { } func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return err } @@ -223,7 +231,7 @@ func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markpro } func (d *driver) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { - client, _, relOld, err := d.webdavClient(ctx, oldRef) + client, _, relOld, err := d.webdavClient(ctx, nil, oldRef) if err != nil { return err } @@ -263,7 +271,7 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc } webdavProtocol, _ := getWebDAVProtocol(share.Protocols) - return &provider.ResourceInfo{ + ri := provider.ResourceInfo{ Type: t, Id: id, MimeType: mime.Detect(f.IsDir(), f.Name()), @@ -278,11 +286,17 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc Checksum: &provider.ResourceChecksum{ Type: provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, }, - }, nil + } + + if f.(gowebdav.File).StatusCode() == 425 { + ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "status", "processing") + } + + return &ri, nil } func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, _ []string) (*provider.ResourceInfo, error) { - client, share, rel, err := d.webdavClient(ctx, ref) + client, share, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } @@ -299,7 +313,7 @@ func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, } func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []string, _ []string) ([]*provider.ResourceInfo, error) { - client, share, rel, err := d.webdavClient(ctx, ref) + client, share, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } @@ -320,35 +334,8 @@ func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []st return res, nil } -func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, _ int64, _ map[string]string) (map[string]string, error) { - shareID, rel := shareInfoFromReference(ref) - p := getPathFromShareIDAndRelPath(shareID, rel) - - return map[string]string{ - "simple": p, - }, nil -} - -func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) { - client, _, rel, err := d.webdavClient(ctx, req.Ref) - if err != nil { - return provider.ResourceInfo{}, err - } - client.SetInterceptor(func(method string, rq *http.Request) { - // Set the content length on the request struct directly instead of the header. - // The content-length header gets reset by the golang http library before - // sendind out the request, resulting in chunked encoding to be used which - // breaks the quota checks in ocdav. - if method == "PUT" { - rq.ContentLength = req.Length - } - }) - - return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0) -} - func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { - client, _, rel, err := d.webdavClient(ctx, ref) + client, _, rel, err := d.webdavClient(ctx, nil, ref) if err != nil { return nil, err } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/upload.go b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/upload.go new file mode 100644 index 000000000..1e4a5c1fd --- /dev/null +++ b/vendor/github.com/cs3org/reva/v2/pkg/ocm/storage/received/upload.go @@ -0,0 +1,405 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocm + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "hash/adler32" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/google/uuid" + tusd "github.com/tus/tusd/pkg/handler" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/storage" + "github.com/cs3org/reva/v2/pkg/utils" +) + +var defaultFilePerm = os.FileMode(0664) + +func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + return []storage.UploadSession{}, nil +} +func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + shareID, rel := shareInfoFromReference(ref) + p := getPathFromShareIDAndRelPath(shareID, rel) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := d.NewUpload(ctx, info) + if err != nil { + return nil, err + } + + info, _ = upload.GetInfo(ctx) + + return map[string]string{ + "simple": info.ID, + "tus": info.ID, + }, nil +} + +func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) { + shareID, _ := shareInfoFromReference(req.Ref) + u, err := d.GetUpload(ctx, shareID.OpaqueId) + if err != nil { + return provider.ResourceInfo{}, err + } + + info, err := u.GetInfo(ctx) + if err != nil { + return provider.ResourceInfo{}, err + } + + client, _, rel, err := d.webdavClient(ctx, nil, &provider.Reference{ + Path: filepath.Join(info.MetaData["dir"], info.MetaData["filename"]), + }) + if err != nil { + return provider.ResourceInfo{}, err + } + client.SetInterceptor(func(method string, rq *http.Request) { + // Set the content length on the request struct directly instead of the header. + // The content-length header gets reset by the golang http library before + // sendind out the request, resulting in chunked encoding to be used which + // breaks the quota checks in ocdav. + if method == "PUT" { + rq.ContentLength = req.Length + } + }) + + return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0) +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (d *driver) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(d) + composer.UseTerminater(d) + composer.UseConcater(d) + composer.UseLengthDeferrer(d) +} + +// AsTerminatableUpload returns a TerminatableUpload +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// the storage needs to implement AsTerminatableUpload +func (d *driver) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*upload) +} + +// AsLengthDeclarableUpload returns a LengthDeclarableUpload +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// the storage needs to implement AsLengthDeclarableUpload +func (d *driver) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*upload) +} + +// AsConcatableUpload returns a ConcatableUpload +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// the storage needs to implement AsConcatableUpload +func (d *driver) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*upload) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +// NewUpload returns a new tus Upload instance +func (d *driver) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) { + return NewUpload(ctx, d, d.c.StorageRoot, info) +} + +// GetUpload returns the Upload for the given upload id +func (d *driver) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + return GetUpload(ctx, d, d.c.StorageRoot, id) +} +func NewUpload(ctx context.Context, d *driver, storageRoot string, info tusd.FileInfo) (tusd.Upload, error) { + if info.MetaData["filename"] == "" { + return nil, errors.New("Decomposedfs: missing filename in metadata") + } + if info.MetaData["dir"] == "" { + return nil, errors.New("Decomposedfs: missing dir in metadata") + } + + uploadRoot := filepath.Join(storageRoot, "uploads") + info.ID = uuid.New().String() + + user, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + return nil, errors.New("no user in context") + } + info.MetaData["user"] = user.GetId().GetOpaqueId() + info.MetaData["idp"] = user.GetId().GetIdp() + + info.Storage = map[string]string{ + "Type": "OCM", + "Path": uploadRoot, + } + + u := &upload{ + Info: info, + Ctx: ctx, + d: d, + } + + err := os.MkdirAll(uploadRoot, 0755) + if err != nil { + return nil, err + } + + file, err := os.OpenFile(u.BinPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + err = u.Persist() + if err != nil { + return nil, err + } + return u, nil +} + +func GetUpload(ctx context.Context, d *driver, storageRoot string, id string) (tusd.Upload, error) { + info := tusd.FileInfo{} + data, err := os.ReadFile(filepath.Join(storageRoot, "uploads", id+".info")) + if err != nil { + return nil, err + } + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + upload := &upload{ + Info: info, + Ctx: ctx, + d: d, + } + return upload, nil +} + +type upload struct { + Info tusd.FileInfo + Ctx context.Context + + d *driver +} + +func (u *upload) InfoPath() string { + return filepath.Join(u.Info.Storage["Path"], u.Info.ID+".info") +} + +func (u *upload) BinPath() string { + return filepath.Join(u.Info.Storage["Path"], u.Info.ID) +} + +func (u *upload) Persist() error { + data, err := json.Marshal(u.Info) + if err != nil { + return err + } + return os.WriteFile(u.InfoPath(), data, defaultFilePerm) +} + +func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum + // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... + // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used + // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for the ocis driver it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil && err != io.ErrUnexpectedEOF { + return n, err + } + + u.Info.Offset += n + return n, u.Persist() +} + +func (u *upload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return u.Info, nil +} + +func (u *upload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(u.BinPath()) +} + +func (u *upload) FinishUpload(ctx context.Context) error { + log := appctx.GetLogger(u.Ctx) + + // calculate the checksum of the written bytes + // they will all be written to the metadata later, so we cannot omit any of them + // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present + // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... + sha1h := sha1.New() + md5h := md5.New() + adler32h := adler32.New() + { + f, err := os.Open(u.BinPath()) + if err != nil { + // we can continue if no oc checksum header is set + log.Info().Err(err).Str("binPath", u.BinPath()).Msg("error opening binPath") + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + + _, err = io.Copy(adler32h, r2) + if err != nil { + log.Info().Err(err).Msg("error copying checksums") + } + } + + // compare if they match the sent checksum + // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads + if u.Info.MetaData["checksum"] != "" { + var err error + parts := strings.SplitN(u.Info.MetaData["checksum"], " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1": + err = u.checkHash(parts[1], sha1h) + case "md5": + err = u.checkHash(parts[1], md5h) + case "adler32": + err = u.checkHash(parts[1], adler32h) + default: + err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if err != nil { + u.cleanup() + return err + } + } + + // send to the remote storage via webdav + // shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"]) + // p := getPathFromShareIDAndRelPath(shareID, rel) + + serviceUserCtx, err := utils.GetServiceUserContext(u.d.c.ServiceAccountID, u.d.gateway, u.d.c.ServiceAccountSecret) + if err != nil { + return err + } + client, _, rel, err := u.d.webdavClient(serviceUserCtx, &userpb.UserId{ + OpaqueId: u.Info.MetaData["user"], + Idp: u.Info.MetaData["idp"], + }, &provider.Reference{ + Path: filepath.Join(u.Info.MetaData["dir"], u.Info.MetaData["filename"]), + }) + if err != nil { + u.cleanup() + return err + } + + client.SetInterceptor(func(method string, rq *http.Request) { + // Set the content length on the request struct directly instead of the header. + // The content-length header gets reset by the golang http library before + // sendind out the request, resulting in chunked encoding to be used which + // breaks the quota checks in ocdav. + if method == "PUT" { + rq.ContentLength = u.Info.Size + } + }) + + f, err := os.Open(u.BinPath()) + if err != nil { + return err + } + defer f.Close() + return client.WriteStream(rel, f, 0) +} + +func (u *upload) cleanup() { + _ = os.Remove(u.BinPath()) + _ = os.Remove(u.InfoPath()) +} + +func (u *upload) Terminate(ctx context.Context) error { + u.cleanup() + return nil +} + +func (u *upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) error { + file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*upload) + + src, err := os.Open(fileUpload.BinPath()) + if err != nil { + return err + } + defer src.Close() + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + return nil +} + +func (u *upload) DeclareLength(ctx context.Context, length int64) error { + u.Info.Size = length + u.Info.SizeIsDeferred = false + return nil +} + +func (u *upload) checkHash(expected string, h hash.Hash) error { + if expected != hex.EncodeToString(h.Sum(nil)) { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", u.Info.MetaData["checksum"], h.Sum(nil))) + } + return nil +} diff --git a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go index ec98783d3..ac126c9e8 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go @@ -18,6 +18,7 @@ type Validator interface { // Policies represents a password validation rules type Policies struct { + disabled bool minCharacters int minLowerCaseCharacters int minUpperCaseCharacters int @@ -29,8 +30,9 @@ type Policies struct { } // NewPasswordPolicy returns a new NewPasswordPolicy instance -func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { +func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { p := &Policies{ + disabled: disabled, minCharacters: minCharacters, minLowerCaseCharacters: minLowerCaseCharacters, minUpperCaseCharacters: minUpperCaseCharacters, @@ -46,6 +48,9 @@ func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharac // Validate implements a password validation regarding the policy func (s Policies) Validate(str string) error { + if s.disabled { + return nil + } var allErr error if !utf8.ValidString(str) { return fmt.Errorf("the password contains invalid characters") diff --git a/vendor/github.com/studio-b12/gowebdav/client.go b/vendor/github.com/studio-b12/gowebdav/client.go index 4a4bb1b6c..06a9a94df 100644 --- a/vendor/github.com/studio-b12/gowebdav/client.go +++ b/vendor/github.com/studio-b12/gowebdav/client.go @@ -159,47 +159,63 @@ func (p *propstat) Modified() time.Time { return time.Unix(0, 0) } +func (p *propstat) StatusCode() int { + parts := strings.Split(p.Status, " ") + if len(parts) < 2 { + return -1 + } + + code, err := strconv.Atoi(parts[1]) + if err != nil { + return -1 + } + + return code +} + type response struct { Href string `xml:"DAV: href"` Propstats []propstat `xml:"DAV: propstat"` } -func getPropstat(r *response, status string) *propstat { +func getPropstat(r *response, statuses []string) *propstat { for _, prop := range r.Propstats { - if strings.Contains(prop.Status, status) { - return &prop + for _, status := range statuses { + if strings.Contains(prop.Status, status) { + return &prop + } } } return nil } // ReadDir reads the contents of a remote directory -func (c *Client) ReadDir(path string) ([]os.FileInfo, error) { +func (c *Client) ReadDir(path string) ([]FileInfo, error) { return c.ReadDirWithProps(path, defaultProps) } // ReadDirWithProps reads the contents of the directory at the given path, along with the specified properties. -func (c *Client) ReadDirWithProps(path string, props []string) ([]os.FileInfo, error) { +func (c *Client) ReadDirWithProps(path string, props []string) ([]FileInfo, error) { propfindprops := "" if len(props) > 0 { propfindprops = `` } - files := make([]os.FileInfo, 0) + files := make([]FileInfo, 0) skipSelf := true parse := func(resp interface{}) error { r := resp.(*response) if skipSelf { skipSelf = false - if p := getPropstat(r, "200"); p != nil && p.Type() == "collection" { + if p := getPropstat(r, []string{"200", "425"}); p != nil && p.Type() == "collection" { r.Propstats = nil return nil } return NewPathError("ReadDir", path, 405) } - if p := getPropstat(r, "200"); p != nil { + if p := getPropstat(r, []string{"200", "425"}); p != nil { var name string if ps, err := url.PathUnescape(r.Href); err == nil { name = pathpkg.Base(ps) @@ -227,17 +243,19 @@ func (c *Client) ReadDirWithProps(path string, props []string) ([]os.FileInfo, e } // Stat returns the file stats for a specified path with the default properties -func (c *Client) Stat(path string) (os.FileInfo, error) { +func (c *Client) Stat(path string) (FileInfo, error) { return c.StatWithProps(path, defaultProps) } // StatWithProps returns the FileInfo for the specified path along with the specified properties. -func (c *Client) StatWithProps(path string, props []string) (os.FileInfo, error) { +func (c *Client) StatWithProps(path string, props []string) (FileInfo, error) { var f *File parse := func(resp interface{}) error { r := resp.(*response) - if p := getPropstat(r, "200"); p != nil && f == nil { + if p := getPropstat(r, []string{"200", "425"}); p != nil && f == nil { f = newFile(".", path, p) + } else { + return NewPathError("StatWithProps", path, 404) } r.Propstats = nil @@ -257,10 +275,14 @@ func (c *Client) StatWithProps(path string, props []string) (os.FileInfo, error) if err != nil { if _, ok := err.(*os.PathError); !ok { - return nil, NewPathErrorErr("ReadDir", path, err) + return nil, NewPathErrorErr("StatWithProps", path, err) } return nil, err } + + if f == nil { + return nil, NewPathError("StatWithProps", path, 404) + } return *f, err } diff --git a/vendor/github.com/studio-b12/gowebdav/file.go b/vendor/github.com/studio-b12/gowebdav/file.go index 8c5a09189..e66d1a4ec 100644 --- a/vendor/github.com/studio-b12/gowebdav/file.go +++ b/vendor/github.com/studio-b12/gowebdav/file.go @@ -7,6 +7,11 @@ import ( "time" ) +type FileInfo interface { + os.FileInfo + StatusCode() int +} + // File is our structure for a given file type File struct { path string @@ -16,12 +21,13 @@ type File struct { modified time.Time etag string isdir bool - props Props + propstat propstat + status int } func newFile(path, name string, p *propstat) *File { f := &File{ - props: p.Props, + propstat: *p, } path = FixSlashes(path) @@ -30,7 +36,6 @@ func newFile(path, name string, p *propstat) *File { f.modified = p.Modified() f.etag = p.ETag() f.contentType = p.ContentType() - f.props = p.Props if p.Type() == "collection" { f.path = filepath.Clean(f.path + "/") @@ -90,7 +95,11 @@ func (f File) IsDir() bool { // Sys ???? func (f File) Sys() interface{} { - return f.props + return f.propstat.Props +} + +func (f File) StatusCode() int { + return f.propstat.StatusCode() } // String lets us see file information diff --git a/vendor/modules.txt b/vendor/modules.txt index e4089ff68..d11233043 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d +# github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38 ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime @@ -1721,7 +1721,7 @@ github.com/stretchr/objx github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 +# github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f ## explicit; go 1.17 github.com/studio-b12/gowebdav # github.com/tchap/go-patricia/v2 v2.3.1 @@ -2298,4 +2298,4 @@ stash.kopano.io/kgol/oidc-go ## explicit; go 1.13 stash.kopano.io/kgol/rndm # github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348 -# github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 +# github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f