Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rhttp
import (
"context"
"crypto/tls"
"io"
"net/http"
"go.opencensus.io/plugin/ochttp"
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/pkg/errors"
)
// GetHTTPClient returns an http client with open census tracing support.
// TODO(labkode): harden it.
// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
func GetHTTPClient(opts ...Option) *http.Client {
options := newOptions(opts...)
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.DisableKeepAlives = options.DisableKeepAlive
tr.TLSClientConfig = &tls.Config{
InsecureSkipVerify: options.Insecure,
}
httpClient := &http.Client{
Timeout: options.Timeout,
Transport: &ochttp.Transport{
Base: tr,
},
}
return httpClient
}
// NewRequest creates an HTTP request that sets the token if it is passed in ctx.
func NewRequest(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) {
httpReq, err := http.NewRequest(method, url, body)
if err != nil {
return nil, errors.Wrap(err, "utils: error creating request")
}
// TODO(labkode): make header / auth configurable
tkn, ok := ctxpkg.ContextGetToken(ctx)
if ok {
httpReq.Header.Set(ctxpkg.TokenHeader, tkn)
}
httpReq = httpReq.WithContext(ctx)
return httpReq, nil
}
@@ -0,0 +1,54 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package datatx provides a library to abstract the complexity
// of using various data transfer protocols.
package datatx
import (
"context"
"net/http"
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
// DataTX provides an abstraction around various data transfer protocols.
type DataTX interface {
Handler(fs storage.FS) (http.Handler, error)
}
// EmitFileUploadedEvent is a helper function which publishes a FileUploaded event
func EmitFileUploadedEvent(spaceOwnerOrManager, executant *userv1beta1.UserId, ref *provider.Reference, publisher events.Publisher) error {
if ref == nil || publisher == nil {
return nil
}
uploadedEv := events.FileUploaded{
SpaceOwner: spaceOwnerOrManager,
Owner: spaceOwnerOrManager,
Executant: executant,
Ref: ref,
Timestamp: utils.TSNow(),
}
return events.Publish(context.Background(), publisher, uploadedEv)
}
@@ -0,0 +1,27 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package loader
import (
// Load core data transfer protocols
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple"
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces"
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus"
// Add your own here
)
@@ -0,0 +1,39 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package registry
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
)
// NewFunc is the function that data transfer implementations
// should register at init time.
type NewFunc func(map[string]interface{}, events.Publisher, *zerolog.Logger) (datatx.DataTX, error)
// NewFuncs is a map containing all the registered data transfers.
var NewFuncs = map[string]NewFunc{}
// Register registers a new data transfer new function.
// Not safe for concurrent use. Safe for use from package init.
func Register(name string, f NewFunc) {
NewFuncs[name] = f
}
@@ -0,0 +1,164 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package simple
import (
"net/http"
"time"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/mitchellh/mapstructure"
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
func init() {
registry.Register("simple", New)
}
type manager struct {
conf *cache.Config
publisher events.Publisher
log *zerolog.Logger
}
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
c := &cache.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
l := log.With().Str("datatx", "simple").Logger()
return &manager{
conf: c,
publisher: publisher,
log: &l,
}, nil
}
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sublog := m.log.With().Str("path", r.URL.Path).Logger()
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
ctx := r.Context()
switch r.Method {
case "GET", "HEAD":
if r.Method == "GET" {
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
}
download.GetOrHeadFile(w, r, fs, "")
case "PUT":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
if r.ContentLength == 0 {
sublog.Info().Msg("received invalid 0-byte PUT request")
w.WriteHeader(http.StatusBadRequest)
return
}
fn := r.URL.Path
defer r.Body.Close()
ref := &provider.Reference{Path: fn}
if lockID := r.Header.Get("X-Lock-Id"); lockID != "" {
ctx = ctxpkg.ContextSetLockID(ctx, lockID)
}
info, err := fs.Upload(ctx, storage.UploadRequest{
Ref: ref,
Body: r.Body,
Length: r.ContentLength,
}, func(spaceOwner, owner *userpb.UserId, ref *provider.Reference) {
if err := datatx.EmitFileUploadedEvent(spaceOwner, owner, ref, m.publisher); err != nil {
sublog.Error().Err(err).Msg("failed to publish FileUploaded event")
}
})
switch v := err.(type) {
case nil:
// set etag, mtime and file id
w.Header().Set(net.HeaderETag, info.Etag)
w.Header().Set(net.HeaderOCETag, info.Etag)
if info.Id != nil {
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(info.Id))
}
if info.Mtime != nil {
t := utils.TSToTime(info.Mtime).UTC()
lastModifiedString := t.Format(time.RFC1123Z)
w.Header().Set(net.HeaderLastModified, lastModifiedString)
}
w.WriteHeader(http.StatusOK)
case errtypes.PartialContent:
w.WriteHeader(http.StatusPartialContent)
case errtypes.ChecksumMismatch:
w.WriteHeader(errtypes.StatusChecksumMismatch)
case errtypes.NotFound:
w.WriteHeader(http.StatusNotFound)
case errtypes.PermissionDenied:
w.WriteHeader(http.StatusForbidden)
case errtypes.InvalidCredentials:
w.WriteHeader(http.StatusUnauthorized)
case errtypes.InsufficientStorage:
w.WriteHeader(http.StatusInsufficientStorage)
case errtypes.PreconditionFailed, errtypes.Aborted, errtypes.AlreadyExists:
w.WriteHeader(http.StatusPreconditionFailed)
default:
sublog.Error().Err(v).Msg("error uploading file")
w.WriteHeader(http.StatusInternalServerError)
}
return
default:
w.WriteHeader(http.StatusNotImplemented)
}
})
return h, nil
}
@@ -0,0 +1,167 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package spaces
import (
"net/http"
"path"
"strings"
"time"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
func init() {
registry.Register("spaces", New)
}
type manager struct {
conf *cache.Config
publisher events.Publisher
log *zerolog.Logger
}
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
c := &cache.Config{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
l := log.With().Str("datatx", "spaces").Logger()
return &manager{
conf: c,
publisher: publisher,
log: &l,
}, nil
}
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var spaceID string
spaceID, r.URL.Path = router.ShiftPath(r.URL.Path)
sublog := m.log.With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger()
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
ctx := r.Context()
switch r.Method {
case "GET", "HEAD":
if r.Method == "GET" {
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
}
download.GetOrHeadFile(w, r, fs, spaceID)
case "PUT":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// make a clean relative path
fn := path.Clean(strings.TrimLeft(r.URL.Path, "/"))
defer r.Body.Close()
rid, err := storagespace.ParseID(spaceID)
if err != nil {
sublog.Error().Err(err).Msg("failed to parse resourceID")
}
ref := &provider.Reference{
ResourceId: &rid,
Path: fn,
}
var info *provider.ResourceInfo
info, err = fs.Upload(ctx, storage.UploadRequest{
Ref: ref,
Body: r.Body,
Length: r.ContentLength,
}, func(spaceOwner, owner *userpb.UserId, ref *provider.Reference) {
if err := datatx.EmitFileUploadedEvent(spaceOwner, owner, ref, m.publisher); err != nil {
sublog.Error().Err(err).Msg("failed to publish FileUploaded event")
}
})
switch v := err.(type) {
case nil:
// set etag, mtime and file id
w.Header().Set(net.HeaderETag, info.Etag)
w.Header().Set(net.HeaderOCETag, info.Etag)
if info.Id != nil {
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(info.Id))
}
if info.Mtime != nil {
t := utils.TSToTime(info.Mtime).UTC()
lastModifiedString := t.Format(time.RFC1123Z)
w.Header().Set(net.HeaderLastModified, lastModifiedString)
}
w.WriteHeader(http.StatusOK)
case errtypes.PartialContent:
w.WriteHeader(http.StatusPartialContent)
case errtypes.ChecksumMismatch:
w.WriteHeader(errtypes.StatusChecksumMismatch)
case errtypes.NotFound:
w.WriteHeader(http.StatusNotFound)
case errtypes.PermissionDenied:
w.WriteHeader(http.StatusForbidden)
case errtypes.InvalidCredentials:
w.WriteHeader(http.StatusUnauthorized)
case errtypes.InsufficientStorage:
w.WriteHeader(http.StatusInsufficientStorage)
case errtypes.PreconditionFailed, errtypes.Aborted, errtypes.AlreadyExists:
w.WriteHeader(http.StatusPreconditionFailed)
default:
sublog.Error().Err(v).Msg("error uploading file")
w.WriteHeader(http.StatusInternalServerError)
}
return
default:
w.WriteHeader(http.StatusNotImplemented)
}
})
return h, nil
}
@@ -0,0 +1,312 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package tus
import (
"context"
"fmt"
"net/http"
"path"
"regexp"
"runtime"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"github.com/rs/zerolog"
tusd "github.com/tus/tusd/v2/pkg/handler"
"golang.org/x/exp/slog"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
)
func init() {
registry.Register("tus", New)
}
type TusConfig struct {
CorsEnabled bool `mapstructure:"cors_enabled"`
CorsAllowOrigin string `mapstructure:"cors_allow_origin"`
CorsAllowCredentials bool `mapstructure:"cors_allow_credentials"`
CorsAllowMethods string `mapstructure:"cors_allow_methods"`
CorsAllowHeaders string `mapstructure:"cors_allow_headers"`
CorsMaxAge string `mapstructure:"cors_max_age"`
CorsExposeHeaders string `mapstructure:"cors_expose_headers"`
}
type manager struct {
conf *TusConfig
publisher events.Publisher
log *zerolog.Logger
}
func parseConfig(m map[string]interface{}) (*TusConfig, error) {
c := &TusConfig{}
if err := mapstructure.Decode(m, c); err != nil {
err = errors.Wrap(err, "error decoding conf")
return nil, err
}
return c, nil
}
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
c, err := parseConfig(m)
if err != nil {
return nil, err
}
l := log.With().Str("datatx", "tus").Logger()
return &manager{
conf: c,
publisher: publisher,
log: &l,
}, nil
}
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
composable, ok := fs.(storage.ComposableFS)
if !ok {
return nil, errtypes.NotSupported("file system does not support the tus protocol")
}
// A storage backend for tusd may consist of multiple different parts which
// handle upload creation, locking, termination and so on. The composer is a
// place where all those separated pieces are joined together. In this example
// we only use the file store but you may plug in multiple.
composer := tusd.NewStoreComposer()
// let the composable storage tell tus which extensions it supports
composable.UseIn(composer)
config := tusd.Config{
StoreComposer: composer,
NotifyCompleteUploads: true,
Logger: slog.New(tusdLogger{log: m.log}),
}
if m.conf.CorsEnabled {
allowOrigin, err := regexp.Compile(m.conf.CorsAllowOrigin)
if m.conf.CorsAllowOrigin != "" && err != nil {
return nil, err
}
config.Cors = &tusd.CorsConfig{
Disable: false,
AllowOrigin: allowOrigin,
AllowCredentials: m.conf.CorsAllowCredentials,
AllowMethods: m.conf.CorsAllowMethods,
AllowHeaders: m.conf.CorsAllowHeaders,
MaxAge: m.conf.CorsMaxAge,
ExposeHeaders: m.conf.CorsExposeHeaders,
}
}
handler, err := tusd.NewUnroutedHandler(config)
if err != nil {
return nil, err
}
if usl, ok := fs.(storage.UploadSessionLister); ok {
// We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info
go func() {
for {
ev := <-handler.CompleteUploads
// We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files
// so we create a Progress instance here that is used to read the correct properties
ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID})
if err != nil {
appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session")
} else {
if len(ups) < 1 {
appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found")
continue
}
up := ups[0]
executant := up.Executant()
ref := up.Reference()
if m.publisher != nil {
if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil {
appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event")
}
}
}
}
}()
}
h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sublog := m.log.With().Str("uploadid", r.URL.Path).Logger()
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
method := r.Method
// https://github.com/tus/tus-resumable-upload-protocol/blob/master/protocol.md#x-http-method-override
if r.Header.Get("X-HTTP-Method-Override") != "" {
method = r.Header.Get("X-HTTP-Method-Override")
}
switch method {
case "POST":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// set etag, mtime and file id
setHeaders(fs, w, r)
handler.PostFile(w, r)
case "HEAD":
handler.HeadFile(w, r)
case "PATCH":
metrics.UploadsActive.Add(1)
defer func() {
metrics.UploadsActive.Sub(1)
}()
// set etag, mtime and file id
setHeaders(fs, w, r)
handler.PatchFile(w, r)
case "DELETE":
handler.DelFile(w, r)
case "GET":
metrics.DownloadsActive.Add(1)
defer func() {
metrics.DownloadsActive.Sub(1)
}()
// NOTE: this is breaking change - allthought it does not seem to be used
// We can make a switch here depending on some header value if that is needed
// download.GetOrHeadFile(w, r, fs, "")
handler.GetFile(w, r)
default:
w.WriteHeader(http.StatusNotImplemented)
}
}))
return h, nil
}
func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
id := path.Base(r.URL.Path)
datastore, ok := fs.(tusd.DataStore)
if !ok {
appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore")
return
}
upload, err := datastore.GetUpload(ctx, id)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage")
return
}
info, err := upload.GetInfo(ctx)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload info for upload")
return
}
expires := info.MetaData["expires"]
if expires != "" {
w.Header().Set(net.HeaderTusUploadExpires, expires)
}
resourceid := &provider.ResourceId{
StorageId: info.MetaData["providerID"],
SpaceId: info.Storage["SpaceRoot"],
OpaqueId: info.Storage["NodeId"],
}
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(resourceid))
}
// tusdLogger is a logger implementation (slog) for tusd that uses zerolog.
type tusdLogger struct {
log *zerolog.Logger
}
// Handle handles the record
func (l tusdLogger) Handle(_ context.Context, r slog.Record) error {
var logev *zerolog.Event
switch r.Level {
case slog.LevelDebug:
logev = l.log.Debug()
case slog.LevelInfo:
logev = l.log.Info()
case slog.LevelWarn:
logev = l.log.Warn()
case slog.LevelError:
logev = l.log.Error()
}
// Extract caller information from slog.Record only for debug and info levels
if (r.Level == slog.LevelDebug || r.Level == slog.LevelInfo) && r.PC != 0 {
frames := runtime.CallersFrames([]uintptr{r.PC})
frame, _ := frames.Next()
// add line using zerolog's caller format
logev = logev.Str("line", fmt.Sprintf("%s:%d", frame.File, frame.Line))
}
r.Attrs(func(a slog.Attr) bool {
// Resolve the Attr's value before doing anything else.
a.Value = a.Value.Resolve()
// Ignore empty Attrs.
if a.Equal(slog.Attr{}) {
return true
}
switch a.Value.Kind() {
case slog.KindBool:
logev = logev.Bool(a.Key, a.Value.Bool())
case slog.KindInt64:
logev = logev.Int64(a.Key, a.Value.Int64())
default:
logev = logev.Str(a.Key, a.Value.String())
}
return true
})
logev.Msg(r.Message)
return nil
}
// Enabled returns true
func (l tusdLogger) Enabled(_ context.Context, _ slog.Level) bool { return true }
// WithAttrs creates a new logger with the given attributes
func (l tusdLogger) WithAttrs(attr []slog.Attr) slog.Handler {
fields := make(map[string]interface{}, len(attr))
for _, a := range attr {
switch a.Value.Kind() {
case slog.KindBool:
fields[a.Key] = a.Value.Bool()
case slog.KindInt64:
fields[a.Key] = a.Value.Int64()
case slog.KindUint64:
fields[a.Key] = a.Value.Uint64()
case slog.KindFloat64:
fields[a.Key] = a.Value.Float64()
default:
fields[a.Key] = a.Value.String()
}
}
c := l.log.With().Fields(fields).Logger()
sLog := tusdLogger{log: &c}
return sLog
}
// WithGroup is not implemented
func (l tusdLogger) WithGroup(name string) slog.Handler { return l }
@@ -0,0 +1,60 @@
// Package metrics provides prometheus metrics for the data managers..
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
// DownloadsActive is the number of active downloads
DownloadsActive = promauto.NewGauge(prometheus.GaugeOpts{
Name: "reva_download_active",
Help: "Number of active downloads",
})
// UploadsActive is the number of active uploads
UploadsActive = promauto.NewGauge(prometheus.GaugeOpts{
Name: "reva_upload_active",
Help: "Number of active uploads",
})
// UploadProcessing is the number of uploads in processing
UploadProcessing = promauto.NewGauge(prometheus.GaugeOpts{
Name: "reva_upload_processing",
Help: "Number of uploads in processing",
})
// UploadSessionsInitiated is the number of upload sessions that have been initiated
UploadSessionsInitiated = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_initiated",
Help: "Number of uploads sessions that were initiated",
})
// UploadSessionsBytesReceived is the number of upload sessions that have received all bytes
UploadSessionsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_bytes_received",
Help: "Number of uploads sessions that have received all bytes",
})
// UploadSessionsFinalized is the number of upload sessions that have received all bytes
UploadSessionsFinalized = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_finalized",
Help: "Number of uploads sessions that have successfully completed",
})
// UploadSessionsAborted is the number of upload sessions that have been aborted
UploadSessionsAborted = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_aborted",
Help: "Number of uploads sessions that have aborted by postprocessing",
})
// UploadSessionsDeleted is the number of upload sessions that have been deleted
UploadSessionsDeleted = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_deleted",
Help: "Number of uploads sessions that have been deleted by postprocessing",
})
// UploadSessionsRestarted is the number of upload sessions that have been restarted
UploadSessionsRestarted = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_restarted",
Help: "Number of uploads sessions that have been restarted by postprocessing",
})
// UploadSessionsScanned is the number of upload sessions that have been scanned by antivirus
UploadSessionsScanned = promauto.NewCounter(prometheus.CounterOpts{
Name: "reva_upload_sessions_scanned",
Help: "Number of uploads sessions that have been scanned by antivirus",
})
)
@@ -0,0 +1,285 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
// Package download provides a library to handle file download requests.
package download
import (
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"path"
"strconv"
"strings"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider"
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/storage"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
type contextKey struct{}
var etagKey = contextKey{}
// ContextWithEtag returns a new `context.Context` that holds an etag.
func ContextWithEtag(ctx context.Context, etag string) context.Context {
return context.WithValue(ctx, etagKey, etag)
}
// EtagFromContext returns the etag previously associated with `ctx`, or
// `""` if no such etag could be found.
func EtagFromContext(ctx context.Context) string {
val := ctx.Value(etagKey)
if etag, ok := val.(string); ok {
return etag
}
return ""
}
// GetOrHeadFile returns the requested file content
func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceID string) {
ctx := r.Context()
sublog := appctx.GetLogger(ctx).With().Str("svc", "datatx").Str("handler", "download").Logger()
var fn string
files, ok := r.URL.Query()["filename"]
if !ok || len(files[0]) < 1 {
fn = r.URL.Path
} else {
fn = files[0]
}
var ref *provider.Reference
if spaceID == "" {
// ensure the absolute path starts with '/'
ref = &provider.Reference{Path: path.Join("/", fn)}
} else {
// build a storage space reference
rid, err := storagespace.ParseID(spaceID)
if err != nil {
handleError(w, &sublog, err, "parse ID")
}
ref = &provider.Reference{
ResourceId: &rid,
// ensure the relative path starts with '.'
Path: utils.MakeRelativePath(fn),
}
}
// TODO check preconditions like If-Range, If-Match ...
var md *provider.ResourceInfo
var content io.ReadCloser
var err error
var notModified bool
// do a stat to set Content-Length and etag headers
md, content, err = fs.Download(ctx, ref, func(md *provider.ResourceInfo) bool {
// range requests always need to open the reader to check if it is seekable
if r.Header.Get("Range") != "" {
return true
}
// otherwise, HEAD requests do not need to open a reader
if r.Method == "HEAD" {
return false
}
// check etag, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match
for _, etag := range r.Header.Values(net.HeaderIfNoneMatch) {
if md.Etag == etag {
// When the condition fails for GET and HEAD methods, then the server must return
// HTTP status code 304 (Not Modified). [...] Note that the server generating a
// 304 response MUST generate any of the following header fields that would have
// been sent in a 200 (OK) response to the same request:
// Cache-Control, Content-Location, Date, ETag, Expires, and Vary.
notModified = true
return false
}
}
return true
})
if err != nil {
handleError(w, &sublog, err, "download")
return
}
if content != nil {
defer content.Close()
}
if notModified {
w.Header().Set(net.HeaderETag, md.Etag)
w.WriteHeader(http.StatusNotModified)
return
}
// fill in storage provider id if it is missing
if spaceID != "" && md.GetId().GetStorageId() == "" {
md.Id.StorageId = ref.ResourceId.StorageId
}
var ranges []HTTPRange
if r.Header.Get("Range") != "" {
ranges, err = ParseRange(r.Header.Get("Range"), int64(md.Size))
if err != nil {
if err == ErrNoOverlap {
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", md.Size))
}
sublog.Error().Err(err).Interface("md", md).Interface("ranges", ranges).Msg("range request not satisfiable")
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
if SumRangesSize(ranges) > int64(md.Size) {
// The total number of bytes in all the ranges
// is larger than the size of the file by
// itself, so this is probably an attack, or a
// dumb client. Ignore the range request.
ranges = nil
}
}
code := http.StatusOK
sendSize := int64(md.Size)
var sendContent io.Reader = content
var s io.Seeker
if s, ok = content.(io.Seeker); ok {
// tell clients they can send range requests
w.Header().Set("Accept-Ranges", "bytes")
}
w.Header().Set(net.HeaderContentType, strings.Join([]string{md.MimeType, "charset=UTF-8"}, "; "))
if len(ranges) > 0 {
sublog.Debug().Int64("start", ranges[0].Start).Int64("length", ranges[0].Length).Msg("range request")
if s == nil {
sublog.Error().Int64("start", ranges[0].Start).Int64("length", ranges[0].Length).Msg("ReadCloser is not seekable")
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
code = http.StatusPartialContent
switch {
case len(ranges) == 1:
// RFC 7233, Section 4.1:
// "If a single part is being transferred, the server
// generating the 206 response MUST generate a
// Content-Range header field, describing what range
// of the selected representation is enclosed, and a
// payload consisting of the range.
// ...
// A server MUST NOT generate a multipart response to
// a request for a single range, since a client that
// does not request multiple parts might not support
// multipart responses."
ra := ranges[0]
if _, err := s.Seek(ra.Start, io.SeekStart); err != nil {
sublog.Error().Err(err).Int64("start", ra.Start).Int64("length", ra.Length).Msg("content is not seekable")
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
sendSize = ra.Length
w.Header().Set("Content-Range", ra.ContentRange(int64(md.Size)))
case len(ranges) > 1:
sendSize = RangesMIMESize(ranges, md.MimeType, int64(md.Size))
pr, pw := io.Pipe()
mw := multipart.NewWriter(pw)
w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
sendContent = pr
defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
go func() {
for _, ra := range ranges {
part, err := mw.CreatePart(ra.MimeHeader(md.MimeType+"; charset=UTF-8", int64(md.Size)))
if err != nil {
_ = pw.CloseWithError(err) // CloseWithError always returns nil
return
}
if _, err := s.Seek(ra.Start, io.SeekStart); err != nil {
_ = pw.CloseWithError(err) // CloseWithError always returns nil
return
}
if _, err := io.CopyN(part, content, ra.Length); err != nil {
_ = pw.CloseWithError(err) // CloseWithError always returns nil
return
}
}
mw.Close()
pw.Close()
}()
}
}
if w.Header().Get(net.HeaderContentEncoding) == "" {
w.Header().Set(net.HeaderContentLength, strconv.FormatInt(sendSize, 10))
}
w.Header().Set(net.HeaderContentDisposistion, net.ContentDispositionAttachment(path.Base(md.Path)))
w.Header().Set(net.HeaderETag, md.Etag)
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(md.Id))
w.Header().Set(net.HeaderOCETag, md.Etag)
w.Header().Set(net.HeaderLastModified, net.RFC1123Z(md.Mtime))
if md.Checksum != nil {
w.Header().Set(net.HeaderOCChecksum, fmt.Sprintf("%s:%s", strings.ToUpper(string(storageprovider.GRPC2PKGXS(md.Checksum.Type))), md.Checksum.Sum))
}
w.WriteHeader(code)
if r.Method != "HEAD" {
var c int64
c, err = io.CopyN(w, sendContent, sendSize)
if err != nil {
sublog.Error().Err(err).Interface("resourceid", md.Id).Msg("error copying data to response")
return
}
if c != sendSize {
sublog.Error().Int64("copied", c).Int64("size", sendSize).Msg("copied vs size mismatch")
}
}
}
func handleError(w http.ResponseWriter, log *zerolog.Logger, err error, action string) {
switch err.(type) {
case errtypes.IsNotFound:
log.Debug().Err(err).Str("action", action).Msg("file not found")
w.WriteHeader(http.StatusNotFound)
case errtypes.IsPermissionDenied:
log.Debug().Err(err).Str("action", action).Msg("permission denied")
w.WriteHeader(http.StatusForbidden)
case errtypes.Aborted:
log.Debug().Err(err).Str("action", action).Msg("etags do not match")
w.WriteHeader(http.StatusPreconditionFailed)
case errtypes.TooEarly:
log.Debug().Err(err).Str("action", action).Msg("file is still being processed")
w.WriteHeader(http.StatusTooEarly)
default:
log.Error().Err(err).Str("action", action).Msg("unexpected error")
w.WriteHeader(http.StatusInternalServerError)
}
}
@@ -0,0 +1,158 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package download
import (
"errors"
"fmt"
"mime/multipart"
"net/textproto"
"strconv"
"strings"
)
// taken from https://golang.org/src/net/http/fs.go
// ErrInvalidRange is returned by serveContent's parseRange if the Range is
// malformed or invalid.
var ErrInvalidRange = errors.New("invalid range")
// ErrNoOverlap is returned by serveContent's parseRange if first-byte-pos of
// all of the byte-range-spec values is greater than the content size.
var ErrNoOverlap = fmt.Errorf("%w: failed to overlap", ErrInvalidRange)
// HTTPRange specifies the byte range to be sent to the client.
type HTTPRange struct {
Start, Length int64
}
// ContentRange formats a Range header string as per RFC 7233.
func (r HTTPRange) ContentRange(size int64) string {
return fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Length-1, size)
}
// MimeHeader creates range relevant MimeHeaders
func (r HTTPRange) MimeHeader(contentType string, size int64) textproto.MIMEHeader {
return textproto.MIMEHeader{
"Content-Range": {r.ContentRange(size)},
"Content-Type": {contentType},
}
}
// ParseRange parses a Range header string as per RFC 7233.
// errNoOverlap is returned if none of the ranges overlap.
func ParseRange(s string, size int64) ([]HTTPRange, error) {
if s == "" {
return nil, nil // header not present
}
const b = "bytes="
if !strings.HasPrefix(s, b) {
return nil, ErrInvalidRange
}
ranges := []HTTPRange{}
noOverlap := false
for _, ra := range strings.Split(s[len(b):], ",") {
ra = textproto.TrimString(ra)
if ra == "" {
continue
}
i := strings.Index(ra, "-")
if i < 0 {
return nil, ErrInvalidRange
}
start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:])
var r HTTPRange
if start == "" {
// If no start is specified, end specifies the
// range start relative to the end of the file.
i, err := strconv.ParseInt(end, 10, 64)
if err != nil {
return nil, ErrInvalidRange
}
if i > size {
i = size
}
r.Start = size - i
r.Length = size - r.Start
} else {
i, err := strconv.ParseInt(start, 10, 64)
if err != nil || i < 0 {
return nil, ErrInvalidRange
}
if i >= size {
// If the range begins after the size of the content,
// then it does not overlap.
noOverlap = true
continue
}
r.Start = i
if end == "" {
// If no end is specified, range extends to end of the file.
r.Length = size - r.Start
} else {
i, err := strconv.ParseInt(end, 10, 64)
if err != nil || r.Start > i {
return nil, ErrInvalidRange
}
if i >= size {
i = size - 1
}
r.Length = i - r.Start + 1
}
}
ranges = append(ranges, r)
}
if noOverlap && len(ranges) == 0 {
// The specified ranges did not overlap with the content.
return nil, ErrNoOverlap
}
return ranges, nil
}
// countingWriter counts how many bytes have been written to it.
type countingWriter int64
func (w *countingWriter) Write(p []byte) (n int, err error) {
*w += countingWriter(len(p))
return len(p), nil
}
// RangesMIMESize returns the number of bytes it takes to encode the
// provided ranges as a multipart response.
func RangesMIMESize(ranges []HTTPRange, contentType string, contentSize int64) (encSize int64) {
var w countingWriter
mw := multipart.NewWriter(&w)
for _, ra := range ranges {
// CreatePart might return an error if the io.Copy for the boundaries fails
// here parts are not filled, so we assume for now thet this will always succeed
_, _ = mw.CreatePart(ra.MimeHeader(contentType, contentSize))
encSize += ra.Length
}
_ = mw.Close()
encSize += int64(w)
return
}
// SumRangesSize adds up the length of all ranges
func SumRangesSize(ranges []HTTPRange) (size int64) {
for _, ra := range ranges {
size += ra.Length
}
return
}
@@ -0,0 +1,61 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package global
import (
"net/http"
"github.com/rs/zerolog"
)
// NewMiddlewares contains all the registered new middleware functions.
var NewMiddlewares = map[string]NewMiddleware{}
// NewMiddleware is the function that HTTP middlewares need to register at init time.
type NewMiddleware func(conf map[string]interface{}) (Middleware, int, error)
// RegisterMiddleware registers a new HTTP middleware and its new function.
func RegisterMiddleware(name string, n NewMiddleware) {
NewMiddlewares[name] = n
}
// Middleware is a middleware http handler.
type Middleware func(h http.Handler) http.Handler
// Services is a map of service name and its new function.
var Services = map[string]NewService{}
// Register registers a new HTTP services with name and new function.
func Register(name string, newFunc NewService) {
Services[name] = newFunc
}
// NewService is the function that HTTP services need to register at init time.
type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error)
// Service represents a HTTP service.
type Service interface {
Handler() http.Handler
Prefix() string
Close() error
// List of url relative to the prefix to be unprotected by the authentication
// middleware. To be seen if we need url-verb fine grained skip checks like
// GET is public and POST is not.
Unprotected() []string
}
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rhttp
import (
"context"
"time"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Context context.Context
Timeout time.Duration
Insecure bool
DisableKeepAlive bool
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Insecure provides a function to set the insecure option.
func Insecure(insecure bool) Option {
return func(o *Options) {
o.Insecure = insecure
}
}
// Timeout provides a function to set the timeout option.
func Timeout(t time.Duration) Option {
return func(o *Options) {
o.Timeout = t
}
}
// DisableKeepAlive provides a function to set the disablee keep alive option.
func DisableKeepAlive(disable bool) Option {
return func(o *Options) {
o.DisableKeepAlive = disable
}
}
+328
View File
@@ -0,0 +1,328 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package rhttp
import (
"context"
"fmt"
"net"
"net/http"
"path"
"sort"
"time"
"github.com/mitchellh/mapstructure"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/appctx"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/log"
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/providerauthorizer"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/global"
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
rtrace "github.com/opencloud-eu/reva/v2/pkg/trace"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
// name is the Tracer name used to identify this instrumentation library.
const tracerName = "rhttp"
// New returns a new server
func New(m interface{}, l zerolog.Logger, tp trace.TracerProvider) (*Server, error) {
conf := &config{}
if err := mapstructure.Decode(m, conf); err != nil {
return nil, err
}
conf.init()
httpServer := &http.Server{}
s := &Server{
httpServer: httpServer,
conf: conf,
svcs: map[string]global.Service{},
unprotected: []string{},
handlers: map[string]http.Handler{},
log: l,
tracerProvider: tp,
}
return s, nil
}
// Server contains the server info.
type Server struct {
httpServer *http.Server
conf *config
listener net.Listener
svcs map[string]global.Service // map key is svc Prefix
unprotected []string
handlers map[string]http.Handler
middlewares []*middlewareTriple
log zerolog.Logger
tracerProvider trace.TracerProvider
}
type config struct {
Network string `mapstructure:"network"`
Address string `mapstructure:"address"`
Services map[string]map[string]interface{} `mapstructure:"services"`
Middlewares map[string]map[string]interface{} `mapstructure:"middlewares"`
CertFile string `mapstructure:"certfile"`
KeyFile string `mapstructure:"keyfile"`
}
func (c *config) init() {
// apply defaults
if c.Network == "" {
c.Network = "tcp"
}
if c.Address == "" {
c.Address = "0.0.0.0:19001"
}
}
// Start starts the server
func (s *Server) Start(ln net.Listener) error {
if err := s.registerServices(); err != nil {
return err
}
if err := s.registerMiddlewares(); err != nil {
return err
}
handler, err := s.getHandler()
if err != nil {
return errors.Wrap(err, "rhttp: error creating http handler")
}
s.httpServer.Handler = handler
s.listener = ln
if (s.conf.CertFile != "") && (s.conf.KeyFile != "") {
s.log.Info().Msgf("https server listening at https://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile)
err = s.httpServer.ServeTLS(s.listener, s.conf.CertFile, s.conf.KeyFile)
} else {
s.log.Info().Msgf("http server listening at http://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile)
err = s.httpServer.Serve(s.listener)
}
if err == nil || err == http.ErrServerClosed {
return nil
}
return err
}
// Stop stops the server.
func (s *Server) Stop() error {
// TODO(labkode): set ctx deadline to zero
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
defer s.closeServices()
return s.httpServer.Shutdown(ctx)
}
// TODO(labkode): we can't stop the server shutdown because a service cannot be shutdown.
// What do we do in case a service cannot be properly closed? Now we just log the error.
// TODO(labkode): the close should be given a deadline using context.Context.
func (s *Server) closeServices() {
for _, svc := range s.svcs {
if err := svc.Close(); err != nil {
s.log.Error().Err(err).Msgf("error closing service %q", svc.Prefix())
} else {
s.log.Info().Msgf("service %q correctly closed", svc.Prefix())
}
}
}
// Network return the network type.
func (s *Server) Network() string {
return s.conf.Network
}
// Address returns the network address.
func (s *Server) Address() string {
return s.conf.Address
}
// GracefulStop gracefully stops the server.
func (s *Server) GracefulStop() error {
defer s.closeServices()
return s.httpServer.Shutdown(context.Background())
}
// middlewareTriple represents a middleware with the
// priority to be chained.
type middlewareTriple struct {
Name string
Priority int
Middleware global.Middleware
}
func (s *Server) registerMiddlewares() error {
middlewares := []*middlewareTriple{}
for name, newFunc := range global.NewMiddlewares {
if s.isMiddlewareEnabled(name) {
m, prio, err := newFunc(s.conf.Middlewares[name])
if err != nil {
err = errors.Wrapf(err, "error creating new middleware: %s,", name)
return err
}
middlewares = append(middlewares, &middlewareTriple{
Name: name,
Priority: prio,
Middleware: m,
})
s.log.Info().Msgf("http middleware enabled: %s", name)
}
}
s.middlewares = middlewares
return nil
}
func (s *Server) isMiddlewareEnabled(name string) bool {
_, ok := s.conf.Middlewares[name]
return ok
}
func (s *Server) registerServices() error {
for svcName := range s.conf.Services {
if s.isServiceEnabled(svcName) {
newFunc := global.Services[svcName]
svc, err := newFunc(s.conf.Services[svcName], &s.log)
if err != nil {
err = errors.Wrapf(err, "http service %s could not be started,", svcName)
return err
}
// instrument services with opencensus tracing.
h := traceHandler(svcName, svc.Handler(), s.tracerProvider)
s.handlers[svc.Prefix()] = h
s.svcs[svc.Prefix()] = svc
s.unprotected = append(s.unprotected, getUnprotected(svc.Prefix(), svc.Unprotected())...)
s.log.Info().Msgf("http service enabled: %s@/%s", svcName, svc.Prefix())
} else {
message := fmt.Sprintf("http service %s does not exist", svcName)
return errors.New(message)
}
}
return nil
}
func (s *Server) isServiceEnabled(svcName string) bool {
_, ok := global.Services[svcName]
return ok
}
// TODO(labkode): if the http server is exposed under a basename we need to prepend
// to prefix.
func getUnprotected(prefix string, unprotected []string) []string {
for i := range unprotected {
unprotected[i] = path.Join("/", prefix, unprotected[i])
}
return unprotected
}
func (s *Server) getHandler() (http.Handler, error) {
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
head, tail := router.ShiftPath(r.URL.Path)
if h, ok := s.handlers[head]; ok {
r.URL.Path = tail
s.log.Debug().Msgf("http routing: head=%s tail=%s svc=%s", head, r.URL.Path, head)
h.ServeHTTP(w, r)
return
}
// when a service is exposed at the root.
if h, ok := s.handlers[""]; ok {
r.URL.Path = "/" + head + tail
s.log.Debug().Msgf("http routing: head= tail=%s svc=root", r.URL.Path)
h.ServeHTTP(w, r)
return
}
s.log.Debug().Msgf("http routing: head=%s tail=%s svc=not-found", head, tail)
w.WriteHeader(http.StatusNotFound)
})
// sort middlewares by priority.
sort.SliceStable(s.middlewares, func(i, j int) bool {
return s.middlewares[i].Priority > s.middlewares[j].Priority
})
handler := http.Handler(h)
for _, triple := range s.middlewares {
s.log.Info().Msgf("chaining http middleware %s with priority %d", triple.Name, triple.Priority)
handler = triple.Middleware(traceHandler(triple.Name, handler, s.tracerProvider))
}
for _, v := range s.unprotected {
s.log.Info().Msgf("unprotected URL: %s", v)
}
authMiddle, err := auth.New(s.conf.Middlewares["auth"], s.unprotected, s.tracerProvider)
if err != nil {
return nil, errors.Wrap(err, "rhttp: error creating auth middleware")
}
// add always the logctx middleware as most priority, this middleware is internal
// and cannot be configured from the configuration.
coreMiddlewares := []*middlewareTriple{}
providerAuthMiddle, err := addProviderAuthMiddleware(s.conf, s.unprotected)
if err != nil {
return nil, errors.Wrap(err, "rhttp: error creating providerauthorizer middleware")
}
if providerAuthMiddle != nil {
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: providerAuthMiddle, Name: "providerauthorizer"})
}
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: authMiddle, Name: "auth"})
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: log.New(), Name: "log"})
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: appctx.New(s.log, s.tracerProvider), Name: "appctx"})
for _, triple := range coreMiddlewares {
handler = triple.Middleware(traceHandler(triple.Name, handler, s.tracerProvider))
}
return handler, nil
}
func traceHandler(name string, h http.Handler, tp trace.TracerProvider) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := rtrace.Propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
t := tp.Tracer(tracerName)
ctx, span := t.Start(ctx, name)
defer span.End()
rtrace.Propagator.Inject(ctx, propagation.HeaderCarrier(r.Header))
h.ServeHTTP(w, r.WithContext(ctx))
})
}
func addProviderAuthMiddleware(conf *config, unprotected []string) (global.Middleware, error) {
_, ocmdRegistered := global.Services["ocmd"]
_, ocmdEnabled := conf.Services["ocmd"]
ocmdPrefix, _ := conf.Services["ocmd"]["prefix"].(string)
if ocmdRegistered && ocmdEnabled {
return providerauthorizer.New(conf.Middlewares["providerauthorizer"], unprotected, ocmdPrefix)
}
return nil, nil
}
@@ -0,0 +1,41 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package router
import (
"path"
"strings"
)
// ShiftPath splits off the first component of p, which will be cleaned of
// relative components before processing. head will never contain a slash and
// tail will always be a rooted path without trailing slash.
// see https://blog.merovius.de/2017/06/18/how-not-to-use-an-http-router.html
// and https://gist.github.com/weatherglass/62bd8a704d4dfdc608fe5c5cb5a6980c#gistcomment-2161690 for the zero alloc code below
func ShiftPath(p string) (head, tail string) {
if p == "" {
return "", "/"
}
p = strings.TrimPrefix(path.Clean(p), "/")
i := strings.Index(p, "/")
if i < 0 {
return p, "/"
}
return p[:i], p[i:]
}