Initial QSfera import
This commit is contained in:
+70
@@ -0,0 +1,70 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package rhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// GetHTTPClient returns an http client with open census tracing support.
|
||||
// TODO(labkode): harden it.
|
||||
// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
|
||||
func GetHTTPClient(opts ...Option) *http.Client {
|
||||
options := newOptions(opts...)
|
||||
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
tr.DisableKeepAlives = options.DisableKeepAlive
|
||||
tr.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: options.Insecure,
|
||||
}
|
||||
|
||||
httpClient := &http.Client{
|
||||
Timeout: options.Timeout,
|
||||
Transport: &ochttp.Transport{
|
||||
Base: tr,
|
||||
},
|
||||
}
|
||||
|
||||
return httpClient
|
||||
}
|
||||
|
||||
// NewRequest creates an HTTP request that sets the token if it is passed in ctx.
|
||||
func NewRequest(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) {
|
||||
httpReq, err := http.NewRequest(method, url, body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "utils: error creating request")
|
||||
}
|
||||
|
||||
// TODO(labkode): make header / auth configurable
|
||||
tkn, ok := ctxpkg.ContextGetToken(ctx)
|
||||
if ok {
|
||||
httpReq.Header.Set(ctxpkg.TokenHeader, tkn)
|
||||
}
|
||||
|
||||
httpReq = httpReq.WithContext(ctx)
|
||||
return httpReq, nil
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
// Package datatx provides a library to abstract the complexity
|
||||
// of using various data transfer protocols.
|
||||
package datatx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// DataTX provides an abstraction around various data transfer protocols.
|
||||
type DataTX interface {
|
||||
Handler(fs storage.FS) (http.Handler, error)
|
||||
}
|
||||
|
||||
// EmitFileUploadedEvent is a helper function which publishes a FileUploaded event
|
||||
func EmitFileUploadedEvent(spaceOwnerOrManager, executant *userv1beta1.UserId, ref *provider.Reference, publisher events.Publisher) error {
|
||||
if ref == nil || publisher == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
uploadedEv := events.FileUploaded{
|
||||
SpaceOwner: spaceOwnerOrManager,
|
||||
Owner: spaceOwnerOrManager,
|
||||
Executant: executant,
|
||||
Ref: ref,
|
||||
Timestamp: utils.TSNow(),
|
||||
}
|
||||
|
||||
return events.Publish(context.Background(), publisher, uploadedEv)
|
||||
}
|
||||
Generated
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package loader
|
||||
|
||||
import (
|
||||
// Load core data transfer protocols
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus"
|
||||
// Add your own here
|
||||
)
|
||||
Generated
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package registry
|
||||
|
||||
import (
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
|
||||
)
|
||||
|
||||
// NewFunc is the function that data transfer implementations
|
||||
// should register at init time.
|
||||
type NewFunc func(map[string]interface{}, events.Publisher, *zerolog.Logger) (datatx.DataTX, error)
|
||||
|
||||
// NewFuncs is a map containing all the registered data transfers.
|
||||
var NewFuncs = map[string]NewFunc{}
|
||||
|
||||
// Register registers a new data transfer new function.
|
||||
// Not safe for concurrent use. Safe for use from package init.
|
||||
func Register(name string, f NewFunc) {
|
||||
NewFuncs[name] = f
|
||||
}
|
||||
Generated
Vendored
+164
@@ -0,0 +1,164 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package simple
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registry.Register("simple", New)
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
conf *cache.Config
|
||||
publisher events.Publisher
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
|
||||
c := &cache.Config{}
|
||||
if err := mapstructure.Decode(m, c); err != nil {
|
||||
err = errors.Wrap(err, "error decoding conf")
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
|
||||
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
|
||||
c, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l := log.With().Str("datatx", "simple").Logger()
|
||||
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
log: &l,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sublog := m.log.With().Str("path", r.URL.Path).Logger()
|
||||
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
|
||||
ctx := r.Context()
|
||||
|
||||
switch r.Method {
|
||||
case "GET", "HEAD":
|
||||
if r.Method == "GET" {
|
||||
metrics.DownloadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.DownloadsActive.Sub(1)
|
||||
}()
|
||||
}
|
||||
download.GetOrHeadFile(w, r, fs, "")
|
||||
case "PUT":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
|
||||
if r.ContentLength == 0 {
|
||||
sublog.Info().Msg("received invalid 0-byte PUT request")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
fn := r.URL.Path
|
||||
defer r.Body.Close()
|
||||
|
||||
ref := &provider.Reference{Path: fn}
|
||||
|
||||
if lockID := r.Header.Get("X-Lock-Id"); lockID != "" {
|
||||
ctx = ctxpkg.ContextSetLockID(ctx, lockID)
|
||||
}
|
||||
|
||||
info, err := fs.Upload(ctx, storage.UploadRequest{
|
||||
Ref: ref,
|
||||
Body: r.Body,
|
||||
Length: r.ContentLength,
|
||||
}, func(spaceOwner, owner *userpb.UserId, ref *provider.Reference) {
|
||||
if err := datatx.EmitFileUploadedEvent(spaceOwner, owner, ref, m.publisher); err != nil {
|
||||
sublog.Error().Err(err).Msg("failed to publish FileUploaded event")
|
||||
}
|
||||
})
|
||||
switch v := err.(type) {
|
||||
case nil:
|
||||
// set etag, mtime and file id
|
||||
w.Header().Set(net.HeaderETag, info.Etag)
|
||||
w.Header().Set(net.HeaderOCETag, info.Etag)
|
||||
if info.Id != nil {
|
||||
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(info.Id))
|
||||
}
|
||||
if info.Mtime != nil {
|
||||
t := utils.TSToTime(info.Mtime).UTC()
|
||||
lastModifiedString := t.Format(time.RFC1123Z)
|
||||
w.Header().Set(net.HeaderLastModified, lastModifiedString)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case errtypes.PartialContent:
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
case errtypes.ChecksumMismatch:
|
||||
w.WriteHeader(errtypes.StatusChecksumMismatch)
|
||||
case errtypes.NotFound:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case errtypes.PermissionDenied:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
case errtypes.InvalidCredentials:
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
case errtypes.InsufficientStorage:
|
||||
w.WriteHeader(http.StatusInsufficientStorage)
|
||||
case errtypes.PreconditionFailed, errtypes.Aborted, errtypes.AlreadyExists:
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
default:
|
||||
sublog.Error().Err(v).Msg("error uploading file")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotImplemented)
|
||||
}
|
||||
})
|
||||
return h, nil
|
||||
}
|
||||
Generated
Vendored
+167
@@ -0,0 +1,167 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package spaces
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registry.Register("spaces", New)
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
conf *cache.Config
|
||||
publisher events.Publisher
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
func parseConfig(m map[string]interface{}) (*cache.Config, error) {
|
||||
c := &cache.Config{}
|
||||
if err := mapstructure.Decode(m, c); err != nil {
|
||||
err = errors.Wrap(err, "error decoding conf")
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
|
||||
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
|
||||
c, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l := log.With().Str("datatx", "spaces").Logger()
|
||||
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
log: &l,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var spaceID string
|
||||
spaceID, r.URL.Path = router.ShiftPath(r.URL.Path)
|
||||
|
||||
sublog := m.log.With().Str("spaceid", spaceID).Str("path", r.URL.Path).Logger()
|
||||
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
|
||||
ctx := r.Context()
|
||||
|
||||
switch r.Method {
|
||||
case "GET", "HEAD":
|
||||
if r.Method == "GET" {
|
||||
metrics.DownloadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.DownloadsActive.Sub(1)
|
||||
}()
|
||||
}
|
||||
download.GetOrHeadFile(w, r, fs, spaceID)
|
||||
case "PUT":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
|
||||
// make a clean relative path
|
||||
fn := path.Clean(strings.TrimLeft(r.URL.Path, "/"))
|
||||
defer r.Body.Close()
|
||||
|
||||
rid, err := storagespace.ParseID(spaceID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("failed to parse resourceID")
|
||||
}
|
||||
ref := &provider.Reference{
|
||||
ResourceId: &rid,
|
||||
Path: fn,
|
||||
}
|
||||
var info *provider.ResourceInfo
|
||||
info, err = fs.Upload(ctx, storage.UploadRequest{
|
||||
Ref: ref,
|
||||
Body: r.Body,
|
||||
Length: r.ContentLength,
|
||||
}, func(spaceOwner, owner *userpb.UserId, ref *provider.Reference) {
|
||||
if err := datatx.EmitFileUploadedEvent(spaceOwner, owner, ref, m.publisher); err != nil {
|
||||
sublog.Error().Err(err).Msg("failed to publish FileUploaded event")
|
||||
}
|
||||
})
|
||||
switch v := err.(type) {
|
||||
case nil:
|
||||
// set etag, mtime and file id
|
||||
w.Header().Set(net.HeaderETag, info.Etag)
|
||||
w.Header().Set(net.HeaderOCETag, info.Etag)
|
||||
if info.Id != nil {
|
||||
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(info.Id))
|
||||
}
|
||||
if info.Mtime != nil {
|
||||
t := utils.TSToTime(info.Mtime).UTC()
|
||||
lastModifiedString := t.Format(time.RFC1123Z)
|
||||
w.Header().Set(net.HeaderLastModified, lastModifiedString)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case errtypes.PartialContent:
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
case errtypes.ChecksumMismatch:
|
||||
w.WriteHeader(errtypes.StatusChecksumMismatch)
|
||||
case errtypes.NotFound:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case errtypes.PermissionDenied:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
case errtypes.InvalidCredentials:
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
case errtypes.InsufficientStorage:
|
||||
w.WriteHeader(http.StatusInsufficientStorage)
|
||||
case errtypes.PreconditionFailed, errtypes.Aborted, errtypes.AlreadyExists:
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
default:
|
||||
sublog.Error().Err(v).Msg("error uploading file")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotImplemented)
|
||||
}
|
||||
})
|
||||
return h, nil
|
||||
}
|
||||
Generated
Vendored
+312
@@ -0,0 +1,312 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package tus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
"golang.org/x/exp/slog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registry.Register("tus", New)
|
||||
}
|
||||
|
||||
type TusConfig struct {
|
||||
CorsEnabled bool `mapstructure:"cors_enabled"`
|
||||
CorsAllowOrigin string `mapstructure:"cors_allow_origin"`
|
||||
CorsAllowCredentials bool `mapstructure:"cors_allow_credentials"`
|
||||
CorsAllowMethods string `mapstructure:"cors_allow_methods"`
|
||||
CorsAllowHeaders string `mapstructure:"cors_allow_headers"`
|
||||
CorsMaxAge string `mapstructure:"cors_max_age"`
|
||||
CorsExposeHeaders string `mapstructure:"cors_expose_headers"`
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
conf *TusConfig
|
||||
publisher events.Publisher
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
func parseConfig(m map[string]interface{}) (*TusConfig, error) {
|
||||
c := &TusConfig{}
|
||||
if err := mapstructure.Decode(m, c); err != nil {
|
||||
err = errors.Wrap(err, "error decoding conf")
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// New returns a datatx manager implementation that relies on HTTP PUT/GET.
|
||||
func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logger) (datatx.DataTX, error) {
|
||||
c, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l := log.With().Str("datatx", "tus").Logger()
|
||||
|
||||
return &manager{
|
||||
conf: c,
|
||||
publisher: publisher,
|
||||
log: &l,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
composable, ok := fs.(storage.ComposableFS)
|
||||
if !ok {
|
||||
return nil, errtypes.NotSupported("file system does not support the tus protocol")
|
||||
}
|
||||
|
||||
// A storage backend for tusd may consist of multiple different parts which
|
||||
// handle upload creation, locking, termination and so on. The composer is a
|
||||
// place where all those separated pieces are joined together. In this example
|
||||
// we only use the file store but you may plug in multiple.
|
||||
composer := tusd.NewStoreComposer()
|
||||
|
||||
// let the composable storage tell tus which extensions it supports
|
||||
composable.UseIn(composer)
|
||||
|
||||
config := tusd.Config{
|
||||
StoreComposer: composer,
|
||||
NotifyCompleteUploads: true,
|
||||
Logger: slog.New(tusdLogger{log: m.log}),
|
||||
}
|
||||
|
||||
if m.conf.CorsEnabled {
|
||||
allowOrigin, err := regexp.Compile(m.conf.CorsAllowOrigin)
|
||||
if m.conf.CorsAllowOrigin != "" && err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.Cors = &tusd.CorsConfig{
|
||||
Disable: false,
|
||||
AllowOrigin: allowOrigin,
|
||||
AllowCredentials: m.conf.CorsAllowCredentials,
|
||||
AllowMethods: m.conf.CorsAllowMethods,
|
||||
AllowHeaders: m.conf.CorsAllowHeaders,
|
||||
MaxAge: m.conf.CorsMaxAge,
|
||||
ExposeHeaders: m.conf.CorsExposeHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
handler, err := tusd.NewUnroutedHandler(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if usl, ok := fs.(storage.UploadSessionLister); ok {
|
||||
// We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info
|
||||
go func() {
|
||||
for {
|
||||
ev := <-handler.CompleteUploads
|
||||
// We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files
|
||||
// so we create a Progress instance here that is used to read the correct properties
|
||||
ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID})
|
||||
if err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session")
|
||||
} else {
|
||||
if len(ups) < 1 {
|
||||
appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found")
|
||||
continue
|
||||
}
|
||||
up := ups[0]
|
||||
executant := up.Executant()
|
||||
ref := up.Reference()
|
||||
if m.publisher != nil {
|
||||
if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sublog := m.log.With().Str("uploadid", r.URL.Path).Logger()
|
||||
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
|
||||
method := r.Method
|
||||
// https://github.com/tus/tus-resumable-upload-protocol/blob/master/protocol.md#x-http-method-override
|
||||
if r.Header.Get("X-HTTP-Method-Override") != "" {
|
||||
method = r.Header.Get("X-HTTP-Method-Override")
|
||||
}
|
||||
|
||||
switch method {
|
||||
case "POST":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
// set etag, mtime and file id
|
||||
setHeaders(fs, w, r)
|
||||
handler.PostFile(w, r)
|
||||
case "HEAD":
|
||||
handler.HeadFile(w, r)
|
||||
case "PATCH":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
// set etag, mtime and file id
|
||||
setHeaders(fs, w, r)
|
||||
handler.PatchFile(w, r)
|
||||
case "DELETE":
|
||||
handler.DelFile(w, r)
|
||||
case "GET":
|
||||
metrics.DownloadsActive.Add(1)
|
||||
defer func() {
|
||||
metrics.DownloadsActive.Sub(1)
|
||||
}()
|
||||
// NOTE: this is breaking change - allthought it does not seem to be used
|
||||
// We can make a switch here depending on some header value if that is needed
|
||||
// download.GetOrHeadFile(w, r, fs, "")
|
||||
handler.GetFile(w, r)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotImplemented)
|
||||
}
|
||||
}))
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
id := path.Base(r.URL.Path)
|
||||
datastore, ok := fs.(tusd.DataStore)
|
||||
if !ok {
|
||||
appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore")
|
||||
return
|
||||
}
|
||||
upload, err := datastore.GetUpload(ctx, id)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage")
|
||||
return
|
||||
}
|
||||
info, err := upload.GetInfo(ctx)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload info for upload")
|
||||
return
|
||||
}
|
||||
expires := info.MetaData["expires"]
|
||||
if expires != "" {
|
||||
w.Header().Set(net.HeaderTusUploadExpires, expires)
|
||||
}
|
||||
resourceid := &provider.ResourceId{
|
||||
StorageId: info.MetaData["providerID"],
|
||||
SpaceId: info.Storage["SpaceRoot"],
|
||||
OpaqueId: info.Storage["NodeId"],
|
||||
}
|
||||
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(resourceid))
|
||||
}
|
||||
|
||||
// tusdLogger is a logger implementation (slog) for tusd that uses zerolog.
|
||||
type tusdLogger struct {
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
// Handle handles the record
|
||||
func (l tusdLogger) Handle(_ context.Context, r slog.Record) error {
|
||||
var logev *zerolog.Event
|
||||
switch r.Level {
|
||||
case slog.LevelDebug:
|
||||
logev = l.log.Debug()
|
||||
case slog.LevelInfo:
|
||||
logev = l.log.Info()
|
||||
case slog.LevelWarn:
|
||||
logev = l.log.Warn()
|
||||
case slog.LevelError:
|
||||
logev = l.log.Error()
|
||||
}
|
||||
// Extract caller information from slog.Record only for debug and info levels
|
||||
if (r.Level == slog.LevelDebug || r.Level == slog.LevelInfo) && r.PC != 0 {
|
||||
frames := runtime.CallersFrames([]uintptr{r.PC})
|
||||
frame, _ := frames.Next()
|
||||
// add line using zerolog's caller format
|
||||
logev = logev.Str("line", fmt.Sprintf("%s:%d", frame.File, frame.Line))
|
||||
}
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
// Resolve the Attr's value before doing anything else.
|
||||
a.Value = a.Value.Resolve()
|
||||
// Ignore empty Attrs.
|
||||
if a.Equal(slog.Attr{}) {
|
||||
return true
|
||||
}
|
||||
switch a.Value.Kind() {
|
||||
case slog.KindBool:
|
||||
logev = logev.Bool(a.Key, a.Value.Bool())
|
||||
case slog.KindInt64:
|
||||
logev = logev.Int64(a.Key, a.Value.Int64())
|
||||
default:
|
||||
logev = logev.Str(a.Key, a.Value.String())
|
||||
}
|
||||
return true
|
||||
})
|
||||
logev.Msg(r.Message)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Enabled returns true
|
||||
func (l tusdLogger) Enabled(_ context.Context, _ slog.Level) bool { return true }
|
||||
|
||||
// WithAttrs creates a new logger with the given attributes
|
||||
func (l tusdLogger) WithAttrs(attr []slog.Attr) slog.Handler {
|
||||
fields := make(map[string]interface{}, len(attr))
|
||||
for _, a := range attr {
|
||||
switch a.Value.Kind() {
|
||||
case slog.KindBool:
|
||||
fields[a.Key] = a.Value.Bool()
|
||||
case slog.KindInt64:
|
||||
fields[a.Key] = a.Value.Int64()
|
||||
case slog.KindUint64:
|
||||
fields[a.Key] = a.Value.Uint64()
|
||||
case slog.KindFloat64:
|
||||
fields[a.Key] = a.Value.Float64()
|
||||
default:
|
||||
fields[a.Key] = a.Value.String()
|
||||
}
|
||||
}
|
||||
c := l.log.With().Fields(fields).Logger()
|
||||
sLog := tusdLogger{log: &c}
|
||||
return sLog
|
||||
}
|
||||
|
||||
// WithGroup is not implemented
|
||||
func (l tusdLogger) WithGroup(name string) slog.Handler { return l }
|
||||
Generated
Vendored
+60
@@ -0,0 +1,60 @@
|
||||
// Package metrics provides prometheus metrics for the data managers..
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
// DownloadsActive is the number of active downloads
|
||||
DownloadsActive = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "reva_download_active",
|
||||
Help: "Number of active downloads",
|
||||
})
|
||||
// UploadsActive is the number of active uploads
|
||||
UploadsActive = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "reva_upload_active",
|
||||
Help: "Number of active uploads",
|
||||
})
|
||||
// UploadProcessing is the number of uploads in processing
|
||||
UploadProcessing = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "reva_upload_processing",
|
||||
Help: "Number of uploads in processing",
|
||||
})
|
||||
// UploadSessionsInitiated is the number of upload sessions that have been initiated
|
||||
UploadSessionsInitiated = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_initiated",
|
||||
Help: "Number of uploads sessions that were initiated",
|
||||
})
|
||||
// UploadSessionsBytesReceived is the number of upload sessions that have received all bytes
|
||||
UploadSessionsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_bytes_received",
|
||||
Help: "Number of uploads sessions that have received all bytes",
|
||||
})
|
||||
// UploadSessionsFinalized is the number of upload sessions that have received all bytes
|
||||
UploadSessionsFinalized = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_finalized",
|
||||
Help: "Number of uploads sessions that have successfully completed",
|
||||
})
|
||||
// UploadSessionsAborted is the number of upload sessions that have been aborted
|
||||
UploadSessionsAborted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_aborted",
|
||||
Help: "Number of uploads sessions that have aborted by postprocessing",
|
||||
})
|
||||
// UploadSessionsDeleted is the number of upload sessions that have been deleted
|
||||
UploadSessionsDeleted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_deleted",
|
||||
Help: "Number of uploads sessions that have been deleted by postprocessing",
|
||||
})
|
||||
// UploadSessionsRestarted is the number of upload sessions that have been restarted
|
||||
UploadSessionsRestarted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_restarted",
|
||||
Help: "Number of uploads sessions that have been restarted by postprocessing",
|
||||
})
|
||||
// UploadSessionsScanned is the number of upload sessions that have been scanned by antivirus
|
||||
UploadSessionsScanned = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_scanned",
|
||||
Help: "Number of uploads sessions that have been scanned by antivirus",
|
||||
})
|
||||
)
|
||||
Generated
Vendored
+285
@@ -0,0 +1,285 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
// Package download provides a library to handle file download requests.
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/rs/zerolog"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
type contextKey struct{}
|
||||
|
||||
var etagKey = contextKey{}
|
||||
|
||||
// ContextWithEtag returns a new `context.Context` that holds an etag.
|
||||
func ContextWithEtag(ctx context.Context, etag string) context.Context {
|
||||
return context.WithValue(ctx, etagKey, etag)
|
||||
}
|
||||
|
||||
// EtagFromContext returns the etag previously associated with `ctx`, or
|
||||
// `""` if no such etag could be found.
|
||||
func EtagFromContext(ctx context.Context) string {
|
||||
val := ctx.Value(etagKey)
|
||||
if etag, ok := val.(string); ok {
|
||||
return etag
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetOrHeadFile returns the requested file content
|
||||
func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceID string) {
|
||||
ctx := r.Context()
|
||||
sublog := appctx.GetLogger(ctx).With().Str("svc", "datatx").Str("handler", "download").Logger()
|
||||
|
||||
var fn string
|
||||
files, ok := r.URL.Query()["filename"]
|
||||
if !ok || len(files[0]) < 1 {
|
||||
fn = r.URL.Path
|
||||
} else {
|
||||
fn = files[0]
|
||||
}
|
||||
|
||||
var ref *provider.Reference
|
||||
if spaceID == "" {
|
||||
// ensure the absolute path starts with '/'
|
||||
ref = &provider.Reference{Path: path.Join("/", fn)}
|
||||
} else {
|
||||
// build a storage space reference
|
||||
rid, err := storagespace.ParseID(spaceID)
|
||||
if err != nil {
|
||||
handleError(w, &sublog, err, "parse ID")
|
||||
}
|
||||
ref = &provider.Reference{
|
||||
ResourceId: &rid,
|
||||
// ensure the relative path starts with '.'
|
||||
Path: utils.MakeRelativePath(fn),
|
||||
}
|
||||
}
|
||||
// TODO check preconditions like If-Range, If-Match ...
|
||||
|
||||
var md *provider.ResourceInfo
|
||||
var content io.ReadCloser
|
||||
var err error
|
||||
var notModified bool
|
||||
|
||||
// do a stat to set Content-Length and etag headers
|
||||
|
||||
md, content, err = fs.Download(ctx, ref, func(md *provider.ResourceInfo) bool {
|
||||
// range requests always need to open the reader to check if it is seekable
|
||||
if r.Header.Get("Range") != "" {
|
||||
return true
|
||||
}
|
||||
// otherwise, HEAD requests do not need to open a reader
|
||||
if r.Method == "HEAD" {
|
||||
return false
|
||||
}
|
||||
|
||||
// check etag, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match
|
||||
for _, etag := range r.Header.Values(net.HeaderIfNoneMatch) {
|
||||
if md.Etag == etag {
|
||||
// When the condition fails for GET and HEAD methods, then the server must return
|
||||
// HTTP status code 304 (Not Modified). [...] Note that the server generating a
|
||||
// 304 response MUST generate any of the following header fields that would have
|
||||
// been sent in a 200 (OK) response to the same request:
|
||||
// Cache-Control, Content-Location, Date, ETag, Expires, and Vary.
|
||||
notModified = true
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
handleError(w, &sublog, err, "download")
|
||||
return
|
||||
}
|
||||
if content != nil {
|
||||
defer content.Close()
|
||||
}
|
||||
if notModified {
|
||||
w.Header().Set(net.HeaderETag, md.Etag)
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
|
||||
// fill in storage provider id if it is missing
|
||||
if spaceID != "" && md.GetId().GetStorageId() == "" {
|
||||
md.Id.StorageId = ref.ResourceId.StorageId
|
||||
}
|
||||
|
||||
var ranges []HTTPRange
|
||||
|
||||
if r.Header.Get("Range") != "" {
|
||||
ranges, err = ParseRange(r.Header.Get("Range"), int64(md.Size))
|
||||
if err != nil {
|
||||
if err == ErrNoOverlap {
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", md.Size))
|
||||
}
|
||||
sublog.Error().Err(err).Interface("md", md).Interface("ranges", ranges).Msg("range request not satisfiable")
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
|
||||
return
|
||||
}
|
||||
if SumRangesSize(ranges) > int64(md.Size) {
|
||||
// The total number of bytes in all the ranges
|
||||
// is larger than the size of the file by
|
||||
// itself, so this is probably an attack, or a
|
||||
// dumb client. Ignore the range request.
|
||||
ranges = nil
|
||||
}
|
||||
}
|
||||
|
||||
code := http.StatusOK
|
||||
sendSize := int64(md.Size)
|
||||
var sendContent io.Reader = content
|
||||
|
||||
var s io.Seeker
|
||||
if s, ok = content.(io.Seeker); ok {
|
||||
// tell clients they can send range requests
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
}
|
||||
|
||||
w.Header().Set(net.HeaderContentType, strings.Join([]string{md.MimeType, "charset=UTF-8"}, "; "))
|
||||
|
||||
if len(ranges) > 0 {
|
||||
sublog.Debug().Int64("start", ranges[0].Start).Int64("length", ranges[0].Length).Msg("range request")
|
||||
if s == nil {
|
||||
sublog.Error().Int64("start", ranges[0].Start).Int64("length", ranges[0].Length).Msg("ReadCloser is not seekable")
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
|
||||
code = http.StatusPartialContent
|
||||
|
||||
switch {
|
||||
case len(ranges) == 1:
|
||||
// RFC 7233, Section 4.1:
|
||||
// "If a single part is being transferred, the server
|
||||
// generating the 206 response MUST generate a
|
||||
// Content-Range header field, describing what range
|
||||
// of the selected representation is enclosed, and a
|
||||
// payload consisting of the range.
|
||||
// ...
|
||||
// A server MUST NOT generate a multipart response to
|
||||
// a request for a single range, since a client that
|
||||
// does not request multiple parts might not support
|
||||
// multipart responses."
|
||||
ra := ranges[0]
|
||||
if _, err := s.Seek(ra.Start, io.SeekStart); err != nil {
|
||||
sublog.Error().Err(err).Int64("start", ra.Start).Int64("length", ra.Length).Msg("content is not seekable")
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
sendSize = ra.Length
|
||||
w.Header().Set("Content-Range", ra.ContentRange(int64(md.Size)))
|
||||
case len(ranges) > 1:
|
||||
sendSize = RangesMIMESize(ranges, md.MimeType, int64(md.Size))
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
mw := multipart.NewWriter(pw)
|
||||
w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
|
||||
sendContent = pr
|
||||
defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
|
||||
go func() {
|
||||
for _, ra := range ranges {
|
||||
part, err := mw.CreatePart(ra.MimeHeader(md.MimeType+"; charset=UTF-8", int64(md.Size)))
|
||||
if err != nil {
|
||||
_ = pw.CloseWithError(err) // CloseWithError always returns nil
|
||||
return
|
||||
}
|
||||
if _, err := s.Seek(ra.Start, io.SeekStart); err != nil {
|
||||
_ = pw.CloseWithError(err) // CloseWithError always returns nil
|
||||
return
|
||||
}
|
||||
if _, err := io.CopyN(part, content, ra.Length); err != nil {
|
||||
_ = pw.CloseWithError(err) // CloseWithError always returns nil
|
||||
return
|
||||
}
|
||||
}
|
||||
mw.Close()
|
||||
pw.Close()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
if w.Header().Get(net.HeaderContentEncoding) == "" {
|
||||
w.Header().Set(net.HeaderContentLength, strconv.FormatInt(sendSize, 10))
|
||||
}
|
||||
|
||||
w.Header().Set(net.HeaderContentDisposistion, net.ContentDispositionAttachment(path.Base(md.Path)))
|
||||
w.Header().Set(net.HeaderETag, md.Etag)
|
||||
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(md.Id))
|
||||
w.Header().Set(net.HeaderOCETag, md.Etag)
|
||||
w.Header().Set(net.HeaderLastModified, net.RFC1123Z(md.Mtime))
|
||||
|
||||
if md.Checksum != nil {
|
||||
w.Header().Set(net.HeaderOCChecksum, fmt.Sprintf("%s:%s", strings.ToUpper(string(storageprovider.GRPC2PKGXS(md.Checksum.Type))), md.Checksum.Sum))
|
||||
}
|
||||
|
||||
w.WriteHeader(code)
|
||||
|
||||
if r.Method != "HEAD" {
|
||||
var c int64
|
||||
c, err = io.CopyN(w, sendContent, sendSize)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Interface("resourceid", md.Id).Msg("error copying data to response")
|
||||
return
|
||||
}
|
||||
if c != sendSize {
|
||||
sublog.Error().Int64("copied", c).Int64("size", sendSize).Msg("copied vs size mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleError(w http.ResponseWriter, log *zerolog.Logger, err error, action string) {
|
||||
switch err.(type) {
|
||||
case errtypes.IsNotFound:
|
||||
log.Debug().Err(err).Str("action", action).Msg("file not found")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case errtypes.IsPermissionDenied:
|
||||
log.Debug().Err(err).Str("action", action).Msg("permission denied")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
case errtypes.Aborted:
|
||||
log.Debug().Err(err).Str("action", action).Msg("etags do not match")
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
case errtypes.TooEarly:
|
||||
log.Debug().Err(err).Str("action", action).Msg("file is still being processed")
|
||||
w.WriteHeader(http.StatusTooEarly)
|
||||
default:
|
||||
log.Error().Err(err).Str("action", action).Msg("unexpected error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+158
@@ -0,0 +1,158 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package download
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// taken from https://golang.org/src/net/http/fs.go
|
||||
|
||||
// ErrInvalidRange is returned by serveContent's parseRange if the Range is
|
||||
// malformed or invalid.
|
||||
var ErrInvalidRange = errors.New("invalid range")
|
||||
|
||||
// ErrNoOverlap is returned by serveContent's parseRange if first-byte-pos of
|
||||
// all of the byte-range-spec values is greater than the content size.
|
||||
var ErrNoOverlap = fmt.Errorf("%w: failed to overlap", ErrInvalidRange)
|
||||
|
||||
// HTTPRange specifies the byte range to be sent to the client.
|
||||
type HTTPRange struct {
|
||||
Start, Length int64
|
||||
}
|
||||
|
||||
// ContentRange formats a Range header string as per RFC 7233.
|
||||
func (r HTTPRange) ContentRange(size int64) string {
|
||||
return fmt.Sprintf("bytes %d-%d/%d", r.Start, r.Start+r.Length-1, size)
|
||||
}
|
||||
|
||||
// MimeHeader creates range relevant MimeHeaders
|
||||
func (r HTTPRange) MimeHeader(contentType string, size int64) textproto.MIMEHeader {
|
||||
return textproto.MIMEHeader{
|
||||
"Content-Range": {r.ContentRange(size)},
|
||||
"Content-Type": {contentType},
|
||||
}
|
||||
}
|
||||
|
||||
// ParseRange parses a Range header string as per RFC 7233.
|
||||
// errNoOverlap is returned if none of the ranges overlap.
|
||||
func ParseRange(s string, size int64) ([]HTTPRange, error) {
|
||||
if s == "" {
|
||||
return nil, nil // header not present
|
||||
}
|
||||
const b = "bytes="
|
||||
if !strings.HasPrefix(s, b) {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
ranges := []HTTPRange{}
|
||||
noOverlap := false
|
||||
for _, ra := range strings.Split(s[len(b):], ",") {
|
||||
ra = textproto.TrimString(ra)
|
||||
if ra == "" {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(ra, "-")
|
||||
if i < 0 {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:])
|
||||
var r HTTPRange
|
||||
if start == "" {
|
||||
// If no start is specified, end specifies the
|
||||
// range start relative to the end of the file.
|
||||
i, err := strconv.ParseInt(end, 10, 64)
|
||||
if err != nil {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
if i > size {
|
||||
i = size
|
||||
}
|
||||
r.Start = size - i
|
||||
r.Length = size - r.Start
|
||||
} else {
|
||||
i, err := strconv.ParseInt(start, 10, 64)
|
||||
if err != nil || i < 0 {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
if i >= size {
|
||||
// If the range begins after the size of the content,
|
||||
// then it does not overlap.
|
||||
noOverlap = true
|
||||
continue
|
||||
}
|
||||
r.Start = i
|
||||
if end == "" {
|
||||
// If no end is specified, range extends to end of the file.
|
||||
r.Length = size - r.Start
|
||||
} else {
|
||||
i, err := strconv.ParseInt(end, 10, 64)
|
||||
if err != nil || r.Start > i {
|
||||
return nil, ErrInvalidRange
|
||||
}
|
||||
if i >= size {
|
||||
i = size - 1
|
||||
}
|
||||
r.Length = i - r.Start + 1
|
||||
}
|
||||
}
|
||||
ranges = append(ranges, r)
|
||||
}
|
||||
if noOverlap && len(ranges) == 0 {
|
||||
// The specified ranges did not overlap with the content.
|
||||
return nil, ErrNoOverlap
|
||||
}
|
||||
return ranges, nil
|
||||
}
|
||||
|
||||
// countingWriter counts how many bytes have been written to it.
|
||||
type countingWriter int64
|
||||
|
||||
func (w *countingWriter) Write(p []byte) (n int, err error) {
|
||||
*w += countingWriter(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// RangesMIMESize returns the number of bytes it takes to encode the
|
||||
// provided ranges as a multipart response.
|
||||
func RangesMIMESize(ranges []HTTPRange, contentType string, contentSize int64) (encSize int64) {
|
||||
var w countingWriter
|
||||
mw := multipart.NewWriter(&w)
|
||||
for _, ra := range ranges {
|
||||
// CreatePart might return an error if the io.Copy for the boundaries fails
|
||||
// here parts are not filled, so we assume for now thet this will always succeed
|
||||
_, _ = mw.CreatePart(ra.MimeHeader(contentType, contentSize))
|
||||
encSize += ra.Length
|
||||
}
|
||||
_ = mw.Close()
|
||||
encSize += int64(w)
|
||||
return
|
||||
}
|
||||
|
||||
// SumRangesSize adds up the length of all ranges
|
||||
func SumRangesSize(ranges []HTTPRange) (size int64) {
|
||||
for _, ra := range ranges {
|
||||
size += ra.Length
|
||||
}
|
||||
return
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package global
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// NewMiddlewares contains all the registered new middleware functions.
|
||||
var NewMiddlewares = map[string]NewMiddleware{}
|
||||
|
||||
// NewMiddleware is the function that HTTP middlewares need to register at init time.
|
||||
type NewMiddleware func(conf map[string]interface{}) (Middleware, int, error)
|
||||
|
||||
// RegisterMiddleware registers a new HTTP middleware and its new function.
|
||||
func RegisterMiddleware(name string, n NewMiddleware) {
|
||||
NewMiddlewares[name] = n
|
||||
}
|
||||
|
||||
// Middleware is a middleware http handler.
|
||||
type Middleware func(h http.Handler) http.Handler
|
||||
|
||||
// Services is a map of service name and its new function.
|
||||
var Services = map[string]NewService{}
|
||||
|
||||
// Register registers a new HTTP services with name and new function.
|
||||
func Register(name string, newFunc NewService) {
|
||||
Services[name] = newFunc
|
||||
}
|
||||
|
||||
// NewService is the function that HTTP services need to register at init time.
|
||||
type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error)
|
||||
|
||||
// Service represents a HTTP service.
|
||||
type Service interface {
|
||||
Handler() http.Handler
|
||||
Prefix() string
|
||||
Close() error
|
||||
// List of url relative to the prefix to be unprotected by the authentication
|
||||
// middleware. To be seen if we need url-verb fine grained skip checks like
|
||||
// GET is public and POST is not.
|
||||
Unprotected() []string
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package rhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Context context.Context
|
||||
Timeout time.Duration
|
||||
Insecure bool
|
||||
DisableKeepAlive bool
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Insecure provides a function to set the insecure option.
|
||||
func Insecure(insecure bool) Option {
|
||||
return func(o *Options) {
|
||||
o.Insecure = insecure
|
||||
}
|
||||
}
|
||||
|
||||
// Timeout provides a function to set the timeout option.
|
||||
func Timeout(t time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.Timeout = t
|
||||
}
|
||||
}
|
||||
|
||||
// DisableKeepAlive provides a function to set the disablee keep alive option.
|
||||
func DisableKeepAlive(disable bool) Option {
|
||||
return func(o *Options) {
|
||||
o.DisableKeepAlive = disable
|
||||
}
|
||||
}
|
||||
+328
@@ -0,0 +1,328 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package rhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"path"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/log"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/interceptors/providerauthorizer"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/global"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
|
||||
rtrace "github.com/opencloud-eu/reva/v2/pkg/trace"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// name is the Tracer name used to identify this instrumentation library.
|
||||
const tracerName = "rhttp"
|
||||
|
||||
// New returns a new server
|
||||
func New(m interface{}, l zerolog.Logger, tp trace.TracerProvider) (*Server, error) {
|
||||
conf := &config{}
|
||||
if err := mapstructure.Decode(m, conf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conf.init()
|
||||
|
||||
httpServer := &http.Server{}
|
||||
s := &Server{
|
||||
httpServer: httpServer,
|
||||
conf: conf,
|
||||
svcs: map[string]global.Service{},
|
||||
unprotected: []string{},
|
||||
handlers: map[string]http.Handler{},
|
||||
log: l,
|
||||
tracerProvider: tp,
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Server contains the server info.
|
||||
type Server struct {
|
||||
httpServer *http.Server
|
||||
conf *config
|
||||
listener net.Listener
|
||||
svcs map[string]global.Service // map key is svc Prefix
|
||||
unprotected []string
|
||||
handlers map[string]http.Handler
|
||||
middlewares []*middlewareTriple
|
||||
log zerolog.Logger
|
||||
tracerProvider trace.TracerProvider
|
||||
}
|
||||
|
||||
type config struct {
|
||||
Network string `mapstructure:"network"`
|
||||
Address string `mapstructure:"address"`
|
||||
Services map[string]map[string]interface{} `mapstructure:"services"`
|
||||
Middlewares map[string]map[string]interface{} `mapstructure:"middlewares"`
|
||||
CertFile string `mapstructure:"certfile"`
|
||||
KeyFile string `mapstructure:"keyfile"`
|
||||
}
|
||||
|
||||
func (c *config) init() {
|
||||
// apply defaults
|
||||
if c.Network == "" {
|
||||
c.Network = "tcp"
|
||||
}
|
||||
|
||||
if c.Address == "" {
|
||||
c.Address = "0.0.0.0:19001"
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the server
|
||||
func (s *Server) Start(ln net.Listener) error {
|
||||
if err := s.registerServices(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.registerMiddlewares(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
handler, err := s.getHandler()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "rhttp: error creating http handler")
|
||||
}
|
||||
|
||||
s.httpServer.Handler = handler
|
||||
s.listener = ln
|
||||
|
||||
if (s.conf.CertFile != "") && (s.conf.KeyFile != "") {
|
||||
s.log.Info().Msgf("https server listening at https://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile)
|
||||
err = s.httpServer.ServeTLS(s.listener, s.conf.CertFile, s.conf.KeyFile)
|
||||
} else {
|
||||
s.log.Info().Msgf("http server listening at http://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile)
|
||||
err = s.httpServer.Serve(s.listener)
|
||||
}
|
||||
if err == nil || err == http.ErrServerClosed {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Stop stops the server.
|
||||
func (s *Server) Stop() error {
|
||||
// TODO(labkode): set ctx deadline to zero
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
defer s.closeServices()
|
||||
return s.httpServer.Shutdown(ctx)
|
||||
}
|
||||
|
||||
// TODO(labkode): we can't stop the server shutdown because a service cannot be shutdown.
|
||||
// What do we do in case a service cannot be properly closed? Now we just log the error.
|
||||
// TODO(labkode): the close should be given a deadline using context.Context.
|
||||
func (s *Server) closeServices() {
|
||||
for _, svc := range s.svcs {
|
||||
if err := svc.Close(); err != nil {
|
||||
s.log.Error().Err(err).Msgf("error closing service %q", svc.Prefix())
|
||||
} else {
|
||||
s.log.Info().Msgf("service %q correctly closed", svc.Prefix())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Network return the network type.
|
||||
func (s *Server) Network() string {
|
||||
return s.conf.Network
|
||||
}
|
||||
|
||||
// Address returns the network address.
|
||||
func (s *Server) Address() string {
|
||||
return s.conf.Address
|
||||
}
|
||||
|
||||
// GracefulStop gracefully stops the server.
|
||||
func (s *Server) GracefulStop() error {
|
||||
defer s.closeServices()
|
||||
return s.httpServer.Shutdown(context.Background())
|
||||
}
|
||||
|
||||
// middlewareTriple represents a middleware with the
|
||||
// priority to be chained.
|
||||
type middlewareTriple struct {
|
||||
Name string
|
||||
Priority int
|
||||
Middleware global.Middleware
|
||||
}
|
||||
|
||||
func (s *Server) registerMiddlewares() error {
|
||||
middlewares := []*middlewareTriple{}
|
||||
for name, newFunc := range global.NewMiddlewares {
|
||||
if s.isMiddlewareEnabled(name) {
|
||||
m, prio, err := newFunc(s.conf.Middlewares[name])
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error creating new middleware: %s,", name)
|
||||
return err
|
||||
}
|
||||
middlewares = append(middlewares, &middlewareTriple{
|
||||
Name: name,
|
||||
Priority: prio,
|
||||
Middleware: m,
|
||||
})
|
||||
s.log.Info().Msgf("http middleware enabled: %s", name)
|
||||
}
|
||||
}
|
||||
s.middlewares = middlewares
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) isMiddlewareEnabled(name string) bool {
|
||||
_, ok := s.conf.Middlewares[name]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *Server) registerServices() error {
|
||||
for svcName := range s.conf.Services {
|
||||
if s.isServiceEnabled(svcName) {
|
||||
newFunc := global.Services[svcName]
|
||||
svc, err := newFunc(s.conf.Services[svcName], &s.log)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "http service %s could not be started,", svcName)
|
||||
return err
|
||||
}
|
||||
|
||||
// instrument services with opencensus tracing.
|
||||
h := traceHandler(svcName, svc.Handler(), s.tracerProvider)
|
||||
s.handlers[svc.Prefix()] = h
|
||||
s.svcs[svc.Prefix()] = svc
|
||||
s.unprotected = append(s.unprotected, getUnprotected(svc.Prefix(), svc.Unprotected())...)
|
||||
s.log.Info().Msgf("http service enabled: %s@/%s", svcName, svc.Prefix())
|
||||
} else {
|
||||
message := fmt.Sprintf("http service %s does not exist", svcName)
|
||||
return errors.New(message)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) isServiceEnabled(svcName string) bool {
|
||||
_, ok := global.Services[svcName]
|
||||
return ok
|
||||
}
|
||||
|
||||
// TODO(labkode): if the http server is exposed under a basename we need to prepend
|
||||
// to prefix.
|
||||
func getUnprotected(prefix string, unprotected []string) []string {
|
||||
for i := range unprotected {
|
||||
unprotected[i] = path.Join("/", prefix, unprotected[i])
|
||||
}
|
||||
return unprotected
|
||||
}
|
||||
|
||||
func (s *Server) getHandler() (http.Handler, error) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
head, tail := router.ShiftPath(r.URL.Path)
|
||||
if h, ok := s.handlers[head]; ok {
|
||||
r.URL.Path = tail
|
||||
s.log.Debug().Msgf("http routing: head=%s tail=%s svc=%s", head, r.URL.Path, head)
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// when a service is exposed at the root.
|
||||
if h, ok := s.handlers[""]; ok {
|
||||
r.URL.Path = "/" + head + tail
|
||||
s.log.Debug().Msgf("http routing: head= tail=%s svc=root", r.URL.Path)
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Debug().Msgf("http routing: head=%s tail=%s svc=not-found", head, tail)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
})
|
||||
|
||||
// sort middlewares by priority.
|
||||
sort.SliceStable(s.middlewares, func(i, j int) bool {
|
||||
return s.middlewares[i].Priority > s.middlewares[j].Priority
|
||||
})
|
||||
|
||||
handler := http.Handler(h)
|
||||
|
||||
for _, triple := range s.middlewares {
|
||||
s.log.Info().Msgf("chaining http middleware %s with priority %d", triple.Name, triple.Priority)
|
||||
handler = triple.Middleware(traceHandler(triple.Name, handler, s.tracerProvider))
|
||||
}
|
||||
|
||||
for _, v := range s.unprotected {
|
||||
s.log.Info().Msgf("unprotected URL: %s", v)
|
||||
}
|
||||
authMiddle, err := auth.New(s.conf.Middlewares["auth"], s.unprotected, s.tracerProvider)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "rhttp: error creating auth middleware")
|
||||
}
|
||||
|
||||
// add always the logctx middleware as most priority, this middleware is internal
|
||||
// and cannot be configured from the configuration.
|
||||
coreMiddlewares := []*middlewareTriple{}
|
||||
|
||||
providerAuthMiddle, err := addProviderAuthMiddleware(s.conf, s.unprotected)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "rhttp: error creating providerauthorizer middleware")
|
||||
}
|
||||
if providerAuthMiddle != nil {
|
||||
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: providerAuthMiddle, Name: "providerauthorizer"})
|
||||
}
|
||||
|
||||
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: authMiddle, Name: "auth"})
|
||||
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: log.New(), Name: "log"})
|
||||
coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: appctx.New(s.log, s.tracerProvider), Name: "appctx"})
|
||||
|
||||
for _, triple := range coreMiddlewares {
|
||||
handler = triple.Middleware(traceHandler(triple.Name, handler, s.tracerProvider))
|
||||
}
|
||||
|
||||
return handler, nil
|
||||
}
|
||||
|
||||
func traceHandler(name string, h http.Handler, tp trace.TracerProvider) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := rtrace.Propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
|
||||
t := tp.Tracer(tracerName)
|
||||
ctx, span := t.Start(ctx, name)
|
||||
defer span.End()
|
||||
|
||||
rtrace.Propagator.Inject(ctx, propagation.HeaderCarrier(r.Header))
|
||||
h.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
func addProviderAuthMiddleware(conf *config, unprotected []string) (global.Middleware, error) {
|
||||
_, ocmdRegistered := global.Services["ocmd"]
|
||||
_, ocmdEnabled := conf.Services["ocmd"]
|
||||
ocmdPrefix, _ := conf.Services["ocmd"]["prefix"].(string)
|
||||
if ocmdRegistered && ocmdEnabled {
|
||||
return providerauthorizer.New(conf.Middlewares["providerauthorizer"], unprotected, ocmdPrefix)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package router
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ShiftPath splits off the first component of p, which will be cleaned of
|
||||
// relative components before processing. head will never contain a slash and
|
||||
// tail will always be a rooted path without trailing slash.
|
||||
// see https://blog.merovius.de/2017/06/18/how-not-to-use-an-http-router.html
|
||||
// and https://gist.github.com/weatherglass/62bd8a704d4dfdc608fe5c5cb5a6980c#gistcomment-2161690 for the zero alloc code below
|
||||
func ShiftPath(p string) (head, tail string) {
|
||||
if p == "" {
|
||||
return "", "/"
|
||||
}
|
||||
p = strings.TrimPrefix(path.Clean(p), "/")
|
||||
i := strings.Index(p, "/")
|
||||
if i < 0 {
|
||||
return p, "/"
|
||||
}
|
||||
return p[:i], p[i:]
|
||||
}
|
||||
Reference in New Issue
Block a user