Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+19
View File
@@ -0,0 +1,19 @@
Copyright (c) 2013-2017 Transloadit Ltd and Contributors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+147
View File
@@ -0,0 +1,147 @@
package handler
import (
"errors"
"io"
"net"
"net/http"
"os"
"strings"
"sync"
"sync/atomic"
"time"
)
// bodyReader is an io.Reader, which is intended to wrap the request
// body reader. If an error occurr during reading the request body, it
// will not return this error to the reading entity, but instead store
// the error and close the io.Reader, so that the error can be checked
// afterwards. This is helpful, so that the stores do not have to handle
// the error but this can instead be done in the handler.
// In addition, the bodyReader keeps track of how many bytes were read.
type bodyReader struct {
// bytesCounter is the first field to ensure that it's properly aligned,
// otherwise we run into alignment issues on some 32-bit builds.
// See https://github.com/tus/tusd/issues/1047
// See https://pkg.go.dev/sync/atomic#pkg-note-BUG
// TODO: In the future we should move all of these values to the safe
// atomic.Uint64 type, which takes care of alignment automatically.
bytesCounter int64
ctx *httpContext
reader io.ReadCloser
onReadDone func()
// lock protects concurrent access to err.
lock sync.RWMutex
err error
}
func newBodyReader(c *httpContext, maxSize int64) *bodyReader {
return &bodyReader{
ctx: c,
reader: http.MaxBytesReader(c.res, c.req.Body, maxSize),
onReadDone: func() {},
}
}
func (r *bodyReader) Read(b []byte) (int, error) {
r.lock.RLock()
hasErrored := r.err != nil
r.lock.RUnlock()
if hasErrored {
return 0, io.EOF
}
n, err := r.reader.Read(b)
atomic.AddInt64(&r.bytesCounter, int64(n))
if !errors.Is(err, os.ErrDeadlineExceeded) {
// If the timeout wasn't exceeded (due to SetReadDeadline), invoke
// the callback so the deadline can be extended
r.onReadDone()
}
if err != nil {
// Note: if an error occurs while reading the body, we must set `r.err` (either in here
// or somewhere else, such as in closeWithError). Otherwise, the PATCH handler might not know
// that an error occurred and assumes that a request was transferred succesfully even though
// it was interrupted. This leads to problems with the RUFH draft.
// io.EOF means that the request body was fully read and does not represent an error.
if err == io.EOF {
return n, io.EOF
}
// http.ErrBodyReadAfterClose means that the bodyReader closed the request body because the upload is
// is stopped or the server shuts down. In this case, the closeWithError method already
// set `r.err` and thus we don't overerwrite it here but just return.
if err == http.ErrBodyReadAfterClose {
return n, io.EOF
}
// All of the following errors can be understood as the input stream ending too soon:
// - io.ErrClosedPipe is returned in the package's unit test with io.Pipe()
// - io.UnexpectedEOF means that the client aborted the request.
if err == io.ErrClosedPipe || err == io.ErrUnexpectedEOF {
err = ErrUnexpectedEOF
}
// Connection resets are not dropped silently, but responded to the client.
// We change the error because otherwise the message would contain the local address,
// which is unnecessary to be included in the response.
if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
err = ErrConnectionReset
}
// For timeouts, we also send a nicer response to the clients.
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
err = ErrReadTimeout
}
// MaxBytesError is returned from http.MaxBytesReader, which we use to limit
// the request body size.
maxBytesErr := &http.MaxBytesError{}
if errors.As(err, &maxBytesErr) {
err = ErrSizeExceeded
}
// Other errors are stored for retrival with hasError, but is not returned
// to the consumer. We do not overwrite an error if it has been set already.
r.lock.Lock()
if r.err == nil {
r.err = err
}
r.lock.Unlock()
}
return n, nil
}
func (r *bodyReader) hasError() error {
r.lock.RLock()
err := r.err
r.lock.RUnlock()
if err == io.EOF {
return nil
}
return err
}
func (r *bodyReader) bytesRead() int64 {
return atomic.LoadInt64(&r.bytesCounter)
}
func (r *bodyReader) closeWithError(err error) {
r.lock.Lock()
r.err = err
r.lock.Unlock()
// SetReadDeadline with the current time causes concurrent reads to the body to time out,
// so the body will be closed sooner with less delay.
if err := r.ctx.resC.SetReadDeadline(time.Now()); err != nil {
r.ctx.log.Warn("NetworkTimeoutError", "error", err)
}
r.reader.Close()
}
+94
View File
@@ -0,0 +1,94 @@
package handler
// StoreComposer represents a composable data store. It consists of the core
// data store and optional extensions. Please consult the package's overview
// for a more detailed introduction in how to use this structure.
type StoreComposer struct {
Core DataStore
UsesTerminater bool
Terminater TerminaterDataStore
UsesLocker bool
Locker Locker
UsesConcater bool
Concater ConcaterDataStore
UsesLengthDeferrer bool
LengthDeferrer LengthDeferrerDataStore
ContentServer ContentServerDataStore
UsesContentServer bool
}
// NewStoreComposer creates a new and empty store composer.
func NewStoreComposer() *StoreComposer {
return &StoreComposer{}
}
// Capabilities returns a string representing the provided extensions in a
// human-readable format meant for debugging.
func (store *StoreComposer) Capabilities() string {
str := "Core: "
if store.Core != nil {
str += "✓"
} else {
str += "✗"
}
str += ` Terminater: `
if store.UsesTerminater {
str += "✓"
} else {
str += "✗"
}
str += ` Locker: `
if store.UsesLocker {
str += "✓"
} else {
str += "✗"
}
str += ` Concater: `
if store.UsesConcater {
str += "✓"
} else {
str += "✗"
}
str += ` LengthDeferrer: `
if store.UsesLengthDeferrer {
str += "✓"
} else {
str += "✗"
}
return str
}
// UseCore will set the used core data store. If the argument is nil, the
// property will be unset.
func (store *StoreComposer) UseCore(core DataStore) {
store.Core = core
}
func (store *StoreComposer) UseTerminater(ext TerminaterDataStore) {
store.UsesTerminater = ext != nil
store.Terminater = ext
}
func (store *StoreComposer) UseLocker(ext Locker) {
store.UsesLocker = ext != nil
store.Locker = ext
}
func (store *StoreComposer) UseConcater(ext ConcaterDataStore) {
store.UsesConcater = ext != nil
store.Concater = ext
}
func (store *StoreComposer) UseLengthDeferrer(ext LengthDeferrerDataStore) {
store.UsesLengthDeferrer = ext != nil
store.LengthDeferrer = ext
}
func (store *StoreComposer) UseContentServer(ext ContentServerDataStore) {
store.UsesContentServer = ext != nil
store.ContentServer = ext
}
+74
View File
@@ -0,0 +1,74 @@
package handler
#define USE_FUNC(TYPE) \
func (store *StoreComposer) Use ## TYPE(ext TYPE ## DataStore) { \
store.Uses ## TYPE = ext != nil; \
store.TYPE = ext; \
}
#define USE_FIELD(TYPE) Uses ## TYPE bool; \
TYPE TYPE ## DataStore
#define USE_FROM(TYPE) if mod, ok := store.(TYPE ## DataStore); ok { \
composer.Use ## TYPE (mod) \
}
#define USE_CAP(TYPE) str += ` TYPE: `; \
if store.Uses ## TYPE { \
str += "✓" \
} else { \
str += "✗" \
}
// StoreComposer represents a composable data store. It consists of the core
// data store and optional extensions. Please consult the package's overview
// for a more detailed introduction in how to use this structure.
type StoreComposer struct {
Core DataStore
USE_FIELD(Terminater)
USE_FIELD(Finisher)
USE_FIELD(Locker)
USE_FIELD(GetReader)
USE_FIELD(Concater)
USE_FIELD(LengthDeferrer)
}
// NewStoreComposer creates a new and empty store composer.
func NewStoreComposer() *StoreComposer {
return &StoreComposer{}
}
// Capabilities returns a string representing the provided extensions in a
// human-readable format meant for debugging.
func (store *StoreComposer) Capabilities() string {
str := "Core: "
if store.Core != nil {
str += "✓"
} else {
str += "✗"
}
USE_CAP(Terminater)
USE_CAP(Finisher)
USE_CAP(Locker)
USE_CAP(GetReader)
USE_CAP(Concater)
USE_CAP(LengthDeferrer)
return str
}
// UseCore will set the used core data store. If the argument is nil, the
// property will be unset.
func (store *StoreComposer) UseCore(core DataStore) {
store.Core = core
}
USE_FUNC(Terminater)
USE_FUNC(Finisher)
USE_FUNC(Locker)
USE_FUNC(GetReader)
USE_FUNC(Concater)
USE_FUNC(LengthDeferrer)
+205
View File
@@ -0,0 +1,205 @@
package handler
import (
"errors"
"net/url"
"regexp"
"time"
"golang.org/x/exp/slog"
)
// Config provides a way to configure the Handler depending on your needs.
type Config struct {
// StoreComposer points to the store composer from which the core data store
// and optional dependencies should be taken. May only be nil if DataStore is
// set.
StoreComposer *StoreComposer
// MaxSize defines how many bytes may be stored in one single upload. If its
// value is is 0 or smaller no limit will be enforced.
MaxSize int64
// BasePath defines the URL path used for handling uploads, e.g. "/files/".
// If no trailing slash is presented it will be added. You may specify an
// absolute URL containing a scheme, e.g. "http://tus.io"
BasePath string
isAbs bool
// EnableExperimentalProtocol controls whether the new resumable upload protocol draft
// from the IETF's HTTP working group is accepted next to the current tus v1 protocol.
// See https://datatracker.ietf.org/doc/draft-ietf-httpbis-resumable-upload/
EnableExperimentalProtocol bool
// DisableDownload indicates whether the server will refuse downloads of the
// uploaded file, by not mounting the GET handler.
DisableDownload bool
// DisableTermination indicates whether the server will refuse termination
// requests of the uploaded file, by not mounting the DELETE handler.
DisableTermination bool
// DisableConcatenation indicates whether the server will refuse POST requests
// for creating uploads that use the concatenation extension.
DisableConcatenation bool
// Cors can be used to customize the handling of Cross-Origin Resource Sharing (CORS).
// See the CorsConfig struct for more details.
// Defaults to DefaultCorsConfig.
Cors *CorsConfig
// NotifyCompleteUploads indicates whether sending notifications about
// completed uploads using the CompleteUploads channel should be enabled.
NotifyCompleteUploads bool
// NotifyTerminatedUploads indicates whether sending notifications about
// terminated uploads using the TerminatedUploads channel should be enabled.
NotifyTerminatedUploads bool
// NotifyUploadProgress indicates whether sending notifications about
// the upload progress using the UploadProgress channel should be enabled.
NotifyUploadProgress bool
// NotifyCreatedUploads indicates whether sending notifications about
// the upload having been created using the CreatedUploads channel should be enabled.
NotifyCreatedUploads bool
// UploadProgressInterval specifies the interval at which the upload progress
// notifications are sent to the UploadProgress channel, if enabled.
// Defaults to 1s.
UploadProgressInterval time.Duration
// Logger is the logger to use internally, mostly for printing requests.
Logger *slog.Logger
// Respect the X-Forwarded-Host, X-Forwarded-Proto and Forwarded headers
// potentially set by proxies when generating an absolute URL in the
// response to POST requests.
RespectForwardedHeaders bool
// PreUploadCreateCallback will be invoked before a new upload is created, if the
// property is supplied. If the callback returns no error, the upload will be created
// and optional values from HTTPResponse will be contained in the HTTP response.
// If the error is non-nil, the upload will not be created. This can be used to implement
// validation of upload metadata etc. Furthermore, HTTPResponse will be ignored and
// the error value can contain values for the HTTP response.
// If the error is nil, FileInfoChanges can be filled out to specify individual properties
// that should be overwriten before the upload is create. See its type definition for
// more details on its behavior. If you do not want to make any changes, return an empty struct.
PreUploadCreateCallback func(hook HookEvent) (HTTPResponse, FileInfoChanges, error)
// PreFinishResponseCallback will be invoked after an upload is completed but before
// a response is returned to the client. This can be used to implement post-processing validation.
// If the callback returns no error, optional values from HTTPResponse will be contained in the HTTP response.
// If the error is non-nil, the error will be forwarded to the client. Furthermore,
// HTTPResponse will be ignored and the error value can contain values for the HTTP response.
PreFinishResponseCallback func(hook HookEvent) (HTTPResponse, error)
// PreUploadTerminateCallback will be invoked on DELETE requests before an upload is terminated,
// giving the application the opportunity to reject the termination. For example, to ensure resources
// used by other services are not deleted.
// If the callback returns no error, optional values from HTTPResponse will be contained in the HTTP response.
// If the error is non-nil, the error will be forwarded to the client. Furthermore,
// HTTPResponse will be ignored and the error value can contain values for the HTTP response.
PreUploadTerminateCallback func(hook HookEvent) (HTTPResponse, error)
// GracefulRequestCompletionTimeout is the timeout for operations to complete after an HTTP
// request has ended (successfully or by error). For example, if an HTTP request is interrupted,
// instead of stopping immediately, the handler and data store will be given some additional
// time to wrap up their operations and save any uploaded data. GracefulRequestCompletionTimeout
// controls this time.
// See HookEvent.Context for more details.
// Defaults to 10s.
GracefulRequestCompletionTimeout time.Duration
// AcquireLockTimeout is the duration that a request handler will wait to acquire a lock for
// an upload. If the timeout is reached, it will stop waiting and send an error response to the
// client.
// Defaults to 20s.
AcquireLockTimeout time.Duration
// NetworkTimeout is the timeout for individual read operations on the request body. If the
// read operation succeeds in this time window, the handler will continue consuming the body.
// If a read operation times out, the handler will stop reading and close the request.
// This ensures that an upload is consumed while data is being transmitted, while also closing
// dead connections.
// Under the hood, this is passed to ResponseController.SetReadDeadline
// Defaults to 60s
NetworkTimeout time.Duration
}
// CorsConfig provides a way to customize the the handling of Cross-Origin Resource Sharing (CORS).
// More details about CORS are available at https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS.
type CorsConfig struct {
// Disable instructs the handler to ignore all CORS-related headers and never set a
// CORS-related header in a response. This is useful if CORS is already handled by a proxy.
Disable bool
// AllowOrigin is a regular expression used to check if a request is allowed to participate in the
// CORS protocol. If the request's Origin header matches the regular expression, CORS is allowed.
// If not, a 403 Forbidden response is sent, rejecting the CORS request.
AllowOrigin *regexp.Regexp
// AllowCredentials defines whether the `Access-Control-Allow-Credentials: true` header should be
// included in CORS responses. This allows clients to share credentials using the Cookie and
// Authorization header
AllowCredentials bool
// AllowMethods defines the value for the `Access-Control-Allow-Methods` header in the response to
// preflight requests. You can add custom methods here, but make sure that all tus-specific methods
// from DefaultConfig.AllowMethods are included as well.
AllowMethods string
// AllowHeaders defines the value for the `Access-Control-Allow-Headers` header in the response to
// preflight requests. You can add custom headers here, but make sure that all tus-specific header
// from DefaultConfig.AllowHeaders are included as well.
AllowHeaders string
// MaxAge defines the value for the `Access-Control-Max-Age` header in the response to preflight
// requests.
MaxAge string
// ExposeHeaders defines the value for the `Access-Control-Expose-Headers` header in the response to
// actual requests. You can add custom headers here, but make sure that all tus-specific header
// from DefaultConfig.ExposeHeaders are included as well.
ExposeHeaders string
}
// DefaultCorsConfig is the configuration that will be used in none is provided.
var DefaultCorsConfig = CorsConfig{
Disable: false,
AllowOrigin: regexp.MustCompile(".*"),
AllowCredentials: false,
AllowMethods: "POST, HEAD, PATCH, OPTIONS, GET, DELETE",
AllowHeaders: "Authorization, Origin, X-Requested-With, X-Request-ID, X-HTTP-Method-Override, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat, Upload-Incomplete, Upload-Complete, Upload-Draft-Interop-Version",
MaxAge: "86400",
ExposeHeaders: "Upload-Offset, Location, Upload-Length, Tus-Version, Tus-Resumable, Tus-Max-Size, Tus-Extension, Upload-Metadata, Upload-Defer-Length, Upload-Concat, Upload-Incomplete, Upload-Complete, Upload-Draft-Interop-Version",
}
func (config *Config) validate() error {
if config.Logger == nil {
config.Logger = slog.Default()
}
base := config.BasePath
uri, err := url.Parse(base)
if err != nil {
return err
}
// Ensure base path ends with slash to remove logic from absFileURL
if base != "" && string(base[len(base)-1]) != "/" {
base += "/"
}
// Ensure base path begins with slash if not absolute (starts with scheme)
if !uri.IsAbs() && len(base) > 0 && string(base[0]) != "/" {
base = "/" + base
}
config.BasePath = base
config.isAbs = uri.IsAbs()
if config.StoreComposer == nil {
return errors.New("tusd: StoreComposer must no be nil")
}
if config.StoreComposer.Core == nil {
return errors.New("tusd: StoreComposer in Config needs to contain a non-nil core")
}
if config.UploadProgressInterval <= 0 {
config.UploadProgressInterval = 1 * time.Second
}
if config.GracefulRequestCompletionTimeout <= 0 {
config.GracefulRequestCompletionTimeout = 10 * time.Second
}
if config.AcquireLockTimeout <= 0 {
config.AcquireLockTimeout = 20 * time.Second
}
if config.NetworkTimeout <= 0 {
config.NetworkTimeout = 60 * time.Second
}
if config.Cors == nil {
config.Cors = &DefaultCorsConfig
}
return nil
}
+97
View File
@@ -0,0 +1,97 @@
package handler
import (
"context"
"errors"
"net/http"
"time"
"golang.org/x/exp/slog"
)
// httpContext is wrapper around context.Context that also carries the
// corresponding HTTP request and response writer, as well as an
// optional body reader
type httpContext struct {
context.Context
// res and req are the native request and response instances
res http.ResponseWriter
resC *http.ResponseController
req *http.Request
// body is nil by default and set by the user if the request body is consumed.
body *bodyReader
// cancel allows a user to cancel the internal request context, causing
// the request body to be closed.
cancel context.CancelCauseFunc
// log is the logger for this request. It gets extended with more properties as the
// request progresses and is identified.
log *slog.Logger
}
// newContext constructs a new httpContext for the given request. This should only be done once
// per request and the context should be stored in the request, so it can be fetched with getContext.
func (h UnroutedHandler) newContext(w http.ResponseWriter, r *http.Request) *httpContext {
// requestCtx is the context from the native request instance. It gets cancelled
// if the connection closes, the request is cancelled (HTTP/2), ServeHTTP returns
// or the server's base context is cancelled.
requestCtx := r.Context()
// On top of requestCtx, we construct a context that we can cancel, for example when
// the post-receive hook stops an upload or if another uploads requests a lock to be released.
cancellableCtx, cancelHandling := context.WithCancelCause(requestCtx)
// On top of cancellableCtx, we construct a new context which gets cancelled with a delay.
// See HookEvent.Context for more details, but the gist is that we want to give data stores
// some more time to finish their buisness.
delayedCtx := newDelayedContext(cancellableCtx, h.config.GracefulRequestCompletionTimeout)
ctx := &httpContext{
Context: delayedCtx,
res: w,
resC: http.NewResponseController(w),
req: r,
body: nil, // body can be filled later for PATCH requests
cancel: cancelHandling,
log: h.logger.With("method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r)),
}
go func() {
<-cancellableCtx.Done()
// If the cause is one of our own errors, close a potential body and relay the error.
cause := context.Cause(cancellableCtx)
if (errors.Is(cause, ErrServerShutdown) || errors.Is(cause, ErrUploadInterrupted) || errors.Is(cause, ErrUploadStoppedByServer)) && ctx.body != nil {
ctx.body.closeWithError(cause)
}
}()
return ctx
}
// getContext tries to retrieve a httpContext from the request or constructs a new one.
func (h UnroutedHandler) getContext(w http.ResponseWriter, r *http.Request) *httpContext {
c, ok := r.Context().(*httpContext)
if !ok {
c = h.newContext(w, r)
}
return c
}
// newDelayedContext returns a context with delayed cancellation propagation. If the parent context
// is done, the new context will also be cancelled but only after waiting the specified delay.
// Note: The parent context MUST be cancelled or otherwise this will leak resources. In the
// case of http.Request.Context, the net/http package ensures that the context is always cancelled.
func newDelayedContext(parent context.Context, delay time.Duration) context.Context {
// Use context.WithoutCancel to preserve the values.
ctx, cancel := context.WithCancel(context.WithoutCancel(parent))
go func() {
<-parent.Done()
<-time.After(delay)
cancel()
}()
return ctx
}
+212
View File
@@ -0,0 +1,212 @@
package handler
import (
"context"
"io"
"net/http"
)
type MetaData map[string]string
// FileInfo contains information about a single upload resource.
type FileInfo struct {
// ID is the unique identifier of the upload resource.
ID string
// Total file size in bytes specified in the NewUpload call
Size int64
// Indicates whether the total file size is deferred until later
SizeIsDeferred bool
// Offset in bytes (zero-based)
Offset int64
MetaData MetaData
// Indicates that this is a partial upload which will later be used to form
// a final upload by concatenation. Partial uploads should not be processed
// when they are finished since they are only incomplete chunks of files.
IsPartial bool
// Indicates that this is a final upload
IsFinal bool
// If the upload is a final one (see IsFinal) this will be a non-empty
// ordered slice containing the ids of the uploads of which the final upload
// will consist after concatenation.
PartialUploads []string
// Storage contains information about where the data storage saves the upload,
// for example a file path. The available values vary depending on what data
// store is used. This map may also be nil.
Storage map[string]string
// stopUpload is a callback for communicating that an upload should by stopped
// and interrupt the writes to DataStore#WriteChunk.
stopUpload func(HTTPResponse)
}
// StopUpload interrupts a running upload from the server-side. This means that
// the current request body is closed, so that the data store does not get any
// more data. Furthermore, a response is sent to notify the client of the
// interrupting and the upload is terminated (if supported by the data store),
// so the upload cannot be resumed anymore. The response to the client can be
// optionally modified by providing values in the HTTPResponse struct.
func (f FileInfo) StopUpload(response HTTPResponse) {
if f.stopUpload != nil {
f.stopUpload(response)
}
}
// FileInfoChanges collects changes the should be made to a FileInfo struct. This
// can be done using the PreUploadCreateCallback to modify certain properties before
// an upload is created. Properties which should not be modified (e.g. Size or Offset)
// are intentionally left out here.
//
// Please also consult the documentation for the `ChangeFileInfo` property at
// https://tus.github.io/tusd/advanced-topics/hooks/#hook-requests-and-responses.
type FileInfoChanges struct {
// If ID is not empty, it will be passed to the data store, allowing
// hooks to influence the upload ID. Be aware that a data store is not required to
// respect a pre-defined upload ID and might overwrite or modify it. However,
// all data stores in the github.com/tus/tusd package do respect pre-defined IDs.
ID string
// If MetaData is not nil, it replaces the entire user-defined meta data from
// the upload creation request. You can add custom meta data fields this way
// or ensure that only certain fields from the user-defined meta data are saved.
// If you want to retain only specific entries from the user-defined meta data, you must
// manually copy them into this MetaData field.
// If you do not want to store any meta data, set this field to an empty map (`MetaData{}`).
// If you want to keep the entire user-defined meta data, set this field to nil.
MetaData MetaData
// If Storage is not nil, it is passed to the data store to allow for minor adjustments
// to the upload storage (e.g. destination file name). The details are specific for each
// data store and should be looked up in their respective documentation.
// Please be aware that this behavior is currently not supported by any data store in
// the github.com/tus/tusd package.
Storage map[string]string
}
type Upload interface {
// Write the chunk read from src into the file specified by the id at the
// given offset. The handler will take care of validating the offset and
// limiting the size of the src to not overflow the file's size.
// The handler will also lock resources while they are written to ensure only one
// write happens per time.
// The function call must return the number of bytes written.
WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error)
// Read the fileinformation used to validate the offset and respond to HEAD
// requests.
GetInfo(ctx context.Context) (FileInfo, error)
// GetReader returns an io.ReadCloser which allows iterating of the content of an
// upload. It should attempt to provide a reader even if the upload has not
// been finished yet but it's not required.
GetReader(ctx context.Context) (io.ReadCloser, error)
// FinisherDataStore is the interface which can be implemented by DataStores
// which need to do additional operations once an entire upload has been
// completed. These tasks may include but are not limited to freeing unused
// resources or notifying other services. For example, S3Store uses this
// interface for removing a temporary object.
FinishUpload(ctx context.Context) error
}
// DataStore is the base interface for storages to implement. It provides functions
// to create new uploads and fetch existing ones.
//
// Note: the context values passed to all functions is not the request's context,
// but a similar context. See HookEvent.Context for more details.
type DataStore interface {
// Create a new upload using the size as the file's length. The method must
// return an unique id which is used to identify the upload. If no backend
// (e.g. Riak) specifes the id you may want to use the uid package to
// generate one. The properties Size and MetaData will be filled.
NewUpload(ctx context.Context, info FileInfo) (upload Upload, err error)
// GetUpload fetches the upload with a given ID. If no such upload can be found,
// ErrNotFound must be returned.
GetUpload(ctx context.Context, id string) (upload Upload, err error)
}
type TerminatableUpload interface {
// Terminate an upload so any further requests to the upload resource will
// return the ErrNotFound error.
Terminate(ctx context.Context) error
}
// TerminaterDataStore is the interface which must be implemented by DataStores
// if they want to receive DELETE requests using the Handler. If this interface
// is not implemented, no request handler for this method is attached.
type TerminaterDataStore interface {
AsTerminatableUpload(upload Upload) TerminatableUpload
}
// ConcaterDataStore is the interface required to be implemented if the
// Concatenation extension should be enabled. Only in this case, the handler
// will parse and respect the Upload-Concat header.
type ConcaterDataStore interface {
AsConcatableUpload(upload Upload) ConcatableUpload
}
type ConcatableUpload interface {
// ConcatUploads concatenates the content from the provided partial uploads
// and writes the result in the destination upload.
// The caller (usually the handler) must and will ensure that this
// destination upload has been created before with enough space to hold all
// partial uploads. The order, in which the partial uploads are supplied,
// must be respected during concatenation.
ConcatUploads(ctx context.Context, partialUploads []Upload) error
}
// LengthDeferrerDataStore is the interface that must be implemented if the
// creation-defer-length extension should be enabled. The extension enables a
// client to upload files when their total size is not yet known. Instead, the
// client must send the total size as soon as it becomes known.
type LengthDeferrerDataStore interface {
AsLengthDeclarableUpload(upload Upload) LengthDeclarableUpload
}
type LengthDeclarableUpload interface {
DeclareLength(ctx context.Context, length int64) error
}
// Locker is the interface required for custom lock persisting mechanisms.
// Common ways to store this information is in memory, on disk or using an
// external service, such as Redis.
// When multiple processes are attempting to access an upload, whether it be
// by reading or writing, a synchronization mechanism is required to prevent
// data corruption, especially to ensure correct offset values and the proper
// order of chunks inside a single upload.
type Locker interface {
// NewLock creates a new unlocked lock object for the given upload ID.
NewLock(id string) (Lock, error)
}
// Lock is the interface for a lock as returned from a Locker.
type Lock interface {
// Lock attempts to obtain an exclusive lock for the upload specified
// by its id.
// If the lock can be acquired, it will return without error. The requestUnlock
// callback is invoked when another caller attempts to create a lock. In this
// case, the holder of the lock should attempt to release the lock as soon
// as possible
// If the lock is already held, the holder's requestUnlock function will be
// invoked to request the lock to be released. If the context is cancelled before
// the lock can be acquired, ErrLockTimeout will be returned without acquiring
// the lock.
Lock(ctx context.Context, requestUnlock func()) error
// Unlock releases an existing lock for the given upload.
Unlock() error
}
type ServableUpload interface {
// ServeContent serves the uploaded data as specified by the GET request.
// It allows data stores to delegate the handling of range requests and conditional
// requests to their underlying providers.
// The tusd handler will set the Content-Type and Content-Disposition headers
// before calling ServeContent, but the implementation can override them.
// After calling ServeContent, the handler will not take any further action
// other than handling a potential error.
ServeContent(ctx context.Context, w http.ResponseWriter, r *http.Request) error
}
// ContentServerDataStore is the interface for DataStores that can serve content directly.
// When the handler serves a GET request, it will pass the request to ServeContent
// and delegate its handling to the DataStore, instead of using GetReader to obtain the content.
type ContentServerDataStore interface {
AsServableUpload(upload Upload) ServableUpload
}
+69
View File
@@ -0,0 +1,69 @@
/*
Package handler provides ways to accept tus 1.0 calls using HTTP.
tus is a protocol based on HTTP for resumable file uploads. Resumable means that
an upload can be interrupted at any moment and can be resumed without
re-uploading the previous data again. An interruption may happen willingly, if
the user wants to pause, or by accident in case of an network issue or server
outage (http://tus.io).
# The basics of tusd
tusd was designed in way which allows an flexible and customizable usage. We
wanted to avoid binding this package to a specific storage system particularly
a proprietary third-party software. Therefore tusd is an abstract layer whose
only job is to accept incoming HTTP requests, validate them according to the
specification and finally passes them to the data store.
The data store is another important component in tusd's architecture whose
purpose is to do the actual file handling. It has to write the incoming upload
to a persistent storage system and retrieve information about an upload's
current state. Therefore it is the only part of the system which communicates
directly with the underlying storage system, whether it be the local disk, a
remote FTP server or cloud providers such as AWS S3.
# Using a store composer
The only hard requirements for a data store can be found in the DataStore
interface. It contains methods for creating uploads (NewUpload), writing to
them (WriteChunk) and retrieving their status (GetInfo). However, there
are many more features which are not mandatory but may still be used.
These are contained in their own interfaces which all share the *DataStore
suffix. For example, GetReaderDataStore which enables downloading uploads or
TerminaterDataStore which allows uploads to be terminated.
The store composer offers a way to combine the basic data store - the core -
implementation and these additional extensions:
composer := tusd.NewStoreComposer()
composer.UseCore(dataStore) // Implements DataStore
composer.UseTerminater(terminater) // Implements TerminaterDataStore
composer.UseLocker(locker) // Implements LockerDataStore
The corresponding methods for adding an extension to the composer are prefixed
with Use* followed by the name of the corresponding interface. However, most
data store provide multiple extensions and adding all of them manually can be
tedious and error-prone. Therefore, all data store distributed with tusd provide
an UseIn() method which does this job automatically. For example, this is the
S3 store in action (see S3Store.UseIn):
store := s3store.New(…)
locker := memorylocker.New()
composer := tusd.NewStoreComposer()
store.UseIn(composer)
locker.UseIn(composer)
Finally, once you are done with composing your data store, you can pass it
inside the Config struct in order to create create a new tusd HTTP handler:
config := tusd.Config{
StoreComposer: composer,
BasePath: "/files/",
}
handler, err := tusd.NewHandler(config)
This handler can then be mounted to a specific path, e.g. /files:
http.Handle("/files/", http.StripPrefix("/files/", handler))
*/
package handler
+37
View File
@@ -0,0 +1,37 @@
package handler
// Error represents an error with the intent to be sent in the HTTP
// response to the client. Therefore, it also contains a HTTPResponse,
// next to an error code and error message.
type Error struct {
ErrorCode string
Message string
HTTPResponse HTTPResponse
}
func (e Error) Error() string {
return e.ErrorCode + ": " + e.Message
}
func (e1 Error) Is(target error) bool {
e2, ok := target.(Error)
return ok && e1.ErrorCode == e2.ErrorCode
}
// NewError constructs a new Error object with the given error code and message.
// The corresponding HTTP response will have the provided status code
// and a body consisting of the error details.
// responses. See the net/http package for standardized status codes.
func NewError(errCode string, message string, statusCode int) Error {
return Error{
ErrorCode: errCode,
Message: message,
HTTPResponse: HTTPResponse{
StatusCode: statusCode,
Body: errCode + ": " + message + "\n",
Header: HTTPHeader{
"Content-Type": "text/plain; charset=utf-8",
},
},
}
}
+77
View File
@@ -0,0 +1,77 @@
package handler
import (
"net/http"
"strings"
)
// Handler is a ready to use handler with routing
type Handler struct {
*UnroutedHandler
http.Handler
}
// NewHandler creates a routed tus protocol handler. This is the simplest
// way to use tusd but may not be as configurable as you require. If you are
// integrating this into an existing app you may like to use tusd.NewUnroutedHandler
// instead. Using tusd.NewUnroutedHandler allows the tus handlers to be combined into
// your existing router (aka mux) directly. It also allows the GET and DELETE
// endpoints to be customized. These are not part of the protocol so can be
// changed depending on your needs.
func NewHandler(config Config) (*Handler, error) {
if err := config.validate(); err != nil {
return nil, err
}
handler, err := NewUnroutedHandler(config)
if err != nil {
return nil, err
}
routedHandler := &Handler{
UnroutedHandler: handler,
}
mux := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
method := r.Method
path := strings.Trim(r.URL.Path, "/")
switch path {
case "":
// Root endpoint for upload creation
switch method {
case "POST":
handler.PostFile(w, r)
default:
w.Header().Add("Allow", "POST")
w.WriteHeader(http.StatusMethodNotAllowed)
w.Write([]byte(`method not allowed`))
}
default:
// URL points to an upload resource
switch {
case method == "HEAD" && r.URL.Path != "":
// Offset retrieval
handler.HeadFile(w, r)
case method == "PATCH" && r.URL.Path != "":
// Upload apppending
handler.PatchFile(w, r)
case method == "GET" && r.URL.Path != "" && !config.DisableDownload:
// Upload download
handler.GetFile(w, r)
case method == "DELETE" && r.URL.Path != "" && config.StoreComposer.UsesTerminater && !config.DisableTermination:
// Upload termination
handler.DelFile(w, r)
default:
// TODO: Only add GET and DELETE if they are supported
w.Header().Add("Allow", "GET, HEAD, PATCH, DELETE")
w.WriteHeader(http.StatusMethodNotAllowed)
w.Write([]byte(`method not allowed`))
}
}
})
routedHandler.Handler = handler.Middleware(mux)
return routedHandler, nil
}
+49
View File
@@ -0,0 +1,49 @@
package handler
import (
"context"
)
// HookEvent represents an event from tusd which can be handled by the application.
type HookEvent struct {
// Context provides access to the context from the HTTP request. This context is
// not the exact value as the request context from http.Request.Context() but
// a similar context that retains the same values as the request context. In
// addition, Context will be cancelled after a short delay when the request context
// is done. This delay is controlled by Config.GracefulRequestCompletionTimeout.
//
// The reason is that we want stores to be able to continue processing a request after
// its context has been cancelled. For example, assume a PATCH request is incoming. If
// the end-user pauses the upload, the connection is closed causing the request context
// to be cancelled immediately. However, we want the store to be able to save the last
// few bytes that were transmitted before the request was aborted. To allow this, we
// copy the request context but cancel it with a brief delay to give the data store
// time to finish its operations.
Context context.Context `json:"-"`
// Upload contains information about the upload that caused this hook
// to be fired.
Upload FileInfo
// HTTPRequest contains details about the HTTP request that reached
// tusd.
HTTPRequest HTTPRequest
}
func newHookEvent(c *httpContext, info FileInfo) HookEvent {
// The Host header field is not present in the header map, see https://pkg.go.dev/net/http#Request:
// > For incoming requests, the Host header is promoted to the
// > Request.Host field and removed from the Header map.
// That's why we add it back manually.
copiedHeader := c.req.Header.Clone()
copiedHeader.Set("Host", c.req.Host)
return HookEvent{
Context: c,
Upload: info,
HTTPRequest: HTTPRequest{
Method: c.req.Method,
URI: c.req.RequestURI,
RemoteAddr: c.req.RemoteAddr,
Header: copiedHeader,
},
}
}
+76
View File
@@ -0,0 +1,76 @@
package handler
import (
"maps"
"net/http"
"strconv"
)
// HTTPRequest contains basic details of an incoming HTTP request.
type HTTPRequest struct {
// Method is the HTTP method, e.g. POST or PATCH.
Method string
// URI is the full HTTP request URI, e.g. /files/fooo.
URI string
// RemoteAddr contains the network address that sent the request.
RemoteAddr string
// Header contains all HTTP headers as present in the HTTP request.
Header http.Header
}
type HTTPHeader map[string]string
// HTTPResponse contains basic details of an outgoing HTTP response.
type HTTPResponse struct {
// StatusCode is status code, e.g. 200 or 400.
StatusCode int
// Body is the response body.
Body string
// Header contains additional HTTP headers for the response.
Header HTTPHeader
}
// writeTo writes the HTTP response into w, as specified by the fields in resp.
func (resp HTTPResponse) writeTo(w http.ResponseWriter) {
headers := w.Header()
for key, value := range resp.Header {
headers.Set(key, value)
}
if len(resp.Body) > 0 {
headers.Set("Content-Length", strconv.Itoa(len(resp.Body)))
}
w.WriteHeader(resp.StatusCode)
if len(resp.Body) > 0 {
w.Write([]byte(resp.Body))
}
}
// MergeWith returns a copy of resp1, where non-default values from resp2 overwrite
// values from resp1.
func (resp1 HTTPResponse) MergeWith(resp2 HTTPResponse) HTTPResponse {
// Clone the response 1 and use it as a basis
newResp := resp1
// Take the status code and body from response 2 to
// overwrite values from response 1.
if resp2.StatusCode != 0 {
newResp.StatusCode = resp2.StatusCode
}
if len(resp2.Body) > 0 {
newResp.Body = resp2.Body
}
// For the headers, me must make a new map to avoid writing
// into the header map from response 1.
newResp.Header = make(HTTPHeader, len(resp1.Header)+len(resp2.Header))
maps.Copy(newResp.Header, resp1.Header)
maps.Copy(newResp.Header, resp2.Header)
return newResp
}
+131
View File
@@ -0,0 +1,131 @@
package handler
import (
"maps"
"sync"
"sync/atomic"
)
// Metrics provides numbers about the usage of the tusd handler. Since these may
// be accessed from multiple goroutines, it is necessary to read and modify them
// atomically using the functions exposed in the sync/atomic package, such as
// atomic.LoadUint64. In addition the maps must not be modified to prevent data
// races.
type Metrics struct {
// RequestTotal counts the number of incoming requests per method
RequestsTotal map[string]*uint64
// ErrorsTotal counts the number of returned errors by their message
ErrorsTotal *ErrorsTotalMap
BytesReceived *uint64
UploadsFinished *uint64
UploadsCreated *uint64
UploadsTerminated *uint64
}
// incRequestsTotal increases the counter for this request method atomically by
// one. The method must be one of GET, HEAD, POST, PATCH, DELETE.
func (m Metrics) incRequestsTotal(method string) {
if ptr, ok := m.RequestsTotal[method]; ok {
atomic.AddUint64(ptr, 1)
}
}
// incErrorsTotal increases the counter for this error atomically by one.
func (m Metrics) incErrorsTotal(err Error) {
ptr := m.ErrorsTotal.retrievePointerFor(err)
atomic.AddUint64(ptr, 1)
}
// incBytesReceived increases the number of received bytes atomically be the
// specified number.
func (m Metrics) incBytesReceived(delta uint64) {
atomic.AddUint64(m.BytesReceived, delta)
}
// incUploadsFinished increases the counter for finished uploads atomically by one.
func (m Metrics) incUploadsFinished() {
atomic.AddUint64(m.UploadsFinished, 1)
}
// incUploadsCreated increases the counter for completed uploads atomically by one.
func (m Metrics) incUploadsCreated() {
atomic.AddUint64(m.UploadsCreated, 1)
}
// incUploadsTerminated increases the counter for completed uploads atomically by one.
func (m Metrics) incUploadsTerminated() {
atomic.AddUint64(m.UploadsTerminated, 1)
}
func newMetrics() Metrics {
return Metrics{
RequestsTotal: map[string]*uint64{
"GET": new(uint64),
"HEAD": new(uint64),
"POST": new(uint64),
"PATCH": new(uint64),
"DELETE": new(uint64),
"OPTIONS": new(uint64),
},
ErrorsTotal: newErrorsTotalMap(),
BytesReceived: new(uint64),
UploadsFinished: new(uint64),
UploadsCreated: new(uint64),
UploadsTerminated: new(uint64),
}
}
// ErrorsTotalMap stores the counters for the different HTTP errors.
type ErrorsTotalMap struct {
lock sync.RWMutex
counter map[ErrorsTotalMapEntry]*uint64
}
type ErrorsTotalMapEntry struct {
ErrorCode string
StatusCode int
}
func newErrorsTotalMap() *ErrorsTotalMap {
m := make(map[ErrorsTotalMapEntry]*uint64, 20)
return &ErrorsTotalMap{
counter: m,
}
}
// retrievePointerFor returns (after creating it if necessary) the pointer to
// the counter for the error.
func (e *ErrorsTotalMap) retrievePointerFor(err Error) *uint64 {
serr := ErrorsTotalMapEntry{
ErrorCode: err.ErrorCode,
StatusCode: err.HTTPResponse.StatusCode,
}
e.lock.RLock()
ptr, ok := e.counter[serr]
e.lock.RUnlock()
if ok {
return ptr
}
// For pointer creation, a write-lock is required
e.lock.Lock()
// We ensure that the pointer wasn't created in the meantime
if ptr, ok = e.counter[serr]; !ok {
ptr = new(uint64)
e.counter[serr] = ptr
}
e.lock.Unlock()
return ptr
}
// Load retrieves the map of the counter pointers atomically
func (e *ErrorsTotalMap) Load() map[ErrorsTotalMapEntry]*uint64 {
m := make(map[ErrorsTotalMapEntry]*uint64, len(e.counter))
e.lock.RLock()
maps.Copy(m, e.counter)
e.lock.RUnlock()
return m
}
File diff suppressed because it is too large Load Diff