Bump reva

This commit is contained in:
André Duffeck
2025-03-13 08:51:41 +01:00
parent 747df432d0
commit b6db5f7677
80 changed files with 1507 additions and 2685 deletions
+7
View File
@@ -14,6 +14,8 @@ type StoreComposer struct {
Concater ConcaterDataStore
UsesLengthDeferrer bool
LengthDeferrer LengthDeferrerDataStore
ContentServer ContentServerDataStore
UsesContentServer bool
}
// NewStoreComposer creates a new and empty store composer.
@@ -85,3 +87,8 @@ func (store *StoreComposer) UseLengthDeferrer(ext LengthDeferrerDataStore) {
store.UsesLengthDeferrer = ext != nil
store.LengthDeferrer = ext
}
func (store *StoreComposer) UseContentServer(ext ContentServerDataStore) {
store.UsesContentServer = ext != nil
store.ContentServer = ext
}
+7
View File
@@ -75,6 +75,13 @@ type Config struct {
// If the error is non-nil, the error will be forwarded to the client. Furthermore,
// HTTPResponse will be ignored and the error value can contain values for the HTTP response.
PreFinishResponseCallback func(hook HookEvent) (HTTPResponse, error)
// PreUploadTerminateCallback will be invoked on DELETE requests before an upload is terminated,
// giving the application the opportunity to reject the termination. For example, to ensure resources
// used by other services are not deleted.
// If the callback returns no error, optional values from HTTPResponse will be contained in the HTTP response.
// If the error is non-nil, the error will be forwarded to the client. Furthermore,
// HTTPResponse will be ignored and the error value can contain values for the HTTP response.
PreUploadTerminateCallback func(hook HookEvent) (HTTPResponse, error)
// GracefulRequestCompletionTimeout is the timeout for operations to complete after an HTTP
// request has ended (successfully or by error). For example, if an HTTP request is interrupted,
// instead of stopping immediately, the handler and data store will be given some additional
+19
View File
@@ -3,6 +3,7 @@ package handler
import (
"context"
"io"
"net/http"
)
type MetaData map[string]string
@@ -191,3 +192,21 @@ type Lock interface {
// Unlock releases an existing lock for the given upload.
Unlock() error
}
type ServableUpload interface {
// ServeContent serves the uploaded data as specified by the GET request.
// It allows data stores to delegate the handling of range requests and conditional
// requests to their underlying providers.
// The tusd handler will set the Content-Type and Content-Disposition headers
// before calling ServeContent, but the implementation can override them.
// After calling ServeContent, the handler will not take any further action
// other than handling a potential error.
ServeContent(ctx context.Context, w http.ResponseWriter, r *http.Request) error
}
// ContentServerDataStore is the interface for DataStores that can serve content directly.
// When the handler serves a GET request, it will pass the request to ServeContent
// and delegate its handling to the DataStore, instead of using GetReader to obtain the content.
type ContentServerDataStore interface {
AsServableUpload(upload Upload) ServableUpload
}
+88 -20
View File
@@ -60,6 +60,7 @@ var (
ErrInvalidUploadDeferLength = NewError("ERR_INVALID_UPLOAD_LENGTH_DEFER", "invalid Upload-Defer-Length header", http.StatusBadRequest)
ErrUploadStoppedByServer = NewError("ERR_UPLOAD_STOPPED", "upload has been stopped by server", http.StatusBadRequest)
ErrUploadRejectedByServer = NewError("ERR_UPLOAD_REJECTED", "upload creation has been rejected by server", http.StatusBadRequest)
ErrUploadTerminationRejected = NewError("ERR_UPLOAD_TERMINATION_REJECTED", "upload termination has been rejected by server", http.StatusBadRequest)
ErrUploadInterrupted = NewError("ERR_UPLOAD_INTERRUPTED", "upload has been interrupted by another request for this upload resource", http.StatusBadRequest)
ErrServerShutdown = NewError("ERR_SERVER_SHUTDOWN", "request has been interrupted because the server is shutting down", http.StatusServiceUnavailable)
ErrOriginNotAllowed = NewError("ERR_ORIGIN_NOT_ALLOWED", "request origin is not allowed", http.StatusForbidden)
@@ -177,10 +178,10 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
// We also update the write deadline, but makes sure that it is larger than the read deadline, so we
// can still write a response in the case of a read timeout.
if err := c.resC.SetReadDeadline(time.Now().Add(handler.config.NetworkTimeout)); err != nil {
c.log.Warn("NetworkControlError", "error", err)
c.log.WarnContext(c, "NetworkControlError", "error", err)
}
if err := c.resC.SetWriteDeadline(time.Now().Add(2 * handler.config.NetworkTimeout)); err != nil {
c.log.Warn("NetworkControlError", "error", err)
c.log.WarnContext(c, "NetworkControlError", "error", err)
}
// Allow overriding the HTTP method. The reason for this is
@@ -190,7 +191,7 @@ func (handler *UnroutedHandler) Middleware(h http.Handler) http.Handler {
r.Method = newMethod
}
c.log.Info("RequestIncoming")
c.log.InfoContext(c, "RequestIncoming")
handler.Metrics.incRequestsTotal(r.Method)
@@ -405,7 +406,7 @@ func (handler *UnroutedHandler) PostFile(w http.ResponseWriter, r *http.Request)
handler.Metrics.incUploadsCreated()
c.log = c.log.With("id", id)
c.log.Info("UploadCreated", "size", size, "url", url)
c.log.InfoContext(c, "UploadCreated", "size", size, "url", url)
if handler.config.NotifyCreatedUploads {
handler.CreatedUploads <- newHookEvent(c, info)
@@ -572,7 +573,7 @@ func (handler *UnroutedHandler) PostFileV2(w http.ResponseWriter, r *http.Reques
handler.Metrics.incUploadsCreated()
c.log = c.log.With("id", id)
c.log.Info("UploadCreated", "size", info.Size, "url", url)
c.log.InfoContext(c, "UploadCreated", "size", info.Size, "url", url)
if handler.config.NotifyCreatedUploads {
handler.CreatedUploads <- newHookEvent(c, info)
@@ -891,7 +892,7 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
maxSize = length
}
c.log.Info("ChunkWriteStart", "maxSize", maxSize, "offset", offset)
c.log.InfoContext(c, "ChunkWriteStart", "maxSize", maxSize, "offset", offset)
var bytesWritten int64
var err error
@@ -907,12 +908,12 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
// Update the read deadline for every successful read operation. This ensures that the request handler
// keeps going while data is transmitted but that dead connections can also time out and be cleaned up.
if err := c.resC.SetReadDeadline(time.Now().Add(handler.config.NetworkTimeout)); err != nil {
c.log.Warn("NetworkTimeoutError", "error", err)
c.log.WarnContext(c, "NetworkTimeoutError", "error", err)
}
// The write deadline is updated accordingly to ensure that we can also write responses.
if err := c.resC.SetWriteDeadline(time.Now().Add(2 * handler.config.NetworkTimeout)); err != nil {
c.log.Warn("NetworkTimeoutError", "error", err)
c.log.WarnContext(c, "NetworkTimeoutError", "error", err)
}
}
@@ -935,7 +936,7 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
// it in the response, if the store did not also return an error.
bodyErr := c.body.hasError()
if bodyErr != nil {
c.log.Error("BodyReadError", "error", bodyErr.Error())
c.log.ErrorContext(c, "BodyReadError", "error", bodyErr.Error())
if err == nil {
err = bodyErr
}
@@ -947,12 +948,12 @@ func (handler *UnroutedHandler) writeChunk(c *httpContext, resp HTTPResponse, up
if terminateErr := handler.terminateUpload(c, upload, info); terminateErr != nil {
// We only log this error and not show it to the user since this
// termination error is not relevant to the uploading client
c.log.Error("UploadStopTerminateError", "error", terminateErr.Error())
c.log.ErrorContext(c, "UploadStopTerminateError", "error", terminateErr.Error())
}
}
}
c.log.Info("ChunkWriteComplete", "bytesWritten", bytesWritten)
c.log.InfoContext(c, "ChunkWriteComplete", "bytesWritten", bytesWritten)
// Send new offset to client
newOffset := offset + bytesWritten
@@ -1003,7 +1004,7 @@ func (handler *UnroutedHandler) emitFinishEvents(c *httpContext, resp HTTPRespon
resp = resp.MergeWith(resp2)
}
c.log.Info("UploadFinished", "size", info.Size)
c.log.InfoContext(c, "UploadFinished", "size", info.Size)
handler.Metrics.incUploadsFinished()
if handler.config.NotifyCompleteUploads {
@@ -1047,6 +1048,7 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request)
return
}
// Fall back to the existing GetReader implementation if ContentServerDataStore is not implemented
contentType, contentDisposition := filterContentType(info)
resp := HTTPResponse{
StatusCode: http.StatusOK,
@@ -1058,6 +1060,27 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request)
Body: "", // Body is intentionally left empty, and we copy it manually in later.
}
// If the data store implements ContentServerDataStore, use delegate the handling
// of GET requests to the data store.
// Otherwise, we will use the existing GetReader implementation.
if handler.composer.UsesContentServer {
servableUpload := handler.composer.ContentServer.AsServableUpload(upload)
// Pass file type and name to the implementation, but it may override them.
w.Header().Set("Content-Type", resp.Header["Content-Type"])
w.Header().Set("Content-Disposition", resp.Header["Content-Disposition"])
// Use loggingResponseWriter to get the ResponseOutgoing log entry that
// normally handler.sendResp would produce.
loggingW := &loggingResponseWriter{ResponseWriter: w, logger: c.log}
err = servableUpload.ServeContent(c, loggingW, r)
if err != nil {
handler.sendError(c, err)
}
return
}
// If no data has been uploaded yet, respond with an empty "204 No Content" status.
if info.Offset == 0 {
resp.StatusCode = http.StatusNoContent
@@ -1065,6 +1088,15 @@ func (handler *UnroutedHandler) GetFile(w http.ResponseWriter, r *http.Request)
return
}
if handler.composer.UsesContentServer {
servableUpload := handler.composer.ContentServer.AsServableUpload(upload)
err = servableUpload.ServeContent(c, w, r)
if err != nil {
handler.sendError(c, err)
}
return
}
src, err := upload.GetReader(c)
if err != nil {
handler.sendError(c, err)
@@ -1172,7 +1204,7 @@ func (handler *UnroutedHandler) DelFile(w http.ResponseWriter, r *http.Request)
}
var info FileInfo
if handler.config.NotifyTerminatedUploads {
if handler.config.NotifyTerminatedUploads || handler.config.PreUploadTerminateCallback != nil {
info, err = upload.GetInfo(c)
if err != nil {
handler.sendError(c, err)
@@ -1180,15 +1212,26 @@ func (handler *UnroutedHandler) DelFile(w http.ResponseWriter, r *http.Request)
}
}
resp := HTTPResponse{
StatusCode: http.StatusNoContent,
}
if handler.config.PreUploadTerminateCallback != nil {
resp2, err := handler.config.PreUploadTerminateCallback(newHookEvent(c, info))
if err != nil {
handler.sendError(c, err)
return
}
resp = resp.MergeWith(resp2)
}
err = handler.terminateUpload(c, upload, info)
if err != nil {
handler.sendError(c, err)
return
}
handler.sendResp(c, HTTPResponse{
StatusCode: http.StatusNoContent,
})
handler.sendResp(c, resp)
}
// terminateUpload passes a given upload to the DataStore's Terminater,
@@ -1208,7 +1251,7 @@ func (handler *UnroutedHandler) terminateUpload(c *httpContext, upload Upload, i
handler.TerminatedUploads <- newHookEvent(c, info)
}
c.log.Info("UploadTerminated")
c.log.InfoContext(c, "UploadTerminated")
handler.Metrics.incUploadsTerminated()
return nil
@@ -1222,7 +1265,7 @@ func (handler *UnroutedHandler) sendError(c *httpContext, err error) {
var detailedErr Error
if !errors.As(err, &detailedErr) {
c.log.Error("InternalServerError", "message", err.Error())
c.log.ErrorContext(c, "InternalServerError", "message", err.Error())
detailedErr = NewError("ERR_INTERNAL_SERVER_ERROR", err.Error(), http.StatusInternalServerError)
}
@@ -1240,7 +1283,7 @@ func (handler *UnroutedHandler) sendError(c *httpContext, err error) {
func (handler *UnroutedHandler) sendResp(c *httpContext, resp HTTPResponse) {
resp.writeTo(c.res)
c.log.Info("ResponseOutgoing", "status", resp.StatusCode, "body", resp.Body)
c.log.InfoContext(c, "ResponseOutgoing", "status", resp.StatusCode, "body", resp.Body)
}
// Make an absolute URLs to the given upload id. If the base path is absolute
@@ -1323,6 +1366,14 @@ func getHostAndProtocol(r *http.Request, allowForwarded bool) (host, proto strin
}
}
// Remove default ports
if proto == "http" {
host = strings.TrimSuffix(host, ":80")
}
if proto == "https" {
host = strings.TrimSuffix(host, ":443")
}
return
}
@@ -1393,7 +1444,7 @@ func (handler *UnroutedHandler) lockUpload(c *httpContext, id string) (Lock, err
// No need to wrap this in a sync.OnceFunc because c.cancel will be a noop after the first call.
releaseLock := func() {
c.log.Info("UploadInterrupted")
c.log.InfoContext(c, "UploadInterrupted")
c.cancel(ErrUploadInterrupted)
}
@@ -1671,3 +1722,20 @@ func validateUploadId(newId string) error {
return nil
}
// loggingResponseWriter is a wrapper around http.ResponseWriter that logs the
// final status code similar to UnroutedHandler.sendResp.
type loggingResponseWriter struct {
http.ResponseWriter
logger *slog.Logger
}
func (w *loggingResponseWriter) WriteHeader(statusCode int) {
if statusCode >= 200 {
w.logger.Info("ResponseOutgoing", "status", statusCode)
}
w.ResponseWriter.WriteHeader(statusCode)
}
// Unwrap provides access to the underlying http.ResponseWriter.
func (w *loggingResponseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter }