Initial QSfera import

Server/services/thumbnails/pkg/service/grpc/v0/decorators/base.go

@@ -0,0 +1,63 @@
+package decorators
+
+import (
+	"context"
+
+	thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
+)
+
+// DecoratedService is an interface acting as facade, holding all the interfaces that this
+// thumbnails microservice is expecting to implement.
+// For now, only the thumbnailssvc.ThumbnailServiceHandler is present,
+// but a future configsvc.ConfigServiceHandler is expected to be added here
+//
+// This interface will also act as the base interface to implement
+// a decorator pattern.
+type DecoratedService interface {
+	thumbnailssvc.ThumbnailServiceHandler
+}
+
+// Decorator is the base type to implement the decorators. It will provide a basic implementation
+// by delegating to the decoratedService
+//
+// Expected implementations will be like:
+// ```
+//
+//	type MyDecorator struct {
+//	  Decorator
+//	  myCustomOpts *opts
+//	  additionalSrv *srv
+//	}
+//
+//	func NewMyDecorator(next DecoratedService, customOpts *customOpts) DecoratedService {
+//	  .....
+//	  return MyDecorator{
+//	    Decorator: Decorator{next: next},
+//	    myCustomOpts: opts,
+//	    additionalSrv: srv,
+//	  }
+//	}
+//
+// ```
+type Decorator struct {
+	next DecoratedService
+}
+
+// GetThumbnail is the base implementation for the thumbnailssvc.GetThumbnail.
+// It will just delegate to the underlying decoratedService
+//
+// Your custom decorator is expected to overwrite this function,
+// but it MUST call the underlying decoratedService at some point
+// ```
+//
+//	func (d MyDecorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
+//	  doSomething()
+//	  err := d.next.GetThumbnail(ctx, req, resp)
+//	  doAnotherThing()
+//	  return err
+//	}
+//
+// ```
+func (deco Decorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
+	return deco.next.GetThumbnail(ctx, req, resp)
+}

Server/services/thumbnails/pkg/service/grpc/v0/decorators/instrument.go

@@ -0,0 +1,39 @@
+package decorators
+
+import (
+	"context"
+
+	thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
+	"github.com/qsfera/server/services/thumbnails/pkg/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// NewInstrument returns a service that instruments metrics.
+func NewInstrument(next DecoratedService, metrics *metrics.Metrics) DecoratedService {
+	return instrument{
+		Decorator: Decorator{next: next},
+		metrics:   metrics,
+	}
+}
+
+type instrument struct {
+	Decorator
+	metrics *metrics.Metrics
+}
+
+// GetThumbnail implements the ThumbnailServiceHandler interface.
+func (i instrument) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
+	timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
+		us := v * 1000_000
+		i.metrics.Latency.WithLabelValues().Observe(us)
+		i.metrics.Duration.WithLabelValues().Observe(v)
+	}))
+	defer timer.ObserveDuration()
+
+	err := i.next.GetThumbnail(ctx, req, rsp)
+
+	if err != nil {
+		i.metrics.Counter.WithLabelValues().Inc()
+	}
+	return err
+}

Server/services/thumbnails/pkg/service/grpc/v0/decorators/logging.go

@@ -0,0 +1,53 @@
+package decorators
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/qsfera/server/pkg/log"
+	thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
+	merrors "go-micro.dev/v4/errors"
+)
+
+// NewLogging returns a service that logs messages.
+func NewLogging(next DecoratedService, logger log.Logger) DecoratedService {
+	return logging{
+		Decorator: Decorator{next: next},
+		logger:    logger,
+	}
+}
+
+type logging struct {
+	Decorator
+	logger log.Logger
+}
+
+// GetThumbnail implements the ThumbnailServiceHandler interface.
+func (l logging) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
+	start := time.Now()
+	err := l.next.GetThumbnail(ctx, req, rsp)
+
+	logger := l.logger.With().
+		Str("method", "Thumbnails.GetThumbnail").
+		Dur("duration", time.Since(start)).
+		Logger()
+
+	if err != nil {
+		fromError := merrors.FromError(err)
+		switch fromError.GetCode() {
+		case http.StatusNotFound:
+			logger.Debug().
+				Str("error_detail", fromError.GetDetail()).
+				Msg("no thumbnail found")
+		default:
+			logger.Warn().
+				Err(err).
+				Msg("Failed to execute")
+		}
+	} else {
+		logger.Debug().
+			Msg("")
+	}
+	return err
+}

Server/services/thumbnails/pkg/service/grpc/v0/decorators/tracing.go

@@ -0,0 +1,46 @@
+package decorators
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/trace"
+
+	thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+// NewTracing returns a service that instruments traces.
+func NewTracing(next DecoratedService, tp trace.TracerProvider) DecoratedService {
+	return tracing{
+		Decorator: Decorator{next: next},
+		tp:        tp,
+	}
+}
+
+type tracing struct {
+	Decorator
+	tp trace.TracerProvider
+}
+
+// GetThumbnail implements the ThumbnailServiceHandler interface.
+func (t tracing) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
+	var span trace.Span
+
+	if t.tp != nil {
+		tracer := t.tp.Tracer("thumbnails")
+		spanOpts := []trace.SpanStartOption{
+			trace.WithSpanKind(trace.SpanKindServer),
+		}
+		ctx, span = tracer.Start(ctx, "Thumbnails.GetThumbnail", spanOpts...)
+		defer span.End()
+
+		span.SetAttributes(
+			attribute.KeyValue{Key: "filepath", Value: attribute.StringValue(req.GetFilepath())},
+			attribute.KeyValue{Key: "thumbnail_type", Value: attribute.StringValue(req.GetThumbnailType().String())},
+			attribute.KeyValue{Key: "width", Value: attribute.IntValue(int(req.GetWidth()))},
+			attribute.KeyValue{Key: "height", Value: attribute.IntValue(int(req.GetHeight()))},
+		)
+	}
+
+	return t.next.GetThumbnail(ctx, req, rsp)
+}

Server/services/thumbnails/pkg/service/grpc/v0/option.go

@@ -0,0 +1,79 @@
+package svc
+
+import (
+	"net/http"
+
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/services/thumbnails/pkg/config"
+	"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
+	"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
+	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
+)
+
+// Option defines a single option function.
+type Option func(o *Options)
+
+// Options defines the available options for this package.
+type Options struct {
+	Logger           log.Logger
+	Config           *config.Config
+	Middleware       []func(http.Handler) http.Handler
+	ThumbnailStorage storage.Storage
+	ImageSource      imgsource.Source
+	CS3Source        imgsource.Source
+	GatewaySelector  pool.Selectable[gateway.GatewayAPIClient]
+}
+
+// newOptions initializes the available default options.
+func newOptions(opts ...Option) Options {
+	opt := Options{}
+
+	for _, o := range opts {
+		o(&opt)
+	}
+
+	return opt
+}
+
+// Logger provides a function to set the logger option.
+func Logger(val log.Logger) Option {
+	return func(o *Options) {
+		o.Logger = val
+	}
+}
+
+// Config provides a function to set the config option.
+func Config(val *config.Config) Option {
+	return func(o *Options) {
+		o.Config = val
+	}
+}
+
+// ThumbnailStorage provides a function to set the thumbnail storage option.
+func ThumbnailStorage(val storage.Storage) Option {
+	return func(o *Options) {
+		o.ThumbnailStorage = val
+	}
+}
+
+// ThumbnailSource provides a function to set the image source option.
+func ThumbnailSource(val imgsource.Source) Option {
+	return func(o *Options) {
+		o.ImageSource = val
+	}
+}
+
+// CS3Source provides a function to set the CS3Source option
+func CS3Source(val imgsource.Source) Option {
+	return func(o *Options) {
+		o.CS3Source = val
+	}
+}
+
+// GatewaySelector adds a grpc client selector for the gateway service
+func GatewaySelector(val pool.Selectable[gateway.GatewayAPIClient]) Option {
+	return func(o *Options) {
+		o.GatewaySelector = val
+	}
+}

Server/services/thumbnails/pkg/service/grpc/v0/service.go

@@ -0,0 +1,331 @@
+package svc
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/pkg/errors"
+	merrors "go-micro.dev/v4/errors"
+	"google.golang.org/grpc/metadata"
+
+	revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
+	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
+	"github.com/opencloud-eu/reva/v2/pkg/storagespace"
+	"github.com/opencloud-eu/reva/v2/pkg/utils"
+
+	"github.com/qsfera/server/pkg/log"
+	thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
+	terrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
+	"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
+	"github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0/decorators"
+	tjwt "github.com/qsfera/server/services/thumbnails/pkg/service/jwt"
+	"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
+	"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
+)
+
+// NewService returns a service implementation for Service.
+func NewService(opts ...Option) decorators.DecoratedService {
+	options := newOptions(opts...)
+	logger := options.Logger
+	resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions)
+	if err != nil {
+		logger.Fatal().Err(err).Msg("resolutions not configured correctly")
+	}
+	svc := Thumbnail{
+		serviceID: options.Config.GRPC.Namespace + "." + options.Config.Service.Name,
+		manager: thumbnail.NewSimpleManager(
+			resolutions,
+			options.ThumbnailStorage,
+			logger,
+			options.Config.Thumbnail.MaxInputWidth,
+			options.Config.Thumbnail.MaxInputHeight,
+		),
+		webdavSource: options.ImageSource,
+		cs3Source:    options.CS3Source,
+		logger:       logger,
+		selector:     options.GatewaySelector,
+		preprocessorOpts: PreprocessorOpts{
+			TxtFontFileMap: options.Config.Thumbnail.FontMapFile,
+		},
+		dataEndpoint:   options.Config.Thumbnail.DataEndpoint,
+		transferSecret: options.Config.Thumbnail.TransferSecret,
+	}
+
+	return svc
+}
+
+// Thumbnail implements the GRPC handler.
+type Thumbnail struct {
+	serviceID        string
+	dataEndpoint     string
+	transferSecret   string
+	manager          thumbnail.Manager
+	webdavSource     imgsource.Source
+	cs3Source        imgsource.Source
+	logger           log.Logger
+	selector         pool.Selectable[gateway.GatewayAPIClient]
+	preprocessorOpts PreprocessorOpts
+}
+
+// PreprocessorOpts holds the options for the preprocessor
+type PreprocessorOpts struct {
+	TxtFontFileMap string
+}
+
+// GetThumbnail retrieves a thumbnail for an image
+func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
+	var err error
+	var key string
+	switch {
+	case req.GetWebdavSource() != nil:
+		key, err = g.handleWebdavSource(ctx, req)
+	case req.GetCs3Source() != nil:
+		key, err = g.handleCS3Source(ctx, req)
+	default:
+		g.logger.Error().Msg("no image source provided")
+		return merrors.BadRequest(g.serviceID, "image source is missing")
+	}
+	if err != nil {
+		return err
+	}
+
+	claims := tjwt.ThumbnailClaims{
+		Key: key,
+		RegisteredClaims: jwt.RegisteredClaims{
+			IssuedAt:  jwt.NewNumericDate(time.Now()),
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Minute)),
+		},
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	transferToken, err := token.SignedString([]byte(g.transferSecret))
+	if err != nil {
+		g.logger.Error().
+			Err(err).
+			Msg("GetThumbnail: failed to sign token")
+		return merrors.InternalServerError(g.serviceID, "couldn't finish request")
+	}
+	rsp.DataEndpoint = g.dataEndpoint
+	rsp.TransferToken = transferToken
+	return nil
+}
+
+func (g Thumbnail) checkThumbnail(req *thumbnailssvc.GetThumbnailRequest, sRes *provider.StatResponse) (string, thumbnail.Request, error) {
+	tr := thumbnail.Request{}
+	if !sRes.GetInfo().GetPermissionSet().GetInitiateFileDownload() {
+		return "", tr, merrors.Forbidden(g.serviceID, "no download permission")
+	}
+
+	tType := thumbnail.GetExtForMime(sRes.GetInfo().GetMimeType())
+	if tType == "" {
+		tType = req.GetThumbnailType().String()
+	}
+	tr, err := thumbnail.PrepareRequest(int(req.GetWidth()), int(req.GetHeight()), tType, sRes.GetInfo().GetChecksum().GetSum(), req.GetProcessor())
+	if err != nil {
+		return "", tr, merrors.BadRequest(g.serviceID, "%s", err.Error())
+	}
+
+	if key, exists := g.manager.CheckThumbnail(tr); exists {
+		return key, tr, nil
+	}
+	return "", tr, nil
+}
+
+func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
+	src := req.GetCs3Source()
+	sRes, err := g.stat(src.GetPath(), src.GetAuthorization())
+	if err != nil {
+		return "", err
+	}
+
+	key, tr, err := g.checkThumbnail(req, sRes)
+	switch {
+	case err != nil:
+		return "", err
+	case key != "":
+		// we have matching thumbnail already, use that
+		return key, nil
+	}
+
+	ctx = imgsource.ContextSetAuthorization(ctx, src.GetAuthorization())
+	r, err := g.cs3Source.Get(ctx, src.GetPath())
+	switch {
+	case errors.Is(err, terrors.ErrImageTooLarge):
+		return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
+	case err != nil:
+		return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
+	}
+
+	defer r.Close()
+	ppOpts := map[string]any{
+		"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
+	}
+	pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
+	img, err := pp.Convert(r)
+	if err != nil {
+		g.logger.Error().Err(err).Msg("failed to convert image")
+	}
+
+	if img == nil || err != nil {
+		return "", merrors.NotFound(g.serviceID, "could not get image")
+	}
+
+	key, err = g.manager.Generate(tr, img)
+	if errors.Is(err, terrors.ErrImageTooLarge) {
+		return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
+	}
+	return key, err
+}
+
+func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
+	src := req.GetWebdavSource()
+	imgURL, err := url.Parse(src.GetUrl())
+	if err != nil {
+		return "", errors.Wrap(err, "source url is invalid")
+	}
+
+	var auth, statPath string
+	if src.GetIsPublicLink() {
+		q := imgURL.Query()
+		var rsp *gateway.AuthenticateResponse
+		client, err := g.selector.Next()
+		if err != nil {
+			return "", merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
+		}
+		if q.Get("signature") != "" && q.Get("expiration") != "" {
+			// Handle pre-signed public links
+			sig := q.Get("signature")
+			exp := q.Get("expiration")
+			rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
+				Type:         "publicshares",
+				ClientId:     src.GetPublicLinkToken(),
+				ClientSecret: strings.Join([]string{"signature", sig, exp}, "|"),
+			})
+		} else {
+			rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
+				Type:     "publicshares",
+				ClientId: src.GetPublicLinkToken(),
+				// We pass an empty password because we expect non pre-signed public links
+				// to not be password protected
+				ClientSecret: "password|",
+			})
+		}
+
+		if err != nil {
+			return "", merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error())
+		}
+		auth = rsp.GetToken()
+		statPath = path.Join("/public", src.GetPublicLinkToken(), req.GetFilepath())
+	} else {
+		auth = src.GetRevaAuthorization()
+		statPath = req.GetFilepath()
+	}
+	sRes, err := g.stat(statPath, auth)
+	if err != nil {
+		return "", err
+	}
+
+	key, tr, err := g.checkThumbnail(req, sRes)
+	switch {
+	case err != nil:
+		return "", err
+	case key != "":
+		// we have matching thumbnail already, use that
+		return key, nil
+	}
+
+	if src.GetWebdavAuthorization() != "" {
+		ctx = imgsource.ContextSetAuthorization(ctx, src.GetWebdavAuthorization())
+	}
+
+	// add signature and expiration to webdav url
+	signature, expiration := imgURL.Query().Get("signature"), imgURL.Query().Get("expiration")
+	params := url.Values{}
+	params.Add("signature", signature)
+	params.Add("expiration", expiration)
+	imgURL.RawQuery = params.Encode()
+
+	r, err := g.webdavSource.Get(ctx, imgURL.String())
+	switch {
+	case errors.Is(err, terrors.ErrImageTooLarge):
+		return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
+	case err != nil:
+		return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
+	}
+	defer r.Close()
+	ppOpts := map[string]any{
+		"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
+	}
+	pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
+	img, err := pp.Convert(r)
+	if img == nil || err != nil {
+		return "", merrors.NotFound(g.serviceID, "could not get image")
+	}
+
+	key, err = g.manager.Generate(tr, img)
+	if errors.Is(err, terrors.ErrImageTooLarge) {
+		return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
+	}
+	return key, err
+}
+
+func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
+	ctx := metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
+
+	ref, err := storagespace.ParseReference(path)
+	if err != nil {
+		// If the path is not a spaces reference try to handle it like a plain
+		// path reference.
+		ref = provider.Reference{
+			Path: path,
+		}
+	}
+
+	client, err := g.selector.Next()
+	if err != nil {
+		return nil, merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
+	}
+	req := &provider.StatRequest{Ref: &ref}
+	rsp, err := client.Stat(ctx, req)
+	if err != nil {
+		g.logger.Error().Err(err).Str("path", path).Msg("could not stat file")
+		return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", err.Error())
+	}
+
+	if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
+		switch rsp.GetStatus().GetCode() {
+		case rpc.Code_CODE_NOT_FOUND:
+			return nil, merrors.NotFound(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
+		default:
+			g.logger.Error().Str("status_message", rsp.GetStatus().GetMessage()).Str("path", path).Msg("could not stat file")
+			return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
+		}
+	}
+	if rsp.GetInfo().GetType() != provider.ResourceType_RESOURCE_TYPE_FILE {
+		return nil, merrors.BadRequest(g.serviceID, "Unsupported file type")
+	}
+	if utils.ReadPlainFromOpaque(rsp.GetInfo().GetOpaque(), "status") == "processing" {
+		return nil, &merrors.Error{
+			Id:     g.serviceID,
+			Code:   http.StatusTooEarly,
+			Detail: "File Processing",
+			Status: http.StatusText(http.StatusTooEarly),
+		}
+	}
+	if rsp.GetInfo().GetChecksum().GetSum() == "" {
+		g.logger.Error().Msg("resource info is missing checksum")
+		return nil, merrors.NotFound(g.serviceID, "resource info is missing a checksum")
+	}
+	if !thumbnail.IsMimeTypeSupported(rsp.GetInfo().GetMimeType()) {
+		return nil, merrors.NotFound(g.serviceID, "Unsupported file type")
+	}
+	return rsp, nil
+}