Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,81 @@
package thumbnail
import (
"image/gif"
"io"
"strings"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
const (
typePng = "png"
typeJpg = "jpg"
typeJpeg = "jpeg"
typeGif = "gif"
typeGgs = "ggs"
typeGgp = "ggp"
typeWebp = "webp"
)
// Encoder encodes the thumbnail to a specific format.
type Encoder interface {
// Encode encodes the image to a format.
Encode(w io.Writer, img any) error
// Types returns the formats suffixes.
Types() []string
// MimeType returns the mimetype used by the encoder.
MimeType() string
}
// GifEncoder encodes to gif
type GifEncoder struct{}
// Encode encodes the image to a gif format
func (e GifEncoder) Encode(w io.Writer, img any) error {
g, ok := img.(*gif.GIF)
if !ok {
return errors.ErrInvalidType
}
return gif.EncodeAll(w, g)
}
// Types returns the supported types of the GifEncoder
func (e GifEncoder) Types() []string {
return []string{typeGif}
}
// MimeType returns the mimetype used by the encoder.
func (e GifEncoder) MimeType() string {
return "image/gif"
}
// EncoderForType returns the encoder for a given file type
// or nil if the type is not supported.
func EncoderForType(fileType string) (Encoder, error) {
switch strings.ToLower(fileType) {
case typePng, typeGgs, typeGgp:
return PngEncoder{}, nil
case typeJpg, typeJpeg, typeWebp:
return JpegEncoder{}, nil
case typeGif:
return GifEncoder{}, nil
default:
return nil, errors.ErrNoEncoderForType
}
}
// GetExtForMime return the supported extension by mime
func GetExtForMime(fileType string) string {
ext := strings.TrimPrefix(strings.TrimSpace(strings.ToLower(fileType)), "image/")
switch ext {
case typeJpg, typeJpeg, typePng, typeGif, typeWebp:
return ext
case "application/vnd.geogebra.slides":
return typeGgs
case "application/vnd.geogebra.pinboard":
return typeGgp
default:
return ""
}
}
@@ -0,0 +1,56 @@
//go:build !enable_vips
package thumbnail
import (
"image"
"image/jpeg"
"image/png"
"io"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// PngEncoder encodes to png
type PngEncoder struct{}
// Encode encodes to png format
func (e PngEncoder) Encode(w io.Writer, img any) error {
m, ok := img.(image.Image)
if !ok {
return errors.ErrInvalidType
}
return png.Encode(w, m)
}
// Types returns the png suffix
func (e PngEncoder) Types() []string {
return []string{typePng}
}
// MimeType returns the mimetype for png files.
func (e PngEncoder) MimeType() string {
return "image/png"
}
// JpegEncoder encodes to jpg
type JpegEncoder struct{}
// Encode encodes to jpg
func (e JpegEncoder) Encode(w io.Writer, img any) error {
m, ok := img.(image.Image)
if !ok {
return errors.ErrInvalidType
}
return jpeg.Encode(w, m, nil)
}
// Types returns the jpg suffixes.
func (e JpegEncoder) Types() []string {
return []string{typeJpeg, typeJpg}
}
// MimeType returns the mimetype for jpg files.
func (e JpegEncoder) MimeType() string {
return "image/jpeg"
}
@@ -0,0 +1,22 @@
package thumbnail
import "testing"
func TestEncoderForType(t *testing.T) {
table := map[string]Encoder{
"jpg": JpegEncoder{},
"JPG": JpegEncoder{},
"jpeg": JpegEncoder{},
"JPEG": JpegEncoder{},
"png": PngEncoder{},
"PNG": PngEncoder{},
"invalid": nil,
}
for k, v := range table {
e, _ := EncoderForType(k)
if e != v {
t.Fail()
}
}
}
@@ -0,0 +1,66 @@
//go:build enable_vips
package thumbnail
import (
"io"
"github.com/davidbyttow/govips/v2/vips"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// PngEncoder encodes to png
type PngEncoder struct{}
// Encode encodes to png format
func (e PngEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(*vips.ImageRef)
if !ok {
return errors.ErrInvalidType
}
buf, _, err := m.ExportPng(vips.NewPngExportParams())
if err != nil {
return err
}
_, err = w.Write(buf)
return err
}
// Types returns the png suffix
func (e PngEncoder) Types() []string {
return []string{typePng}
}
// MimeType returns the mimetype for png files.
func (e PngEncoder) MimeType() string {
return "image/png"
}
// JpegEncoder encodes to jpg
type JpegEncoder struct{}
// Encode encodes to jpg
func (e JpegEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(*vips.ImageRef)
if !ok {
return errors.ErrInvalidType
}
buf, _, err := m.ExportJpeg(vips.NewJpegExportParams())
if err != nil {
return err
}
_, err = w.Write(buf)
return err
}
// Types returns the jpg suffixes.
func (e JpegEncoder) Types() []string {
return []string{typeJpeg, typeJpg}
}
// MimeType returns the mimetype for jpg files.
func (e JpegEncoder) MimeType() string {
return "image/jpeg"
}
@@ -0,0 +1,99 @@
package thumbnail
import (
"image"
"image/color"
"image/draw"
"image/gif"
"strings"
"github.com/kovidgoyal/imaging"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// Generator generates a web friendly file version.
type Generator interface {
Generate(size image.Rectangle, img any) (any, error)
Dimensions(img any) (image.Rectangle, error)
ProcessorID() string
}
// GifGenerator is used to create a web friendly version of the provided gif image.
type GifGenerator struct {
processor Processor
}
func NewGifGenerator(filetype, process string) (GifGenerator, error) {
processor, err := ProcessorFor(process, filetype)
if err != nil {
return GifGenerator{}, err
}
return GifGenerator{processor: processor}, nil
}
// ProcessorID returns the processor identification.
func (g GifGenerator) ProcessorID() string {
return g.processor.ID()
}
// Generate generates a alternative gif version.
func (g GifGenerator) Generate(size image.Rectangle, img any) (any, error) {
// Code inspired by https://github.com/willnorris/gifresize/blob/db93a7e1dcb1c279f7eeb99cc6d90b9e2e23e871/gifresize.go
m, ok := img.(*gif.GIF)
if !ok {
return nil, errors.ErrInvalidType
}
// Create a new RGBA image to hold the incremental frames.
srcX, srcY := m.Config.Width, m.Config.Height
b := image.Rect(0, 0, srcX, srcY)
tmp := image.NewRGBA(b)
for i, frame := range m.Image {
bounds := frame.Bounds()
prev := tmp
draw.Draw(tmp, bounds, frame, bounds.Min, draw.Over)
processed := g.processor.Process(tmp, size.Dx(), size.Dy(), imaging.Lanczos)
m.Image[i] = g.imageToPaletted(processed, frame.Palette)
switch m.Disposal[i] {
case gif.DisposalBackground:
tmp = image.NewRGBA(b)
case gif.DisposalPrevious:
tmp = prev
}
}
m.Config.Width = size.Dx()
m.Config.Height = size.Dy()
return m, nil
}
func (g GifGenerator) Dimensions(img any) (image.Rectangle, error) {
m, ok := img.(*gif.GIF)
if !ok {
return image.Rectangle{}, errors.ErrInvalidType
}
return m.Image[0].Bounds(), nil
}
func (g GifGenerator) imageToPaletted(img image.Image, p color.Palette) *image.Paletted {
b := img.Bounds()
pm := image.NewPaletted(b, p)
draw.FloydSteinberg.Draw(pm, b, img, image.Point{})
return pm
}
// GeneratorFor returns the generator for a given file type
// or nil if the type is not supported.
func GeneratorFor(fileType, processorID string) (Generator, error) {
switch strings.ToLower(fileType) {
case typePng, typeJpg, typeJpeg, typeGgs, typeGgp, typeWebp:
return NewSimpleGenerator(fileType, processorID)
case typeGif:
return NewGifGenerator(fileType, processorID)
default:
return nil, errors.ErrNoGeneratorForType
}
}
@@ -0,0 +1,46 @@
//go:build !enable_vips
package thumbnail
import (
"image"
"github.com/kovidgoyal/imaging"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// SimpleGenerator is the default image generator and is used for all image types expect gif.
type SimpleGenerator struct {
processor Processor
}
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
processor, err := ProcessorFor(process, filetype)
if err != nil {
return SimpleGenerator{}, err
}
return SimpleGenerator{processor: processor}, nil
}
// ProcessorID returns the processor identification.
func (g SimpleGenerator) ProcessorID() string {
return g.processor.ID()
}
// Generate generates a alternative image version.
func (g SimpleGenerator) Generate(size image.Rectangle, img any) (any, error) {
m, ok := img.(image.Image)
if !ok {
return nil, errors.ErrInvalidType
}
return g.processor.Process(m, size.Dx(), size.Dy(), imaging.Lanczos), nil
}
func (g SimpleGenerator) Dimensions(img any) (image.Rectangle, error) {
m, ok := img.(image.Image)
if !ok {
return image.Rectangle{}, errors.ErrInvalidType
}
return m.Bounds(), nil
}
@@ -0,0 +1,84 @@
//go:build enable_vips
package thumbnail
import (
"bytes"
"image"
"strings"
"github.com/davidbyttow/govips/v2/vips"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"golang.org/x/image/bmp"
)
// SimpleGenerator is the default image generator and is used for all image types expect gif.
type SimpleGenerator struct {
crop vips.Interesting
size vips.Size
process string
}
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
switch strings.ToLower(process) {
case "thumbnail":
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
case "fit":
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeBoth}, nil
case "resize":
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeForce}, nil
default:
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
}
}
// ProcessorID returns the processor identification.
func (g SimpleGenerator) ProcessorID() string {
return g.process
}
// Generate generates a alternative image version.
func (g SimpleGenerator) Generate(size image.Rectangle, img interface{}) (interface{}, error) {
var m *vips.ImageRef
var err error
switch img.(type) {
case *image.RGBA:
// This comes from the txt preprocessor
var buf bytes.Buffer
if err = bmp.Encode(&buf, img.(*image.RGBA)); err != nil {
return nil, err
}
m, err = vips.NewImageFromReader(&buf)
if err != nil {
return nil, err
}
case *vips.ImageRef:
m = img.(*vips.ImageRef)
default:
return nil, errors.ErrInvalidType
}
if err := m.ThumbnailWithSize(size.Dx(), 0, g.crop, g.size); err != nil {
return nil, err
}
if err := m.RemoveMetadata(); err != nil {
return nil, err
}
return m, nil
}
func (g SimpleGenerator) Dimensions(img interface{}) (image.Rectangle, error) {
switch img.(type) {
case *image.RGBA:
m := img.(*image.RGBA)
return m.Bounds(), nil
case *vips.ImageRef:
m := img.(*vips.ImageRef)
return image.Rect(0, 0, m.Width(), m.Height()), nil
default:
return image.Rectangle{}, errors.ErrInvalidType
}
}
@@ -0,0 +1,132 @@
package imgsource
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/rhttp"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"google.golang.org/grpc/metadata"
)
const (
// TokenTransportHeader holds the header key for the reva transfer token
// "github.com/opencloud-eu/reva/v2/internal/http/services/datagateway" is internal so we redeclare it here
TokenTransportHeader = "X-Reva-Transfer"
)
// CS3 implements a CS3 image source
type CS3 struct {
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
insecure bool
maxImageFileSize uint64
}
// NewCS3Source configures a new CS3 image source
func NewCS3Source(cfg config.Thumbnail, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], b bytesize.ByteSize) CS3 {
return CS3{
gatewaySelector: gatewaySelector,
insecure: cfg.CS3AllowInsecure,
maxImageFileSize: b.Bytes(),
}
}
// Get downloads the file from a cs3 service
// The caller MUST make sure to close the returned ReadCloser
func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
auth, ok := ContextGetAuthorization(ctx)
if !ok {
return nil, errors.ErrCS3AuthorizationMissing
}
ref, err := storagespace.ParseReference(path)
if err != nil {
// If the path is not a spaces reference try to handle it like a plain
// path reference.
ref = provider.Reference{
Path: path,
}
}
ctx = metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
err = s.checkImageFileSize(ctx, ref)
if err != nil {
return nil, err
}
gwc, err := s.gatewaySelector.Next()
if err != nil {
return nil, err
}
rsp, err := gwc.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &ref})
if err != nil {
return nil, err
}
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load image: %s", rsp.GetStatus().GetMessage())
}
var ep, tk string
for _, p := range rsp.GetProtocols() {
if p.GetProtocol() == "spaces" {
ep, tk = p.GetDownloadEndpoint(), p.GetToken()
break
}
}
if (ep == "" || tk == "") && len(rsp.GetProtocols()) > 0 {
ep, tk = rsp.GetProtocols()[0].GetDownloadEndpoint(), rsp.GetProtocols()[0].GetToken()
}
httpReq, err := rhttp.NewRequest(ctx, "GET", ep, nil)
if err != nil {
return nil, err
}
httpReq.Header.Set(revactx.TokenHeader, auth)
httpReq.Header.Set(TokenTransportHeader, tk)
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: s.insecure, //nolint:gosec
}
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", path, resp.StatusCode)
}
return resp.Body, nil
}
func (s CS3) checkImageFileSize(ctx context.Context, ref provider.Reference) error {
gwc, err := s.gatewaySelector.Next()
if err != nil {
return err
}
stat, err := gwc.Stat(ctx, &provider.StatRequest{Ref: &ref})
if err != nil {
return err
}
if stat.GetStatus().GetCode() != rpc.Code_CODE_OK {
return fmt.Errorf("could not stat image: %s", stat.GetStatus().GetMessage())
}
if stat.GetInfo().GetSize() > s.maxImageFileSize {
return errors.ErrImageTooLarge
}
return nil
}
@@ -0,0 +1,31 @@
package imgsource
import (
"context"
"io"
)
type key int
const (
auth key = iota
)
// Source defines the interface for image sources
type Source interface {
Get(ctx context.Context, path string) (io.ReadCloser, error)
}
// ContextSetAuthorization puts the authorization in the context.
func ContextSetAuthorization(parent context.Context, authorization string) context.Context {
return context.WithValue(parent, auth, authorization)
}
// ContextGetAuthorization gets the authorization from the context.
func ContextGetAuthorization(ctx context.Context) (string, bool) {
val := ctx.Value(auth)
if val == nil {
return "", false
}
return val.(string), true
}
@@ -0,0 +1,75 @@
package imgsource
import (
"context"
"crypto/tls"
"fmt"
_ "image/gif" // Import the gif package so that image.Decode can understand gifs
_ "image/jpeg" // Import the jpeg package so that image.Decode can understand jpegs
_ "image/png" // Import the png package so that image.Decode can understand pngs
"io"
"net/http"
"strconv"
"github.com/qsfera/server/services/thumbnails/pkg/config"
thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
"github.com/pkg/errors"
)
// NewWebDavSource creates a new webdav instance.
func NewWebDavSource(cfg config.Thumbnail, b bytesize.ByteSize) WebDav {
return WebDav{
insecure: cfg.WebdavAllowInsecure,
maxImageFileSize: b.Bytes(),
}
}
// WebDav implements the Source interface for webdav services
type WebDav struct {
insecure bool
maxImageFileSize uint64
}
// Get downloads the file from a webdav service
// The caller MUST make sure to close the returned ReadCloser
func (s WebDav) Get(ctx context.Context, url string) (io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: s.insecure, //nolint:gosec
}
if auth, ok := ContextGetAuthorization(ctx); ok {
req.Header.Add("Authorization", auth)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", url, resp.StatusCode)
}
contentLength := resp.Header.Get("Content-Length")
if contentLength == "" {
// no size information - let's assume it is too big
return nil, thumbnailerErrors.ErrImageTooLarge
}
c, err := strconv.ParseUint(contentLength, 10, 64)
if err != nil {
return nil, errors.Wrapf(err, `could not parse content length of webdav response "%s"`, url)
}
if c > s.maxImageFileSize {
return nil, thumbnailerErrors.ErrImageTooLarge
}
return resp.Body, nil
}
@@ -0,0 +1,22 @@
//go:build !enable_vips
package thumbnail
var (
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
"image/jpeg": {},
"image/gif": {},
"image/bmp": {},
"image/x-ms-bmp": {},
"image/tiff": {},
"text/plain": {},
"audio/flac": {},
"audio/mpeg": {},
"audio/ogg": {},
"application/vnd.geogebra.slides": {},
"application/vnd.geogebra.pinboard": {},
}
)
@@ -0,0 +1,31 @@
//go:build enable_vips
package thumbnail
import "github.com/davidbyttow/govips/v2/vips"
func init() {
// temporary remove TIFF and JP2K from go-vips' list of supported
// imagetypes
delete(vips.ImageTypes, vips.ImageTypeTIFF)
delete(vips.ImageTypes, vips.ImageTypeJP2K)
}
var (
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
"image/jpeg": {},
"image/gif": {},
"image/bmp": {},
"image/x-ms-bmp": {},
"text/plain": {},
"audio/flac": {},
"audio/mpeg": {},
"audio/ogg": {},
"application/vnd.geogebra.slides": {},
"application/vnd.geogebra.pinboard": {},
"image/webp": {},
}
)
@@ -0,0 +1,51 @@
package thumbnail
import (
"image"
"strings"
"github.com/kovidgoyal/imaging"
)
// Processor processes the thumbnail by applying different transformations to it.
type Processor interface {
ID() string
Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
}
// DefinableProcessor is the simplest processor, it holds a replaceable image converter function.
type DefinableProcessor struct {
Slug string
Converter func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
}
// ID returns the processor identification.
func (p DefinableProcessor) ID() string { return p.Slug }
// Process transforms the given image.
func (p DefinableProcessor) Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
return p.Converter(img, width, height, filter)
}
// ProcessorFor returns a matching Processor
func ProcessorFor(id, fileType string) (DefinableProcessor, error) {
switch strings.ToLower(id) {
case "fit":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Fit}, nil
case "resize":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Resize}, nil
case "fill":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
return imaging.Fill(img, width, height, imaging.Center, filter)
}}, nil
case "thumbnail":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Thumbnail}, nil
default:
switch strings.ToLower(fileType) {
case typeGif:
return DefinableProcessor{Converter: imaging.Resize}, nil
default:
return DefinableProcessor{Converter: imaging.Thumbnail}, nil
}
}
}
@@ -0,0 +1,97 @@
package thumbnail_test
import (
"testing"
"github.com/kovidgoyal/imaging"
tAssert "github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
)
func TestProcessorFor(t *testing.T) {
tests := []struct {
id string
fileType string
wantP thumbnail.Processor
wantE error
}{
{
id: "fit",
fileType: "",
wantP: thumbnail.DefinableProcessor{Slug: "fit", Converter: imaging.Fit},
wantE: nil,
},
{
id: "fit",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
wantE: nil,
},
{
id: "FIT",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
wantE: nil,
},
{
id: "resize",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
wantE: nil,
},
{
id: "RESIZE",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
wantE: nil,
},
{
id: "fill",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
wantE: nil,
},
{
id: "FILL",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
wantE: nil,
},
{
id: "thumbnail",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
wantE: nil,
},
{
id: "THUMBNAIL",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
wantE: nil,
},
{
id: "",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{},
wantE: nil,
},
{
id: "",
fileType: "gif",
wantP: thumbnail.DefinableProcessor{},
wantE: nil,
},
}
assert := tAssert.New(t)
for _, tt := range tests {
t.Run("", func(t *testing.T) {
p, e := thumbnail.ProcessorFor(tt.id, tt.fileType)
assert.Equal(p.ID(), tt.wantP.ID())
assert.Equal(e, tt.wantE)
})
}
}
@@ -0,0 +1,101 @@
package thumbnail
import (
"fmt"
"image"
"math"
"strconv"
"strings"
"github.com/pkg/errors"
)
const (
_resolutionSeparator = "x"
)
// ParseResolution returns an image.Rectangle representing the resolution given as a string
func ParseResolution(s string) (image.Rectangle, error) {
parts := strings.Split(s, _resolutionSeparator)
if len(parts) != 2 {
return image.Rectangle{}, fmt.Errorf("failed to parse resolution: %s. Expected format <width>x<height>", s)
}
width, err := strconv.Atoi(parts[0])
if err != nil {
return image.Rectangle{}, fmt.Errorf("width: %s has an invalid value. Expected an integer", parts[0])
}
height, err := strconv.Atoi(parts[1])
if err != nil {
return image.Rectangle{}, fmt.Errorf("height: %s has an invalid value. Expected an integer", parts[1])
}
return image.Rect(0, 0, width, height), nil
}
// Resolutions is a list of image.Rectangle representing resolutions.
type Resolutions []image.Rectangle
// ParseResolutions creates an instance of Resolutions from resolution strings.
func ParseResolutions(strs []string) (Resolutions, error) {
rs := make(Resolutions, 0, len(strs))
for _, s := range strs {
r, err := ParseResolution(s)
if err != nil {
return nil, errors.Wrap(err, "could not parse resolutions")
}
rs = append(rs, r)
}
return rs, nil
}
// ClosestMatch returns the resolution which is closest to the provided resolution.
// If there is no exact match the resolution will be the next higher one.
// If the given resolution is bigger than all available resolutions the biggest available one is used.
func (rs Resolutions) ClosestMatch(requested image.Rectangle, sourceSize image.Rectangle) image.Rectangle {
isLandscape := sourceSize.Dx() > sourceSize.Dy()
sourceLen := dimensionLength(sourceSize, isLandscape)
requestedLen := dimensionLength(requested, isLandscape)
isSourceSmaller := sourceLen < requestedLen
// We don't want to scale images up.
if isSourceSmaller {
return sourceSize
}
if len(rs) == 0 {
return requested
}
var match image.Rectangle
// Since we want to search for the smallest difference we start with the highest possible number
minDiff := math.MaxInt32
for _, current := range rs {
cLen := dimensionLength(current, isLandscape)
diff := requestedLen - cLen
if diff > 0 {
// current is smaller
continue
}
// Convert diff to positive value
// Multiplying by -1 is safe since we aren't getting positive numbers here
// because of the check above
absDiff := diff * -1
if absDiff < minDiff {
minDiff = absDiff
match = current
}
}
if (match == image.Rectangle{}) {
match = rs[len(rs)-1]
}
return match
}
func dimensionLength(rect image.Rectangle, isLandscape bool) int {
if isLandscape {
return rect.Dx()
}
return rect.Dy()
}
@@ -0,0 +1,121 @@
package thumbnail
import (
"image"
"testing"
)
func TestInitWithEmptyArray(t *testing.T) {
rs, err := ParseResolutions([]string{})
if err != nil {
t.Errorf("Init with an empty array should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 0 {
t.Error("Init with an empty array should return an empty Resolutions instance.\n")
}
}
func TestInitWithNil(t *testing.T) {
rs, err := ParseResolutions(nil)
if err != nil {
t.Errorf("Init with nil parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 0 {
t.Error("Init with nil parameter should return an empty Resolutions instance.\n")
}
}
func TestInitWithInvalidValuesInArray(t *testing.T) {
_, err := ParseResolutions([]string{"invalid"})
if err == nil {
t.Error("Init with invalid parameter should fail.\n")
}
}
func TestInit(t *testing.T) {
rs, err := ParseResolutions([]string{"16x16"})
if err != nil {
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 1 {
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), 1)
}
}
func TestInitWithMultipleResolutions(t *testing.T) {
rStrs := []string{"16x16", "32x32", "64x64", "128x128"}
rs, err := ParseResolutions(rStrs)
if err != nil {
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != len(rStrs) {
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), len(rStrs))
}
}
func TestClosestMatchWithEmptyResolutions(t *testing.T) {
rs, _ := ParseResolutions(nil)
want := image.Rect(0, 0, 24, 24)
imgSize := image.Rect(0, 0, 24, 24)
r := rs.ClosestMatch(want, imgSize)
if r.Dx() != want.Dx() || r.Dy() != want.Dy() {
t.Errorf("ClosestMatch from empty resolutions should return the given resolution")
}
}
func TestClosestMatch(t *testing.T) {
rs, _ := ParseResolutions([]string{"16x16", "24x24", "32x32", "64x64", "128x128"})
testData := [][]image.Rectangle{
{image.Rect(0, 0, 17, 17), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 12, 17), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 24, 24), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 20, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 20, 80), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 80, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 48, 48), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 64, 64)},
{image.Rect(0, 0, 1024, 1024), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 256, 36), image.Rect(0, 0, 256, 36)},
}
for _, row := range testData {
given := row[0]
imgSize := row[1]
expected := row[2]
match := rs.ClosestMatch(given, imgSize)
if match != expected {
t.Errorf("Expected resolution %dx%d got %dx%d", expected.Dx(), expected.Dy(), match.Dx(), match.Dy())
}
}
}
func TestParseWithEmptyString(t *testing.T) {
if _, err := ParseResolution(""); err == nil {
t.Error("Parse with empty string should return an error.")
}
}
func TestParseWithInvalidWidth(t *testing.T) {
_, err := ParseResolution("invalidx42")
if err == nil {
t.Error("Parse with invalid width should return an error.")
}
}
func TestParseWithInvalidHeight(t *testing.T) {
_, err := ParseResolution("42xinvalid")
if err == nil {
t.Error("Parse with invalid height should return an error.")
}
}
func TestParseResolution(t *testing.T) {
rStr := "42x23"
r, _ := ParseResolution(rStr)
if r.Dx() != 42 || r.Dy() != 23 {
t.Errorf("Expected resolution %s got %s", rStr, r.String())
}
}
@@ -0,0 +1,118 @@
package storage
import (
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
)
const (
filesDir = "files"
)
// NewFileSystemStorage creates a new instance of FileSystem
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) FileSystem {
return FileSystem{
root: cfg.RootDirectory,
logger: logger,
}
}
// FileSystem represents a storage for the thumbnails using the local file system.
type FileSystem struct {
root string
logger log.Logger
}
// Stat returns if a file for the given key exists on the filesystem
func (s FileSystem) Stat(key string) bool {
img := filepath.Join(s.root, filesDir, key)
if _, err := os.Stat(img); err != nil {
return false
}
return true
}
// Get returns the file content for the given key
func (s FileSystem) Get(key string) ([]byte, error) {
img := filepath.Join(s.root, filesDir, key)
content, err := os.ReadFile(img)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
}
return nil, err
}
return content, nil
}
// Put stores image data in the file system for the given key
func (s FileSystem) Put(key string, img []byte) error {
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)
if err := os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "error while creating directory %s", dir)
}
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
f, err := os.CreateTemp(dir, "tmpthumb")
if err != nil {
return errors.Wrapf(err, "could not create temporary file for \"%s\"", key)
}
defer f.Close()
// if there was a problem writing, remove the temporary file
if _, writeErr := f.Write(img); writeErr != nil {
if remErr := os.Remove(f.Name()); remErr != nil {
return errors.Wrapf(remErr, "could not cleanup temporary file for \"%s\"", key)
}
return errors.Wrapf(writeErr, "could not write to temporary file for \"%s\"", key)
}
// if there wasn't a problem, ensure the data is written into disk
if synErr := f.Sync(); synErr != nil {
return errors.Wrapf(synErr, "could not sync temporary file data into the disk for \"%s\"", key)
}
// rename the temporary file to the final file
if renErr := os.Rename(f.Name(), imgPath); renErr != nil {
// if we couldn't rename, remove the temporary file
if remErr := os.Remove(f.Name()); remErr != nil {
return errors.Wrapf(remErr, "rename failed and could not cleanup temporary file for \"%s\"", key)
}
return errors.Wrapf(renErr, "could not rename temporary file to \"%s\"", key)
}
}
return nil
}
// BuildKey generate the unique key for a thumbnail.
// The key is structure as follows:
//
// <first two letters of checksum>/<next two letters of checksum>/<rest of checksum>/<width>x<height>.<filetype>
//
// e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png
//
// The key also represents the path to the thumbnail in the filesystem under the configured root directory.
func (s FileSystem) BuildKey(r Request) string {
checksum := r.Checksum
filetype := r.Types[0]
parts := []string{strconv.Itoa(r.Resolution.Dx()), "x", strconv.Itoa(r.Resolution.Dy())}
if r.Characteristic != "" {
parts = append(parts, "-", r.Characteristic)
}
parts = append(parts, ".", filetype)
return filepath.Join(checksum[:2], checksum[2:4], checksum[4:], strings.Join(parts, ""))
}
@@ -0,0 +1,64 @@
package storage_test
import (
"image"
"testing"
tAssert "github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
func TestFileSystem_BuildKey(t *testing.T) {
tests := []struct {
r storage.Request
want string
}{
{
r: storage.Request{
Checksum: "120EA8A25E5D487BF68B5F7096440019",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "",
},
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2.png",
},
{
r: storage.Request{
Checksum: "120EA8A25E5D487BF68B5F7096440019",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "fill",
},
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2-fill.png",
},
}
s := storage.FileSystem{}
assert := tAssert.New(t)
for _, tt := range tests {
t.Run("", func(t *testing.T) {
assert.Equal(s.BuildKey(tt.r), tt.want)
})
}
}
@@ -0,0 +1,32 @@
package storage
import (
"image"
)
// Request combines different attributes needed for storage operations.
type Request struct {
// The checksum of the source file
// Will be used to determine if a thumbnail exists
Checksum string
// Types provided by the encoder.
// Contains the mimetypes of the thumbnail.
// In case of jpg/jpeg it will contain both.
Types []string
// The resolution of the thumbnail
Resolution image.Rectangle
// Characteristic defines the different image characteristics,
// for example, if it's scaled up to fit in the bounding box or not,
// is it a chroma version of the image, and so on...
// the main propose for this is to be able to differentiate between images which have
// the same resolution but different characteristics.
Characteristic string
}
// Storage defines the interface for a thumbnail store.
type Storage interface {
Stat(key string) bool
Get(key string) ([]byte, error)
Put(key string, img []byte) error
BuildKey(r Request) string
}
@@ -0,0 +1,131 @@
package thumbnail
import (
"bytes"
"image"
"mime"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
// Request bundles information needed to generate a thumbnail for a file
type Request struct {
Resolution image.Rectangle
Encoder Encoder
Generator Generator
Checksum string
}
// Manager is responsible for generating thumbnails
type Manager interface {
// Generate creates a thumbnail and stores it.
// The function returns a key with which the actual file can be retrieved.
Generate(r Request, img any) (string, error)
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
// The function will return a status if the file exists and the key to the file.
CheckThumbnail(r Request) (string, bool)
// GetThumbnail will load the thumbnail from the storage and return its content.
GetThumbnail(key string) ([]byte, error)
}
// NewSimpleManager creates a new instance of SimpleManager
func NewSimpleManager(resolutions Resolutions, storage storage.Storage, logger log.Logger, maxInputWidth, maxInputHeight int) SimpleManager {
return SimpleManager{
storage: storage,
logger: logger,
resolutions: resolutions,
maxDimension: image.Point{X: maxInputWidth, Y: maxInputHeight},
}
}
// SimpleManager is a simple implementation of Manager
type SimpleManager struct {
storage storage.Storage
logger log.Logger
resolutions Resolutions
maxDimension image.Point
}
// Generate creates a thumbnail and stores it
func (s SimpleManager) Generate(r Request, img any) (string, error) {
var match image.Rectangle
inputDimensions, err := r.Generator.Dimensions(img)
if err != nil {
return "", err
}
match = s.resolutions.ClosestMatch(r.Resolution, inputDimensions)
// validate max input image dimensions - 6016x4000
if inputDimensions.Size().X > s.maxDimension.X || inputDimensions.Size().Y > s.maxDimension.Y {
return "", errors.ErrImageTooLarge
}
thumbnail, err := r.Generator.Generate(match, img)
if err != nil {
return "", err
}
buf := new(bytes.Buffer)
if err := r.Encoder.Encode(buf, thumbnail); err != nil {
return "", err
}
k := s.storage.BuildKey(mapToStorageRequest(r))
if err := s.storage.Put(k, buf.Bytes()); err != nil {
s.logger.Error().Err(err).Msg("could not store thumbnail")
return "", err
}
return k, nil
}
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
func (s SimpleManager) CheckThumbnail(r Request) (string, bool) {
k := s.storage.BuildKey(mapToStorageRequest(r))
return k, s.storage.Stat(k)
}
// GetThumbnail will load the thumbnail from the storage and return its content.
func (s SimpleManager) GetThumbnail(key string) ([]byte, error) {
return s.storage.Get(key)
}
func mapToStorageRequest(r Request) storage.Request {
return storage.Request{
Checksum: r.Checksum,
Resolution: r.Resolution,
Types: r.Encoder.Types(),
Characteristic: r.Generator.ProcessorID(),
}
}
// IsMimeTypeSupported validate if the mime type is supported
func IsMimeTypeSupported(m string) bool {
mimeType, _, err := mime.ParseMediaType(m)
if err != nil {
return false
}
_, supported := SupportedMimeTypes[mimeType]
return supported
}
// PrepareRequest prepare the request based on image parameters
func PrepareRequest(width, height int, tType, checksum, pID string) (Request, error) {
generator, err := GeneratorFor(tType, pID)
if err != nil {
return Request{}, err
}
encoder, err := EncoderForType(tType)
if err != nil {
return Request{}, err
}
return Request{
Resolution: image.Rect(0, 0, width, height),
Generator: generator,
Encoder: encoder,
Checksum: checksum,
}, nil
}
@@ -0,0 +1,198 @@
package thumbnail
import (
"image"
"os"
"path"
"path/filepath"
"reflect"
"testing"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
"github.com/stretchr/testify/assert"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
type NoOpManager struct {
storage.Storage
}
func (m NoOpManager) BuildKey(_ storage.Request) string {
return ""
}
func (m NoOpManager) Set(_, _ string, _ []byte) error {
return nil
}
func BenchmarkGet(b *testing.B) {
sut := NewSimpleManager(
Resolutions{},
NoOpManager{},
log.NewLogger(),
6016,
4000,
)
res, _ := ParseResolution("32x32")
req := Request{
Resolution: res,
Checksum: "1872ade88f3013edeb33decd74a4f947",
}
cwd, _ := os.Getwd()
p := filepath.Join(cwd, "../../testdata/test.png")
f, _ := os.Open(p)
defer f.Close()
img, ext, _ := image.Decode(f)
req.Encoder, _ = EncoderForType(ext)
for i := 0; i < b.N; i++ {
_, _ = sut.Generate(req, img)
}
}
func TestPrepareRequest(t *testing.T) {
type args struct {
width int
height int
tType string
checksum string
}
tests := []struct {
name string
args args
want Request
wantErr bool
}{
{
name: "Test successful prepare the request for jpg",
args: args{
width: 32,
height: 32,
tType: "jpg",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: JpegEncoder{},
Generator: SimpleGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test successful prepare the request for png",
args: args{
width: 32,
height: 32,
tType: "png",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: PngEncoder{},
Generator: SimpleGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test successful prepare the request for gif",
args: args{
width: 32,
height: 32,
tType: "gif",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: GifEncoder{},
Generator: GifGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test error when prepare the request for bmp",
args: args{
width: 32,
height: 32,
tType: "bmp",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := PrepareRequest(tt.args.width, tt.args.height, tt.args.tType, tt.args.checksum, "")
if (err != nil) != tt.wantErr {
t.Errorf("PrepareRequest() error = %v, wantErr %v", err, tt.wantErr)
return
}
// funcs are not reflactable, ignore
if diff := cmp.Diff(tt.want, got, cmpopts.IgnoreFields(Request{}, "Generator")); diff != "" {
t.Errorf("PrepareRequest(): %v", diff)
}
if reflect.TypeOf(got.Generator) != reflect.TypeOf(tt.want.Generator) {
t.Errorf("PrepareRequest() = %v, want %v", reflect.TypeOf(got.Generator), reflect.TypeOf(tt.want.Generator))
}
})
}
}
func TestPreviewGenerationTooBigImage(t *testing.T) {
tests := []struct {
name string
fileName string
mimeType string
}{
{name: "png", mimeType: "image/png", fileName: "../../testdata/test.png"},
{name: "jpg", mimeType: "image/jpeg", fileName: "../../testdata/test.jpg"},
{name: "ggs", mimeType: "application/vnd.geogebra.slides", fileName: "../../testdata/test.ggs"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sut := NewSimpleManager(
Resolutions{},
NoOpManager{},
log.NewLogger(),
1024,
768,
)
res, _ := ParseResolution("32x32")
req := Request{
Resolution: res,
Checksum: "1872ade88f3013edeb33decd74a4f947",
}
cwd, _ := os.Getwd()
p := filepath.Join(cwd, tt.fileName)
f, _ := os.Open(p)
defer f.Close()
preproc := preprocessor.ForType(tt.mimeType, nil)
convert, err := preproc.Convert(f)
if err != nil {
return
}
ext := path.Ext(tt.fileName)
req.Encoder, _ = EncoderForType(ext)
req.Generator, err = GeneratorFor(ext, "fit")
if err != nil {
return
}
generate, err := sut.Generate(req, convert)
if err != nil {
return
}
assert.ErrorIs(t, err, errors.ErrImageTooLarge)
assert.Equal(t, "", generate)
})
}
}