Add 'thumbnails/' from commit '2ebf7b2f23af96b3b499c401fe5498a9349403d8'

git-subtree-dir: thumbnails
git-subtree-mainline: ccc651a11e
git-subtree-split: 2ebf7b2f23

thumbnails/pkg/thumbnail/encoding.go

@@ -0,0 +1,70 @@
+package thumbnail
+
+import (
+	"image"
+	"image/jpeg"
+	"image/png"
+	"io"
+	"strings"
+)
+
+// Encoder encodes the thumbnail to a specific format.
+type Encoder interface {
+	// Encode encodes the image to a format.
+	Encode(io.Writer, image.Image) error
+	// Types returns the formats suffixes.
+	Types() []string
+	// MimeType returns the mimetype used by the encoder.
+	MimeType() string
+}
+
+// PngEncoder encodes to png
+type PngEncoder struct{}
+
+// Encode encodes to png format
+func (e PngEncoder) Encode(w io.Writer, i image.Image) error {
+	return png.Encode(w, i)
+}
+
+// Types returns the png suffix
+func (e PngEncoder) Types() []string {
+	return []string{"png"}
+}
+
+// MimeType returns the mimetype for png files.
+func (e PngEncoder) MimeType() string {
+	return "image/png"
+}
+
+// JpegEncoder encodes to jpg.
+type JpegEncoder struct{}
+
+// Encode encodes to jpg
+func (e JpegEncoder) Encode(w io.Writer, i image.Image) error {
+	return jpeg.Encode(w, i, nil)
+}
+
+// Types returns the jpg suffixes.
+func (e JpegEncoder) Types() []string {
+	return []string{"jpeg", "jpg"}
+}
+
+// MimeType returns the mimetype for jpg files.
+func (e JpegEncoder) MimeType() string {
+	return "image/jpeg"
+}
+
+// EncoderForType returns the encoder for a given file type
+// or nil if the type is not supported.
+func EncoderForType(fileType string) Encoder {
+	switch strings.ToLower(fileType) {
+	case "png":
+		return PngEncoder{}
+	case "jpg":
+		fallthrough
+	case "jpeg":
+		return JpegEncoder{}
+	default:
+		return nil
+	}
+}

thumbnails/pkg/thumbnail/imgsource/filesystem.go

@@ -0,0 +1,39 @@
+package imgsource
+
+import (
+	"context"
+	"fmt"
+	"image"
+	"os"
+	"path/filepath"
+
+	"github.com/owncloud/ocis-thumbnails/pkg/config"
+)
+
+// NewFileSystemSource return a new FileSystem instance
+func NewFileSystemSource(cfg config.FileSystemSource) FileSystem {
+	return FileSystem{
+		basePath: cfg.BasePath,
+	}
+}
+
+// FileSystem is an image source using the local file system
+type FileSystem struct {
+	basePath string
+}
+
+// Get retrieves an image from the filesystem.
+func (s FileSystem) Get(ctx context.Context, file string) (image.Image, error) {
+	imgPath := filepath.Join(s.basePath, file)
+	f, err := os.Open(filepath.Clean(imgPath))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load the file %s from %s error %s", file, imgPath, err.Error())
+	}
+
+	img, _, err := image.Decode(f)
+	if err != nil {
+		return nil, err
+	}
+
+	return img, nil
+}

thumbnails/pkg/thumbnail/imgsource/imgsource.go

@@ -0,0 +1,30 @@
+package imgsource
+
+import (
+	"context"
+	"image"
+)
+
+type key int
+
+const (
+	auth key = iota
+)
+
+// Source defines the interface for image sources
+type Source interface {
+	Get(ctx context.Context, path string) (image.Image, error)
+}
+
+// WithAuthorization puts the authorization in the context.
+func WithAuthorization(parent context.Context, authorization string) context.Context {
+	return context.WithValue(parent, auth, authorization)
+}
+
+func authorization(ctx context.Context) string {
+	val := ctx.Value(auth)
+	if val == nil {
+		return ""
+	}
+	return val.(string)
+}

thumbnails/pkg/thumbnail/imgsource/webdav.go

@@ -0,0 +1,61 @@
+package imgsource
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"image"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/owncloud/ocis-thumbnails/pkg/config"
+)
+
+// NewWebDavSource creates a new webdav instance.
+func NewWebDavSource(cfg config.WebDavSource) WebDav {
+	return WebDav{
+		baseURL:  cfg.BaseURL,
+		insecure: cfg.Insecure,
+	}
+}
+
+// WebDav implements the Source interface for webdav services
+type WebDav struct {
+	baseURL  string
+	insecure bool
+}
+
+// Get downloads the file from a webdav service
+func (s WebDav) Get(ctx context.Context, file string) (image.Image, error) {
+	u, _ := url.Parse(s.baseURL)
+	u.Path = path.Join(u.Path, file)
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("could not get the image \"%s\" error: %s", file, err.Error())
+	}
+
+	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: s.insecure}
+
+	auth := authorization(ctx)
+	if auth == "" {
+		return nil, fmt.Errorf("could not get image \"%s\" error: authorization is missing", file)
+	}
+	req.Header.Add("Authorization", auth)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("could not get the image \"%s\" error: %s", file, err.Error())
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", file, resp.StatusCode)
+	}
+
+	img, _, err := image.Decode(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("could not decode the image \"%s\". error: %s", file, err.Error())
+	}
+	return img, nil
+}

thumbnails/pkg/thumbnail/resolution/resolution.go

@@ -0,0 +1,37 @@
+package resolution
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// Parse parses a resolution string in the form <width>x<height> and returns a resolution instance.
+func Parse(s string) (Resolution, error) {
+	parts := strings.Split(s, "x")
+	if len(parts) != 2 {
+		return Resolution{}, fmt.Errorf("failed to parse resolution: %s. Expected format <width>x<height>", s)
+	}
+	width, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return Resolution{}, fmt.Errorf("width: %s has an invalid value. Expected an integer", parts[0])
+	}
+	height, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return Resolution{}, fmt.Errorf("height: %s has an invalid value. Expected an integer", parts[1])
+	}
+	return Resolution{Width: width, Height: height}, nil
+}
+
+// Resolution defines represents the width and height of a thumbnail.
+type Resolution struct {
+	Width  int
+	Height int
+}
+
+// String returns the resolution in the format:
+//
+// <width>x<height>
+func (r Resolution) String() string {
+	return strconv.Itoa(r.Width) + "x" + strconv.Itoa(r.Height)
+}

thumbnails/pkg/thumbnail/resolution/resolution_test.go

@@ -0,0 +1,40 @@
+package resolution
+
+import "testing"
+
+func TestParseWithEmptyString(t *testing.T) {
+	_, err := Parse("")
+	if err == nil {
+		t.Error("Parse with empty string should return an error.")
+	}
+}
+
+func TestParseWithInvalidWidth(t *testing.T) {
+	_, err := Parse("invalidx42")
+	if err == nil {
+		t.Error("Parse with invalid width should return an error.")
+	}
+}
+
+func TestParseWithInvalidHeight(t *testing.T) {
+	_, err := Parse("42xinvalid")
+	if err == nil {
+		t.Error("Parse with invalid height should return an error.")
+	}
+}
+
+func TestParse(t *testing.T) {
+	rStr := "42x23"
+	r, _ := Parse(rStr)
+	if r.Width != 42 || r.Height != 23 {
+		t.Errorf("Expected resolution %s got %s", rStr, r.String())
+	}
+}
+
+func TestString(t *testing.T) {
+	r := Resolution{Width: 42, Height: 23}
+	expected := "42x23"
+	if r.String() != expected {
+		t.Errorf("Expected string %s got %s", expected, r.String())
+	}
+}

thumbnails/pkg/thumbnail/resolution/resolutions.go

@@ -0,0 +1,74 @@
+package resolution
+
+import (
+	"fmt"
+	"math"
+	"sort"
+)
+
+// New creates an instance of Resolutions from resolution strings.
+func New(rStrs []string) (Resolutions, error) {
+	var rs Resolutions
+	for _, rStr := range rStrs {
+		r, err := Parse(rStr)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize resolutions: %s", err.Error())
+		}
+		rs = append(rs, r)
+	}
+	sort.Slice(rs, func(i, j int) bool {
+		left := rs[i]
+		right := rs[j]
+
+		leftSize := left.Width * left.Height
+		rightSize := right.Width * right.Height
+
+		return leftSize < rightSize
+	})
+
+	return rs, nil
+}
+
+// Resolutions represents the available thumbnail resolutions.
+type Resolutions []Resolution
+
+// ClosestMatch returns the resolution which is closest to the provided resolution.
+// If there is no exact match the resolution will be the next higher one.
+// If the given resolution is bigger than all available resolutions the biggest available one is used.
+func (r Resolutions) ClosestMatch(width, height int) Resolution {
+	if len(r) == 0 {
+		return Resolution{Width: width, Height: height}
+	}
+
+	isLandscape := width > height
+	givenLen := int(math.Max(float64(width), float64(height)))
+
+	// Initialize with the first resolution
+	var match Resolution
+	minDiff := math.MaxInt32
+
+	for _, current := range r {
+		len := dimensionLength(current, isLandscape)
+		diff := givenLen - len
+		if diff > 0 {
+			continue
+		}
+		absDiff := int(math.Abs(float64(diff)))
+		if absDiff < minDiff {
+			minDiff = absDiff
+			match = current
+		}
+	}
+
+	if match == (Resolution{}) {
+		match = r[len(r)-1]
+	}
+	return match
+}
+
+func dimensionLength(r Resolution, landscape bool) int {
+	if landscape {
+		return r.Width
+	}
+	return r.Height
+}

thumbnails/pkg/thumbnail/resolution/resolutions_test.go

@@ -0,0 +1,111 @@
+package resolution
+
+import (
+	"testing"
+)
+
+func TestInitWithEmptyArray(t *testing.T) {
+	rs, err := New([]string{})
+	if err != nil {
+		t.Errorf("Init with an empty array should not fail. Error: %s.\n", err.Error())
+	}
+	if len(rs) != 0 {
+		t.Error("Init with an empty array should return an empty Resolutions instance.\n")
+	}
+}
+
+func TestInitWithNil(t *testing.T) {
+	rs, err := New(nil)
+	if err != nil {
+		t.Errorf("Init with nil parameter should not fail. Error: %s.\n", err.Error())
+	}
+	if len(rs) != 0 {
+		t.Error("Init with nil parameter should return an empty Resolutions instance.\n")
+	}
+}
+
+func TestInitWithInvalidValuesInArray(t *testing.T) {
+	_, err := New([]string{"invalid"})
+	if err == nil {
+		t.Error("Init with invalid parameter should fail.\n")
+	}
+}
+
+func TestInit(t *testing.T) {
+	rs, err := New([]string{"16x16"})
+	if err != nil {
+		t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
+	}
+	if len(rs) != 1 {
+		t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), 1)
+	}
+}
+
+func TestInitWithMultipleResolutions(t *testing.T) {
+	rStrs := []string{"16x16", "32x32", "64x64", "128x128"}
+	rs, err := New(rStrs)
+	if err != nil {
+		t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
+	}
+	if len(rs) != len(rStrs) {
+		t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), len(rStrs))
+	}
+}
+
+func TestInitWithMultipleResolutionsShouldBeSorted(t *testing.T) {
+	rStrs := []string{"32x32", "64x64", "16x16", "128x128"}
+	rs, err := New(rStrs)
+	if err != nil {
+		t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
+	}
+
+	for i := 0; i < len(rs)-1; i++ {
+		current := rs[i]
+		currentSize := current.Width * current.Height
+		next := rs[i]
+		nextSize := next.Width * next.Height
+
+		if currentSize > nextSize {
+			t.Error("Resolutions are not sorted.")
+		}
+
+	}
+}
+func TestClosestMatchWithEmptyResolutions(t *testing.T) {
+	rs, _ := New(nil)
+	width := 24
+	height := 24
+
+	r := rs.ClosestMatch(width, height)
+	if r.Width != width || r.Height != height {
+		t.Errorf("ClosestMatch from empty resolutions should return the given resolution")
+	}
+}
+
+func TestClosestMatch(t *testing.T) {
+	rs, _ := New([]string{"16x16", "24x24", "32x32", "64x64", "128x128"})
+	table := [][]int{
+		// width, height, expectedWidth, expectedHeight
+		[]int{17, 17, 24, 24},
+		[]int{12, 17, 24, 24},
+		[]int{24, 24, 24, 24},
+		[]int{20, 20, 24, 24},
+		[]int{20, 80, 128, 128},
+		[]int{80, 20, 128, 128},
+		[]int{48, 48, 64, 64},
+		[]int{1024, 1024, 128, 128},
+	}
+
+	for _, row := range table {
+		width := row[0]
+		height := row[1]
+		expectedWidth := row[2]
+		expectedHeight := row[3]
+
+		match := rs.ClosestMatch(width, height)
+
+		if match.Width != expectedWidth || match.Height != expectedHeight {
+			t.Errorf("Expected resolution %dx%d got %s", expectedWidth, expectedHeight, match.String())
+		}
+	}
+}

thumbnails/pkg/thumbnail/storage/filesystem.go

@@ -0,0 +1,144 @@
+package storage
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/owncloud/ocis-pkg/v2/log"
+	"github.com/owncloud/ocis-thumbnails/pkg/config"
+	"github.com/pkg/errors"
+)
+
+const (
+	usersDir = "users"
+	filesDir = "files"
+)
+
+// NewFileSystemStorage creates a new instanz of FileSystem
+func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *FileSystem {
+	return &FileSystem{
+		root:   cfg.RootDirectory,
+		logger: logger,
+	}
+}
+
+// FileSystem represents a storage for the thumbnails using the local file system.
+type FileSystem struct {
+	root   string
+	logger log.Logger
+	mux    sync.Mutex
+}
+
+// Get loads the image from the file system.
+func (s *FileSystem) Get(username string, key string) []byte {
+	userDir := s.userDir(username)
+	img := filepath.Join(userDir, key)
+	content, err := ioutil.ReadFile(img)
+	if err != nil {
+		s.logger.Warn().Err(err).Msgf("could not read file %s", key)
+		return nil
+	}
+	return content
+}
+
+// Set writes the image to the file system.
+func (s *FileSystem) Set(username string, key string, img []byte) error {
+	_, err := s.storeImage(key, img)
+	if err != nil {
+		return errors.Wrap(err, "could not store image")
+	}
+	userDir, err := s.createUserDir(username)
+	if err != nil {
+		return err
+	}
+	return s.linkImageToUserDir(key, userDir)
+}
+
+// BuildKey generate the unique key for a thumbnail.
+// The key is structure as follows:
+//
+// <first two letters of etag>/<next two letters of etag>/<rest of etag>/<width>x<height>.<filetype>
+//
+// e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png
+//
+// The key also represents the path to the thumbnail in the filesystem under the configured root directory.
+func (s *FileSystem) BuildKey(r Request) string {
+	etag := r.ETag
+	filetype := r.Types[0]
+	filename := r.Resolution.String() + "." + filetype
+
+	return filepath.Join(etag[:2], etag[2:4], etag[4:], filename)
+}
+
+func (s *FileSystem) rootDir(key string) string {
+	p := strings.Split(key, string(os.PathSeparator))
+	return p[0]
+}
+
+func (s *FileSystem) storeImage(key string, img []byte) (string, error) {
+	s.mux.Lock()
+	defer s.mux.Unlock()
+	imgPath := filepath.Join(s.root, filesDir, key)
+	dir := filepath.Dir(imgPath)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return "", errors.Wrapf(err, "error while creating directory %s", dir)
+	}
+
+	if _, err := os.Stat(imgPath); os.IsNotExist(err) {
+		f, err := os.Create(imgPath)
+		if err != nil {
+			return "", errors.Wrapf(err, "could not create file \"%s\"", key)
+		}
+		defer f.Close()
+
+		_, err = f.Write(img)
+		if err != nil {
+			return "", errors.Wrapf(err, "could not write to file \"%s\"", key)
+		}
+	}
+
+	return imgPath, nil
+}
+
+// userDir returns the path to the user directory.
+// The username is hashed before appending it on the path to prevent bugs caused by invalid folder names.
+// Also the hash is then splitted up in three parts that results in a path which looks as follows:
+// <filestorage-root>/users/<3 characters>/<3 characters>/<48 characters>/
+// This will balance the folders in setups with many users.
+func (s *FileSystem) userDir(username string) string {
+	hash := sha256.New224()
+	hash.Write([]byte(username))
+	unHash := hex.EncodeToString(hash.Sum(nil)) // 224 Bits or 224 / 4 = 56 characters.
+
+	return filepath.Join(s.root, usersDir, unHash[:3], unHash[3:6], unHash[6:])
+}
+
+func (s *FileSystem) createUserDir(username string) (string, error) {
+	userDir := s.userDir(username)
+	if err := os.MkdirAll(userDir, 0700); err != nil {
+		return "", errors.Wrapf(err, "could not create userDir: %s", userDir)
+	}
+
+	return userDir, nil
+}
+
+// linkImageToUserDir links the stored images to the user directory.
+// The goal is to minimize disk usage by linking to the images if they already exist and avoid file duplicaiton.
+func (s *FileSystem) linkImageToUserDir(key string, userDir string) error {
+	imgRootDir := s.rootDir(key)
+
+	s.mux.Lock()
+	defer s.mux.Unlock()
+	err := os.Symlink(filepath.Join(s.root, filesDir, imgRootDir), filepath.Join(userDir, imgRootDir))
+	if err != nil {
+		if !os.IsExist(err) {
+			return errors.Wrap(err, "could not link image to userdir")
+		}
+	}
+	return nil
+}

thumbnails/pkg/thumbnail/storage/inmemory.go

@@ -0,0 +1,46 @@
+package storage
+
+import (
+	"strings"
+)
+
+// NewInMemoryStorage creates a new InMemory instance.
+func NewInMemoryStorage() InMemory {
+	return InMemory{
+		store: make(map[string]map[string][]byte),
+	}
+}
+
+// InMemory represents an in memory storage for thumbnails
+// Can be used during development
+type InMemory struct {
+	store map[string]map[string][]byte
+}
+
+// Get loads the thumbnail from memory.
+func (s InMemory) Get(username string, key string) []byte {
+	userImages := s.store[username]
+	if userImages == nil {
+		return nil
+	}
+	return s.store[username][key]
+}
+
+// Set stores the thumbnail in memory.
+func (s InMemory) Set(username string, key string, thumbnail []byte) error {
+	if _, ok := s.store[username]; !ok {
+		s.store[username] = make(map[string][]byte)
+	}
+	s.store[username][key] = thumbnail
+	return nil
+}
+
+// BuildKey generates a unique key to store and retrieve the thumbnail.
+func (s InMemory) BuildKey(r Request) string {
+	parts := []string{
+		r.ETag,
+		r.Resolution.String(),
+		strings.Join(r.Types, ","),
+	}
+	return strings.Join(parts, "+")
+}

thumbnails/pkg/thumbnail/storage/storage.go

@@ -0,0 +1,17 @@
+package storage
+
+import "github.com/owncloud/ocis-thumbnails/pkg/thumbnail/resolution"
+
+// Request combines different attributes needed for storage operations.
+type Request struct {
+	ETag       string
+	Types      []string
+	Resolution resolution.Resolution
+}
+
+// Storage defines the interface for a thumbnail store.
+type Storage interface {
+	Get(string, string) []byte
+	Set(string, string, []byte) error
+	BuildKey(Request) string
+}

thumbnails/pkg/thumbnail/thumbnails.go

@@ -0,0 +1,84 @@
+package thumbnail
+
+import (
+	"bytes"
+	"image"
+
+	"github.com/nfnt/resize"
+	"github.com/owncloud/ocis-pkg/v2/log"
+	"github.com/owncloud/ocis-thumbnails/pkg/thumbnail/resolution"
+	"github.com/owncloud/ocis-thumbnails/pkg/thumbnail/storage"
+)
+
+// Request bundles information needed to generate a thumbnail for afile
+type Request struct {
+	Resolution resolution.Resolution
+	ImagePath  string
+	Encoder    Encoder
+	ETag       string
+	Username   string
+}
+
+// Manager is responsible for generating thumbnails
+type Manager interface {
+	// Get will return a thumbnail for a file
+	Get(Request, image.Image) ([]byte, error)
+	// GetStored loads the thumbnail from the storage.
+	// It will return nil if no image is stored for the given context.
+	GetStored(Request) []byte
+}
+
+// NewSimpleManager creates a new instance of SimpleManager
+func NewSimpleManager(storage storage.Storage, logger log.Logger) SimpleManager {
+	return SimpleManager{
+		storage: storage,
+		logger:  logger,
+	}
+}
+
+// SimpleManager is a simple implementation of Manager
+type SimpleManager struct {
+	storage storage.Storage
+	logger  log.Logger
+}
+
+// Get implements the Get Method of Manager
+func (s SimpleManager) Get(r Request, img image.Image) ([]byte, error) {
+	thumbnail := s.generate(r, img)
+
+	key := s.storage.BuildKey(mapToStorageRequest(r))
+
+	buf := new(bytes.Buffer)
+	err := r.Encoder.Encode(buf, thumbnail)
+	if err != nil {
+		return nil, err
+	}
+	bytes := buf.Bytes()
+	err = s.storage.Set(r.Username, key, bytes)
+	if err != nil {
+		s.logger.Warn().Err(err).Msg("could not store thumbnail")
+	}
+	return bytes, nil
+}
+
+// GetStored tries to get the stored thumbnail and return it.
+// If there is no cached thumbnail it will return nil
+func (s SimpleManager) GetStored(r Request) []byte {
+	key := s.storage.BuildKey(mapToStorageRequest(r))
+	stored := s.storage.Get(r.Username, key)
+	return stored
+}
+
+func (s SimpleManager) generate(r Request, img image.Image) image.Image {
+	thumbnail := resize.Thumbnail(uint(r.Resolution.Width), uint(r.Resolution.Height), img, resize.Lanczos2)
+	return thumbnail
+}
+
+func mapToStorageRequest(r Request) storage.Request {
+	sR := storage.Request{
+		ETag:       r.ETag,
+		Resolution: r.Resolution,
+		Types:      r.Encoder.Types(),
+	}
+	return sR
+}