sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite thumbnails API

Browse Source 
Improved the thumbnails API so that the binary files won't be
transported via GRPC. GRPC has a limited message size and isn't very
effiefficient with large binary data.
This commit is contained in:
David Christofas
2022-03-08 23:12:43 +01:00
parent d6182a4ea1
commit 95ae3b8762
30 changed files with 821 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
thumbnails/pkg/thumbnail/storage/filesystem.go
+21 -13
View File
@@ -1,12 +1,14 @@
package storage
import (
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
"io/fs"
"os"
"path/filepath"
"strconv"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
)
const (
@@ -14,8 +16,8 @@ const (
)
// NewFileSystemStorage creates a new instance of FileSystem
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *FileSystem {
return &FileSystem{
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) FileSystem {
return FileSystem{
root: cfg.RootDirectory,
logger: logger,
}
@@ -27,21 +29,27 @@ type FileSystem struct {
logger log.Logger
}
// Get loads the image from the file system.
func (s *FileSystem) Get(key string) ([]byte, bool) {
func (s FileSystem) Stat(key string) bool {
img := filepath.Join(s.root, filesDir, key)
if _, err := os.Stat(img); err != nil {
return false
}
return true
}
func (s FileSystem) Get(key string) ([]byte, error) {
img := filepath.Join(s.root, filesDir, key)
content, err := os.ReadFile(img)
if err != nil {
if !os.IsNotExist(err) {
if !errors.Is(err, fs.ErrNotExist) {
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
}
return nil, false
return nil, err
}
return content, true
return content, nil
}
// Set writes the image to the file system.
func (s *FileSystem) Put(key string, img []byte) error {
func (s FileSystem) Put(key string, img []byte) error {
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)
if err := os.MkdirAll(dir, 0700); err != nil {
@@ -71,7 +79,7 @@ func (s *FileSystem) Put(key string, img []byte) error {
// e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png
//
// The key also represents the path to the thumbnail in the filesystem under the configured root directory.
func (s *FileSystem) BuildKey(r Request) string {
func (s FileSystem) BuildKey(r Request) string {
checksum := r.Checksum
filetype := r.Types[0]
filename := strconv.Itoa(r.Resolution.Dx()) + "x" + strconv.Itoa(r.Resolution.Dy()) + "." + filetype
thumbnails/pkg/thumbnail/storage/inmemory.go
+7 -2
View File
@@ -17,9 +17,14 @@ type InMemory struct {
store map[string][]byte
}
func (s InMemory) Stat(key string) bool {
_, exists := s.store[key]
return exists
}
// Get loads the thumbnail from memory.
func (s InMemory) Get(key string) ([]byte, bool) {
return s.store[key], true
func (s InMemory) Get(key string) ([]byte, error) {
return s.store[key], nil
}
// Set stores the thumbnail in memory.
thumbnails/pkg/thumbnail/storage/storage.go
+4 -3
View File
@@ -8,18 +8,19 @@ import (
type Request struct {
// The checksum of the source file
// Will be used to determine if a thumbnail exists
Checksum string
Checksum string
// Types provided by the encoder.
// Contains the mimetypes of the thumbnail.
// In case of jpg/jpeg it will contain both.
Types []string
Types []string
// The resolution of the thumbnail
Resolution image.Rectangle
}
// Storage defines the interface for a thumbnail store.
type Storage interface {
Get(string) ([]byte, bool)
Stat(string) bool
Get(string) ([]byte, error)
Put(string, []byte) error
BuildKey(Request) string
}
thumbnails/pkg/thumbnail/thumbnail.go
+32 -34
View File
@@ -5,19 +5,19 @@ import (
"image"
"image/gif"
"mime"
"strings"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage"
)
var (
SupportedMimeTypes = [...]string{
"image/png",
"image/jpg",
"image/jpeg",
"image/gif",
"text/plain",
// SupportedMimeTypes contains a all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
"image/jpeg": {},
"image/gif": {},
"text/plain": {},
}
)
@@ -31,11 +31,14 @@ type Request struct {
// Manager is responsible for generating thumbnails
type Manager interface {
// Generate will return a thumbnail for a file
Generate(Request, interface{}) ([]byte, error)
// Get loads the thumbnail from the storage.
// It will return nil if no image is stored for the given context.
Get(Request) ([]byte, bool)
// Generate creates a thumbnail and stores it.
// The function returns a key with which the actual file can be retrieved.
Generate(Request, interface{}) (string, error)
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
// The function will return a status if the file exists and the key to the file.
CheckThumbnail(Request) (string, bool)
// GetThumbnail will load the thumbnail from the storage and return its content.
GetThumbnail(key string) ([]byte, error)
}
// NewSimpleManager creates a new instance of SimpleManager
@@ -54,9 +57,7 @@ type SimpleManager struct {
resolutions Resolutions
}
// Generate creates a thumbnail and stores it.
// The created thumbnail is also being returned.
func (s SimpleManager) Generate(r Request, img interface{}) ([]byte, error) {
func (s SimpleManager) Generate(r Request, img interface{}) (string, error) {
var match image.Rectangle
switch m := img.(type) {
case *gif.GIF:
@@ -67,28 +68,29 @@ func (s SimpleManager) Generate(r Request, img interface{}) ([]byte, error) {
thumbnail, err := r.Generator.GenerateThumbnail(match, img)
if err != nil {
return nil, err
return "", err
}
dst := new(bytes.Buffer)
err = r.Encoder.Encode(dst, thumbnail)
if err != nil {
return nil, err
buf := new(bytes.Buffer)
if err := r.Encoder.Encode(buf, thumbnail); err != nil {
return "", err
}
k := s.storage.BuildKey(mapToStorageRequest(r))
err = s.storage.Put(k, dst.Bytes())
if err != nil {
s.logger.Warn().Err(err).Msg("could not store thumbnail")
if err := s.storage.Put(k, buf.Bytes()); err != nil {
s.logger.Error().Err(err).Msg("could not store thumbnail")
return "", err
}
return dst.Bytes(), nil
return k, nil
}
// Get tries to get the stored thumbnail and return it.
// If there is no cached thumbnail it will return nil
func (s SimpleManager) Get(r Request) ([]byte, bool) {
func (s SimpleManager) CheckThumbnail(r Request) (string, bool) {
k := s.storage.BuildKey(mapToStorageRequest(r))
return s.storage.Get(k)
return k, s.storage.Stat(k)
}
func (s SimpleManager) GetThumbnail(key string) ([]byte, error) {
return s.storage.Get(key)
}
func mapToStorageRequest(r Request) storage.Request {
@@ -104,10 +106,6 @@ func IsMimeTypeSupported(m string) bool {
if err != nil {
return false
}
for _, mt := range SupportedMimeTypes {
if strings.EqualFold(mt, mimeType) {
return true
}
}
return false
_, supported := SupportedMimeTypes[mimeType]
return supported
}