sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prepare for public link thumbnails

Browse Source 
- Implement a CS3 image source handler
 - Use checksum instead of etag for unique check
 - Simplify storage layout
This commit is contained in:
David Christofas
2021-04-27 22:09:41 +02:00
parent cc8f0c9b94
commit dceb96ff98
20 changed files with 620 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
thumbnails/pkg/thumbnail/imgsource/cs3.go
+78
View File
@@ -0,0 +1,78 @@
package imgsource
import (
"context"
"crypto/tls"
"fmt"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/rhttp"
"github.com/cs3org/reva/pkg/token"
"github.com/pkg/errors"
"google.golang.org/grpc/metadata"
"image"
"net/http"
)
type CS3 struct {
client gateway.GatewayAPIClient
}
func NewCS3Source(c gateway.GatewayAPIClient) CS3 {
return CS3{
client: c,
}
}
func (s CS3) Get(ctx context.Context, path string) (image.Image, error) {
auth, _ := ContextGetAuthorization(ctx)
ctx = metadata.AppendToOutgoingContext(context.Background(), token.TokenHeader, auth)
rsp, err := s.client.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{
Ref: &provider.Reference{
Spec: &provider.Reference_Path{
Path: path,
},
} ,
})
if err != nil {
return nil, err
}
if rsp.Status.Code != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load image: %s", rsp.Status.Message)
}
var ep, tk string
for _, p := range rsp.Protocols {
if p.Protocol == "simple" {
ep, tk = p.DownloadEndpoint, p.Token
}
}
httpReq, err := rhttp.NewRequest(ctx, "GET", ep, nil)
if err != nil {
return nil, err
}
httpReq.Header.Set(token.TokenHeader, auth)
httpReq.Header.Set("X-REVA-TRANSFER", tk)
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close() //nolint:errcheck
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", path, resp.StatusCode)
}
img, _, err := image.Decode(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, `could not decode the image "%s"`, path)
}
return img, nil
}
thumbnails/pkg/thumbnail/imgsource/webdav.go
+5 -12
View File
@@ -4,37 +4,30 @@ import (
"context"
"crypto/tls"
"fmt"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
"image"
_ "image/gif" // Import the gif package so that image.Decode can understand gifs
_ "image/jpeg" // Import the jpeg package so that image.Decode can understand jpegs
_ "image/png" // Import the png package so that image.Decode can understand pngs
"net/http"
"net/url"
"path"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
)
// NewWebDavSource creates a new webdav instance.
func NewWebDavSource(cfg config.WebDavSource) WebDav {
func NewWebDavSource(cfg config.Thumbnail) WebDav {
return WebDav{
baseURL: cfg.BaseURL,
insecure: cfg.Insecure,
insecure: cfg.WebdavAllowInsecure,
}
}
// WebDav implements the Source interface for webdav services
type WebDav struct {
baseURL string
insecure bool
}
// Get downloads the file from a webdav service
func (s WebDav) Get(ctx context.Context, file string) (image.Image, error) {
u, _ := url.Parse(s.baseURL)
u.Path = path.Join(u.Path, file)
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
req, err := http.NewRequest(http.MethodGet, file, nil)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, file)
}
thumbnails/pkg/thumbnail/storage/filesystem.go
+30 -92
View File
@@ -1,26 +1,20 @@
package storage
import (
"crypto/sha256"
"encoding/hex"
"io/ioutil"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/config"
"github.com/pkg/errors"
)
const (
usersDir = "users"
filesDir = "files"
)
// NewFileSystemStorage creates a new instanz of FileSystem
// NewFileSystemStorage creates a new instance of FileSystem
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *FileSystem {
return &FileSystem{
root: cfg.RootDirectory,
@@ -32,32 +26,42 @@ func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *File
type FileSystem struct {
root string
logger log.Logger
mux sync.Mutex
}
// Get loads the image from the file system.
func (s *FileSystem) Get(username string, key string) []byte {
userDir := s.userDir(username)
img := filepath.Join(userDir, key)
content, err := ioutil.ReadFile(img)
func (s *FileSystem) Get(key string) ([]byte, bool) {
img := filepath.Join(s.root, filesDir, key)
content, err := os.ReadFile(img)
if err != nil {
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
return nil
if !os.IsNotExist(err) {
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
}
return nil, false
}
return content
return content, true
}
// Set writes the image to the file system.
func (s *FileSystem) Set(username string, key string, img []byte) error {
_, err := s.storeImage(key, img)
if err != nil {
return errors.Wrap(err, "could not store image")
func (s *FileSystem) Put(key string, img []byte) error {
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)
if err := os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "error while creating directory %s", dir)
}
userDir, err := s.createUserDir(username)
if err != nil {
return err
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
f, err := os.Create(imgPath)
if err != nil {
return errors.Wrapf(err, "could not create file \"%s\"", key)
}
defer f.Close()
if _, err = f.Write(img); err != nil {
return errors.Wrapf(err, "could not write to file \"%s\"", key)
}
}
return s.linkImageToUserDir(key, userDir)
return nil
}
// BuildKey generate the unique key for a thumbnail.
@@ -80,69 +84,3 @@ func (s *FileSystem) rootDir(key string) string {
p := strings.Split(key, string(os.PathSeparator))
return p[0]
}
func (s *FileSystem) storeImage(key string, img []byte) (string, error) {
s.mux.Lock()
defer s.mux.Unlock()
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)
if err := os.MkdirAll(dir, 0700); err != nil {
return "", errors.Wrapf(err, "error while creating directory %s", dir)
}
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
f, err := os.Create(imgPath)
if err != nil {
return "", errors.Wrapf(err, "could not create file \"%s\"", key)
}
defer f.Close()
_, err = f.Write(img)
if err != nil {
return "", errors.Wrapf(err, "could not write to file \"%s\"", key)
}
}
return imgPath, nil
}
// userDir returns the path to the user directory.
// The username is hashed before appending it on the path to prevent bugs caused by invalid folder names.
// Also the hash is then splitted up in three parts that results in a path which looks as follows:
// <filestorage-root>/users/<3 characters>/<3 characters>/<48 characters>/
// This will balance the folders in setups with many users.
func (s *FileSystem) userDir(username string) string {
hash := sha256.New224()
if _, err := hash.Write([]byte(username)); err != nil {
s.logger.Fatal().Err(err).Msg("failed to create hash")
}
unHash := hex.EncodeToString(hash.Sum(nil)) // 224 Bits or 224 / 4 = 56 characters.
return filepath.Join(s.root, usersDir, unHash[:3], unHash[3:6], unHash[6:])
}
func (s *FileSystem) createUserDir(username string) (string, error) {
userDir := s.userDir(username)
if err := os.MkdirAll(userDir, 0700); err != nil {
return "", errors.Wrapf(err, "could not create userDir: %s", userDir)
}
return userDir, nil
}
// linkImageToUserDir links the stored images to the user directory.
// The goal is to minimize disk usage by linking to the images if they already exist and avoid file duplication.
func (s *FileSystem) linkImageToUserDir(key string, userDir string) error {
imgRootDir := s.rootDir(key)
s.mux.Lock()
defer s.mux.Unlock()
err := os.Symlink(filepath.Join(s.root, filesDir, imgRootDir), filepath.Join(userDir, imgRootDir))
if err != nil {
if !os.IsExist(err) {
return errors.Wrap(err, "could not link image to userdir")
}
}
return nil
}
thumbnails/pkg/thumbnail/storage/inmemory.go
+6 -13
View File
@@ -7,31 +7,24 @@ import (
// NewInMemoryStorage creates a new InMemory instance.
func NewInMemoryStorage() InMemory {
return InMemory{
store: make(map[string]map[string][]byte),
store: make(map[string][]byte),
}
}
// InMemory represents an in memory storage for thumbnails
// Can be used during development
type InMemory struct {
store map[string]map[string][]byte
store map[string][]byte
}
// Get loads the thumbnail from memory.
func (s InMemory) Get(username string, key string) []byte {
userImages := s.store[username]
if userImages == nil {
return nil
}
return s.store[username][key]
func (s InMemory) Get(key string) ([]byte, bool) {
return s.store[key], true
}
// Set stores the thumbnail in memory.
func (s InMemory) Set(username string, key string, thumbnail []byte) error {
if _, ok := s.store[username]; !ok {
s.store[username] = make(map[string][]byte)
}
s.store[username][key] = thumbnail
func (s InMemory) Put(key string, thumbnail []byte) error {
s.store[key] = thumbnail
return nil
}
thumbnails/pkg/thumbnail/storage/storage.go
+2 -2
View File
@@ -13,7 +13,7 @@ type Request struct {
// Storage defines the interface for a thumbnail store.
type Storage interface {
Get(string, string) []byte
Set(string, string, []byte) error
Get(string) ([]byte, bool)
Put(string, []byte) error
BuildKey(Request) string
}
thumbnails/pkg/thumbnail/thumbnail.go
+12 -16
View File
@@ -2,11 +2,10 @@ package thumbnail
import (
"bytes"
"image"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/thumbnails/pkg/thumbnail/storage"
"golang.org/x/image/draw"
"image"
)
// Request bundles information needed to generate a thumbnail for afile
@@ -14,16 +13,15 @@ type Request struct {
Resolution image.Rectangle
Encoder Encoder
ETag string
Username string
}
// Manager is responsible for generating thumbnails
type Manager interface {
// Get will return a thumbnail for a file
Get(Request, image.Image) ([]byte, error)
Generate(Request, image.Image) ([]byte, error)
// GetStored loads the thumbnail from the storage.
// It will return nil if no image is stored for the given context.
GetStored(Request) []byte
Get(Request) ([]byte, bool)
}
// NewSimpleManager creates a new instance of SimpleManager
@@ -43,31 +41,29 @@ type SimpleManager struct {
}
// Get implements the Get Method of Manager
func (s SimpleManager) Get(r Request, img image.Image) ([]byte, error) {
func (s SimpleManager) Generate(r Request, img image.Image) ([]byte, error) {
match := s.resolutions.ClosestMatch(r.Resolution, img.Bounds())
thumbnail := s.generate(match, img)
key := s.storage.BuildKey(mapToStorageRequest(r))
buf := new(bytes.Buffer)
err := r.Encoder.Encode(buf, thumbnail)
dst := new(bytes.Buffer)
err := r.Encoder.Encode(dst, thumbnail)
if err != nil {
return nil, err
}
bytes := buf.Bytes()
err = s.storage.Set(r.Username, key, bytes)
key := s.storage.BuildKey(mapToStorageRequest(r))
err = s.storage.Put(key, dst.Bytes())
if err != nil {
s.logger.Warn().Err(err).Msg("could not store thumbnail")
}
return bytes, nil
return dst.Bytes(), nil
}
// GetStored tries to get the stored thumbnail and return it.
// If there is no cached thumbnail it will return nil
func (s SimpleManager) GetStored(r Request) []byte {
func (s SimpleManager) Get(r Request) ([]byte, bool) {
key := s.storage.BuildKey(mapToStorageRequest(r))
stored := s.storage.Get(r.Username, key)
return stored
return s.storage.Get(key)
}
func (s SimpleManager) generate(r image.Rectangle, img image.Image) image.Image {