prepare for public link thumbnails
- Implement a CS3 image source handler - Use checksum instead of etag for unique check - Simplify storage layout
This commit is contained in:
David Christofas
@@ -1,26 +1,20 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"github.com/owncloud/ocis/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/thumbnails/pkg/config"
|
||||
"github.com/pkg/errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/owncloud/ocis/ocis-pkg/log"
|
||||
"github.com/owncloud/ocis/thumbnails/pkg/config"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
usersDir = "users"
|
||||
filesDir = "files"
|
||||
)
|
||||
|
||||
// NewFileSystemStorage creates a new instanz of FileSystem
|
||||
// NewFileSystemStorage creates a new instance of FileSystem
|
||||
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *FileSystem {
|
||||
return &FileSystem{
|
||||
root: cfg.RootDirectory,
|
||||
@@ -32,32 +26,42 @@ func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) *File
|
||||
type FileSystem struct {
|
||||
root string
|
||||
logger log.Logger
|
||||
mux sync.Mutex
|
||||
}
|
||||
|
||||
// Get loads the image from the file system.
|
||||
func (s *FileSystem) Get(username string, key string) []byte {
|
||||
userDir := s.userDir(username)
|
||||
img := filepath.Join(userDir, key)
|
||||
content, err := ioutil.ReadFile(img)
|
||||
func (s *FileSystem) Get(key string) ([]byte, bool) {
|
||||
img := filepath.Join(s.root, filesDir, key)
|
||||
content, err := os.ReadFile(img)
|
||||
if err != nil {
|
||||
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
|
||||
return nil
|
||||
if !os.IsNotExist(err) {
|
||||
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return content
|
||||
return content, true
|
||||
}
|
||||
|
||||
// Set writes the image to the file system.
|
||||
func (s *FileSystem) Set(username string, key string, img []byte) error {
|
||||
_, err := s.storeImage(key, img)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not store image")
|
||||
func (s *FileSystem) Put(key string, img []byte) error {
|
||||
imgPath := filepath.Join(s.root, filesDir, key)
|
||||
dir := filepath.Dir(imgPath)
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return errors.Wrapf(err, "error while creating directory %s", dir)
|
||||
}
|
||||
userDir, err := s.createUserDir(username)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
|
||||
f, err := os.Create(imgPath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not create file \"%s\"", key)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err = f.Write(img); err != nil {
|
||||
return errors.Wrapf(err, "could not write to file \"%s\"", key)
|
||||
}
|
||||
}
|
||||
return s.linkImageToUserDir(key, userDir)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildKey generate the unique key for a thumbnail.
|
||||
@@ -80,69 +84,3 @@ func (s *FileSystem) rootDir(key string) string {
|
||||
p := strings.Split(key, string(os.PathSeparator))
|
||||
return p[0]
|
||||
}
|
||||
|
||||
func (s *FileSystem) storeImage(key string, img []byte) (string, error) {
|
||||
s.mux.Lock()
|
||||
defer s.mux.Unlock()
|
||||
imgPath := filepath.Join(s.root, filesDir, key)
|
||||
dir := filepath.Dir(imgPath)
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return "", errors.Wrapf(err, "error while creating directory %s", dir)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
|
||||
f, err := os.Create(imgPath)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "could not create file \"%s\"", key)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = f.Write(img)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "could not write to file \"%s\"", key)
|
||||
}
|
||||
}
|
||||
|
||||
return imgPath, nil
|
||||
}
|
||||
|
||||
// userDir returns the path to the user directory.
|
||||
// The username is hashed before appending it on the path to prevent bugs caused by invalid folder names.
|
||||
// Also the hash is then splitted up in three parts that results in a path which looks as follows:
|
||||
// <filestorage-root>/users/<3 characters>/<3 characters>/<48 characters>/
|
||||
// This will balance the folders in setups with many users.
|
||||
func (s *FileSystem) userDir(username string) string {
|
||||
|
||||
hash := sha256.New224()
|
||||
if _, err := hash.Write([]byte(username)); err != nil {
|
||||
s.logger.Fatal().Err(err).Msg("failed to create hash")
|
||||
}
|
||||
unHash := hex.EncodeToString(hash.Sum(nil)) // 224 Bits or 224 / 4 = 56 characters.
|
||||
|
||||
return filepath.Join(s.root, usersDir, unHash[:3], unHash[3:6], unHash[6:])
|
||||
}
|
||||
|
||||
func (s *FileSystem) createUserDir(username string) (string, error) {
|
||||
userDir := s.userDir(username)
|
||||
if err := os.MkdirAll(userDir, 0700); err != nil {
|
||||
return "", errors.Wrapf(err, "could not create userDir: %s", userDir)
|
||||
}
|
||||
|
||||
return userDir, nil
|
||||
}
|
||||
|
||||
// linkImageToUserDir links the stored images to the user directory.
|
||||
// The goal is to minimize disk usage by linking to the images if they already exist and avoid file duplication.
|
||||
func (s *FileSystem) linkImageToUserDir(key string, userDir string) error {
|
||||
imgRootDir := s.rootDir(key)
|
||||
|
||||
s.mux.Lock()
|
||||
defer s.mux.Unlock()
|
||||
err := os.Symlink(filepath.Join(s.root, filesDir, imgRootDir), filepath.Join(userDir, imgRootDir))
|
||||
if err != nil {
|
||||
if !os.IsExist(err) {
|
||||
return errors.Wrap(err, "could not link image to userdir")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,31 +7,24 @@ import (
|
||||
// NewInMemoryStorage creates a new InMemory instance.
|
||||
func NewInMemoryStorage() InMemory {
|
||||
return InMemory{
|
||||
store: make(map[string]map[string][]byte),
|
||||
store: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
// InMemory represents an in memory storage for thumbnails
|
||||
// Can be used during development
|
||||
type InMemory struct {
|
||||
store map[string]map[string][]byte
|
||||
store map[string][]byte
|
||||
}
|
||||
|
||||
// Get loads the thumbnail from memory.
|
||||
func (s InMemory) Get(username string, key string) []byte {
|
||||
userImages := s.store[username]
|
||||
if userImages == nil {
|
||||
return nil
|
||||
}
|
||||
return s.store[username][key]
|
||||
func (s InMemory) Get(key string) ([]byte, bool) {
|
||||
return s.store[key], true
|
||||
}
|
||||
|
||||
// Set stores the thumbnail in memory.
|
||||
func (s InMemory) Set(username string, key string, thumbnail []byte) error {
|
||||
if _, ok := s.store[username]; !ok {
|
||||
s.store[username] = make(map[string][]byte)
|
||||
}
|
||||
s.store[username][key] = thumbnail
|
||||
func (s InMemory) Put(key string, thumbnail []byte) error {
|
||||
s.store[key] = thumbnail
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ type Request struct {
|
||||
|
||||
// Storage defines the interface for a thumbnail store.
|
||||
type Storage interface {
|
||||
Get(string, string) []byte
|
||||
Set(string, string, []byte) error
|
||||
Get(string) ([]byte, bool)
|
||||
Put(string, []byte) error
|
||||
BuildKey(Request) string
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user