Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,54 @@
package command
import (
"fmt"
"net/http"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/config/parser"
"github.com/spf13/cobra"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "health",
Short: "check health status",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
resp, err := http.Get(
fmt.Sprintf(
"http://%s/healthz",
cfg.Debug.Addr,
),
)
if err != nil {
logger.Fatal().
Err(err).
Msg("Failed to request health check")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Fatal().
Int("code", resp.StatusCode).
Msg("Health seems to be in bad state")
}
logger.Debug().
Int("code", resp.StatusCode).
Msg("Health got a good state")
return nil
},
}
}
@@ -0,0 +1,35 @@
package command
import (
"os"
"github.com/qsfera/server/pkg/clihelper"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/spf13/cobra"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) []*cobra.Command {
return []*cobra.Command{
// start this service
Server(cfg),
// interaction with this service
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the qsfera-thumbnails command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cobra.Command{
Use: "thumbnails",
Short: "Example usage",
})
app.AddCommand(GetCommands(cfg)...)
app.SetArgs(os.Args[1:])
return app.ExecuteContext(cfg.Context)
}
@@ -0,0 +1,109 @@
package command
import (
"context"
"fmt"
"os/signal"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/pkg/runner"
ogrpc "github.com/qsfera/server/pkg/service/grpc"
"github.com/qsfera/server/pkg/tracing"
"github.com/qsfera/server/pkg/version"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/config/parser"
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
"github.com/qsfera/server/services/thumbnails/pkg/server/debug"
"github.com/qsfera/server/services/thumbnails/pkg/server/grpc"
"github.com/qsfera/server/services/thumbnails/pkg/server/http"
"github.com/spf13/cobra"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "server",
Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
if err != nil {
return err
}
cfg.GrpcClient, err = ogrpc.NewClient(ogrpc.GetClientOptions(cfg.GRPCClientTLS)...)
if err != nil {
return err
}
var cancel context.CancelFunc
if cfg.Context == nil {
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
defer cancel()
}
ctx := cfg.Context
m := metrics.New()
m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
gr := runner.NewGroup()
service := grpc.NewService(
grpc.Logger(logger),
grpc.Context(ctx),
grpc.Config(cfg),
grpc.Name(cfg.Service.Name),
grpc.Namespace(cfg.GRPC.Namespace),
grpc.Address(cfg.GRPC.Addr),
grpc.Metrics(m),
grpc.TraceProvider(traceProvider),
grpc.MaxConcurrentRequests(cfg.GRPC.MaxConcurrentRequests),
)
gr.Add(runner.NewGoMicroGrpcServerRunner(cfg.Service.Name+".grpc", service))
server, err := debug.Server(
debug.Logger(logger),
debug.Config(cfg),
debug.Context(ctx),
)
if err != nil {
logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGolangHttpServerRunner(cfg.Service.Name+".debug", server))
httpServer, err := http.Server(
http.Logger(logger),
http.Context(ctx),
http.Config(cfg),
http.Metrics(m),
http.Namespace(cfg.HTTP.Namespace),
http.TraceProvider(traceProvider),
)
if err != nil {
logger.Info().
Err(err).
Str("transport", "http").
Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", httpServer))
grResults := gr.Run(ctx)
// return the first non-nil error found in the results
for _, grResult := range grResults {
if grResult.RunnerError != nil {
return grResult.RunnerError
}
}
return nil
},
}
}
@@ -0,0 +1,49 @@
package command
import (
"fmt"
"os"
"github.com/qsfera/server/pkg/registry"
"github.com/qsfera/server/pkg/version"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/olekukonko/tablewriter"
"github.com/olekukonko/tablewriter/tw"
"github.com/spf13/cobra"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "print the version of this binary and the running service instances",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Println("")
reg := registry.GetRegistry()
services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name)
if err != nil {
fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err))
return err
}
if len(services) == 0 {
fmt.Println("No running " + cfg.Service.Name + " service found.")
return nil
}
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
table.Header([]string{"Version", "Address", "Id"})
for _, s := range services {
for _, n := range s.Nodes {
table.Append([]string{s.Version, n.Address, n.Id})
}
}
table.Render()
return nil
},
}
}
@@ -0,0 +1,49 @@
// Package config contains the configuration for the qsfera-thumbnails service
package config
import (
"context"
"github.com/qsfera/server/pkg/shared"
"go-micro.dev/v4/client"
)
// Config combines all available configuration parts.
type Config struct {
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
Service Service `yaml:"-"`
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;THUMBNAILS_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
Debug Debug `yaml:"debug"`
GRPC GRPCConfig `yaml:"grpc"`
HTTP HTTP `yaml:"http"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
GrpcClient client.Client `yaml:"-"`
Thumbnail Thumbnail `yaml:"thumbnail"`
Context context.Context `yaml:"-"`
}
// FileSystemStorage defines the available filesystem storage configuration.
type FileSystemStorage struct {
RootDirectory string `yaml:"root_directory" env:"THUMBNAILS_FILESYSTEMSTORAGE_ROOT" desc:"The directory where the filesystem storage will store the thumbnails. If not defined, the root directory derives from $OC_BASE_DATA_PATH/thumbnails." introductionVersion:"1.0.0"`
}
// Thumbnail defines the available thumbnail related configuration.
type Thumbnail struct {
Resolutions []string `yaml:"resolutions" env:"THUMBNAILS_RESOLUTIONS" desc:"The supported list of target resolutions in the format WidthxHeight like 32x32. You can define any resolution as required. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
FileSystemStorage FileSystemStorage `yaml:"filesystem_storage"`
WebdavAllowInsecure bool `yaml:"webdav_allow_insecure" env:"OC_INSECURE;THUMBNAILS_WEBDAVSOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the webdav source." introductionVersion:"1.0.0"`
CS3AllowInsecure bool `yaml:"cs3_allow_insecure" env:"OC_INSECURE;THUMBNAILS_CS3SOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the CS3 source." introductionVersion:"1.0.0"`
RevaGateway string `yaml:"reva_gateway" env:"OC_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata" introductionVersion:"1.0.0"`
FontMapFile string `yaml:"font_map_file" env:"THUMBNAILS_TXT_FONTMAP_FILE" desc:"The path to a font file for txt thumbnails." introductionVersion:"1.0.0"`
TransferSecret string `yaml:"transfer_secret" env:"THUMBNAILS_TRANSFER_TOKEN" desc:"The secret to sign JWT to download the actual thumbnail file." introductionVersion:"1.0.0"`
DataEndpoint string `yaml:"data_endpoint" env:"THUMBNAILS_DATA_ENDPOINT" desc:"The HTTP endpoint where the actual thumbnail file can be downloaded." introductionVersion:"1.0.0"`
MaxInputWidth int `yaml:"max_input_width" env:"THUMBNAILS_MAX_INPUT_WIDTH" desc:"The maximum width of an input image which is being processed." introductionVersion:"1.0.0"`
MaxInputHeight int `yaml:"max_input_height" env:"THUMBNAILS_MAX_INPUT_HEIGHT" desc:"The maximum height of an input image which is being processed." introductionVersion:"1.0.0"`
MaxInputImageFileSize string `yaml:"max_input_image_file_size" env:"THUMBNAILS_MAX_INPUT_IMAGE_FILE_SIZE" desc:"The maximum file size of an input image which is being processed. Usable common abbreviations: [KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB." introductionVersion:"1.0.0"`
}
@@ -0,0 +1,9 @@
package config
// Debug defines the available debug configuration.
type Debug struct {
Addr string `yaml:"addr" env:"THUMBNAILS_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
Token string `yaml:"token" env:"THUMBNAILS_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
Pprof bool `yaml:"pprof" env:"THUMBNAILS_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
Zpages bool `yaml:"zpages" env:"THUMBNAILS_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
}
@@ -0,0 +1,89 @@
package defaults
import (
"path"
"strings"
"github.com/qsfera/server/pkg/config/defaults"
"github.com/qsfera/server/pkg/shared"
"github.com/qsfera/server/pkg/structs"
"github.com/qsfera/server/services/thumbnails/pkg/config"
)
// FullDefaultConfig returns a fully initialized default configuration
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
Sanitize(cfg)
return cfg
}
// DefaultConfig returns a basic default configuration
func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9189",
Token: "",
Pprof: false,
Zpages: false,
},
GRPC: config.GRPCConfig{
Addr: "127.0.0.1:9185",
Namespace: "qsfera.api",
MaxConcurrentRequests: 0,
},
HTTP: config.HTTP{
Addr: "127.0.0.1:9186",
Root: "/thumbnails",
Namespace: "qsfera.web",
CORS: config.CORS{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With", "X-Request-Id", "Cache-Control"},
AllowCredentials: true,
},
},
Service: config.Service{
Name: "thumbnails",
},
Thumbnail: config.Thumbnail{
Resolutions: []string{"16x16", "32x32", "64x64", "128x128", "1080x1920", "1920x1080", "2160x3840", "3840x2160", "4320x7680", "7680x4320"},
FileSystemStorage: config.FileSystemStorage{
RootDirectory: path.Join(defaults.BaseDataPath(), "thumbnails"),
},
WebdavAllowInsecure: false,
RevaGateway: shared.DefaultRevaConfig().Address,
CS3AllowInsecure: false,
DataEndpoint: "http://127.0.0.1:9186/thumbnails/data",
MaxInputWidth: 7680,
MaxInputHeight: 7680,
MaxInputImageFileSize: "50MB",
},
}
}
// EnsureDefaults adds default values to the configuration if they are not set yet
func EnsureDefaults(cfg *config.Config) {
if cfg.LogLevel == "" {
cfg.LogLevel = "error"
}
if cfg.GRPCClientTLS == nil && cfg.Commons != nil {
cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS)
}
if cfg.GRPC.TLS == nil && cfg.Commons != nil {
cfg.GRPC.TLS = structs.CopyOrZeroValue(cfg.Commons.GRPCServiceTLS)
}
if cfg.Commons != nil {
cfg.HTTP.TLS = cfg.Commons.HTTPServiceTLS
}
}
// Sanitize sanitized the configuration
func Sanitize(cfg *config.Config) {
// nothing to sanitize here atm
if len(cfg.Thumbnail.Resolutions) == 1 && strings.Contains(cfg.Thumbnail.Resolutions[0], ",") {
cfg.Thumbnail.Resolutions = strings.Split(cfg.Thumbnail.Resolutions[0], ",")
}
}
@@ -0,0 +1,11 @@
package config
import "github.com/qsfera/server/pkg/shared"
// GRPCConfig defines the available grpc configuration.
type GRPCConfig struct {
Addr string `yaml:"addr" env:"THUMBNAILS_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
TLS *shared.GRPCServiceTLS `yaml:"tls"`
MaxConcurrentRequests int `yaml:"max_concurrent_requests" env:"THUMBNAILS_MAX_CONCURRENT_REQUESTS" desc:"Number of maximum concurrent thumbnail requests. Default is 0 which is unlimited." introductionVersion:"1.0.0"`
}
@@ -0,0 +1,20 @@
package config
import "github.com/qsfera/server/pkg/shared"
// CORS defines the available cors configuration.
type CORS struct {
AllowedOrigins []string `yaml:"allow_origins" env:"OC_CORS_ALLOW_ORIGINS;THUMBNAILS_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
AllowedMethods []string `yaml:"allow_methods" env:"OC_CORS_ALLOW_METHODS;THUMBNAILS_CORS_ALLOW_METHODS" desc:"A list of allowed CORS methods. See following chapter for more details: *Access-Control-Request-Method* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Method. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
AllowedHeaders []string `yaml:"allow_headers" env:"OC_CORS_ALLOW_HEADERS;THUMBNAILS_CORS_ALLOW_HEADERS" desc:"A list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
AllowCredentials bool `yaml:"allow_credentials" env:"OC_CORS_ALLOW_CREDENTIALS;THUMBNAILS_CORS_ALLOW_CREDENTIALS" desc:"Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials." introductionVersion:"1.0.0"`
}
// HTTP defines the available http configuration.
type HTTP struct {
Addr string `yaml:"addr" env:"THUMBNAILS_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
TLS shared.HTTPServiceTLS `yaml:"tls"`
Root string `yaml:"root" env:"THUMBNAILS_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
CORS CORS `yaml:"cors"`
}
@@ -0,0 +1,39 @@
package parser
import (
"errors"
occfg "github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/config/defaults"
"github.com/qsfera/server/pkg/config/envdecode"
)
// ParseConfig loads configuration from known paths.
func ParseConfig(cfg *config.Config) error {
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
defaults.EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
// sanitize config
defaults.Sanitize(cfg)
return Validate(cfg)
}
// Validate can validate the configuration
func Validate(_ *config.Config) error {
return nil
}
@@ -0,0 +1,6 @@
package config
// Service defines the available service configuration.
type Service struct {
Name string `yaml:"-"`
}
@@ -0,0 +1,22 @@
package errors
import "errors"
var (
// ErrImageTooLarge defines an error when an input image is too large
ErrImageTooLarge = errors.New("thumbnails: image is too large")
// ErrInvalidType represents the error when a type can't be encoded.
ErrInvalidType = errors.New("thumbnails: can't encode this type")
// ErrNoEncoderForType represents the error when an encoder couldn't be found for a type.
ErrNoEncoderForType = errors.New("thumbnails: no encoder for this type found")
// ErrNoGeneratorForType represents the error when a generator couldn't be found for a type.
ErrNoGeneratorForType = errors.New("thumbnails: no generator for this type found")
// ErrNoImageFromAudioFile defines an error when an image cannot be extracted from an audio file
ErrNoImageFromAudioFile = errors.New("thumbnails: could not extract image from audio file")
// ErrNoConverterForExtractedImageFromGgsFile defines an error when the extracted image from an ggs file could not be converted
ErrNoConverterForExtractedImageFromGgsFile = errors.New("thumbnails: could not find converter for image extracted from ggs file")
// ErrNoConverterForExtractedImageFromAudioFile defines an error when the extracted image from an audio file could not be converted
ErrNoConverterForExtractedImageFromAudioFile = errors.New("thumbnails: could not find converter for image extracted from audio file")
// ErrCS3AuthorizationMissing defines an error when the CS3 authorization is missing
ErrCS3AuthorizationMissing = errors.New("thumbnails: cs3source - authorization missing")
)
@@ -0,0 +1,67 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// Namespace defines the namespace for the defines metrics.
Namespace = "qsfera"
// Subsystem defines the subsystem for the defines metrics.
Subsystem = "thumbnails"
)
// Metrics defines the available metrics of this service.
type Metrics struct {
Counter *prometheus.CounterVec
Latency *prometheus.SummaryVec
Duration *prometheus.HistogramVec
BuildInfo *prometheus.GaugeVec
}
// New initializes the available metrics.
func New() *Metrics {
m := &Metrics{
Counter: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "getthumbnail_total",
Help: "How many GetThumbnail requests processed",
}, []string{}),
Latency: prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "getthumbnail_latency_microseconds",
Help: "GetThumbnail request latencies in microseconds",
}, []string{}),
Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "getthumbnail_duration_seconds",
Help: "GetThumbnail method requests time in seconds",
}, []string{}),
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "build_info",
Help: "Build information",
}, []string{"version"}),
}
_ = prometheus.Register(
m.Counter,
)
_ = prometheus.Register(
m.Latency,
)
_ = prometheus.Register(
m.Duration,
)
_ = prometheus.Register(
m.BuildInfo,
)
return m
}
@@ -0,0 +1,192 @@
package preprocessor
import (
"encoding/json"
"os"
"path/filepath"
"time"
"github.com/qsfera/server/pkg/sync"
"golang.org/x/image/font"
"golang.org/x/image/font/gofont/goregular"
"golang.org/x/image/font/opentype"
)
// FontMap maps a script with the target font to be used for that script
// It also uses a DefaultFont in case there isn't a matching script in the map
//
// For cases like Japanese where multiple scripts are used, we rely on the text
// analyzer to use the script which is unique to japanese (Hiragana or Katakana)
// even if it has to overwrite the "official" detected script (Han). This means
// that "Han" should be used just for chinese while "Hiragana" and "Katakana"
// should be used for japanese
type FontMap struct {
FontMap map[string]string `json:"fontMap"`
DefaultFont string `json:"defaultFont"`
}
// FontMapData contains the location of the loaded file (in FLoc) and the FontMap loaded
// from the file
type FontMapData struct {
FMap *FontMap
FLoc string
}
// LoadedFace contains the location of the font used, and the loaded face (font.Face)
// ready to be used
type LoadedFace struct {
FontFile string
Face font.Face
}
// FontLoader represents a FontLoader. Use the "NewFontLoader" to get a instance
type FontLoader struct {
faceCache sync.Cache
fontMapData *FontMapData
faceOpts *opentype.FaceOptions
}
// NewFontLoader creates a new FontLoader based on the fontMapFile. The FaceOptions will
// be the same for all the font loaded by this instance.
// Note that only the fonts described in the fontMapFile will be used.
//
// The fontMapFile has the following structure
//
// {
// "fontMap": {
// "Han": "packaged/myFont-CJK.otf",
// "Arabic": "packaged/myFont-Arab.otf",
// "Latin": "/fonts/regular/myFont.otf"
// }
// "defaultFont": "/fonts/regular/myFont.otf"
// }
//
// The fontMapFile contains paths to where the fonts are located in the FS.
// Absolute paths can be used as shown above. If a relative path is used,
// it will be relative to the fontMapFile location. This should make the
// packaging easier since all the fonts can be placed in the same directory
// where the fontMapFile is, or in inner directories.
func NewFontLoader(fontMapFile string, faceOpts *opentype.FaceOptions) (*FontLoader, error) {
fontMap := &FontMap{}
if fontMapFile != "" {
file, err := os.Open(fontMapFile)
if err != nil {
return nil, err
}
defer file.Close()
parser := json.NewDecoder(file)
if err = parser.Decode(fontMap); err != nil {
return nil, err
}
}
return &FontLoader{
faceCache: sync.NewCache(5),
fontMapData: &FontMapData{
FMap: fontMap,
FLoc: fontMapFile,
},
faceOpts: faceOpts,
}, nil
}
// LoadFaceForScript loads and returns the font face to be used for that script according to the
// FontMap set when the FontLoader was created. If the script doesn't have
// an associated font, a default font will be used. Note that the default font
// might not be able to handle properly the script
func (fl *FontLoader) LoadFaceForScript(script string) (*LoadedFace, error) {
var parsedFont *opentype.Font
var parsingError error
fontFile := fl.fontMapData.FMap.DefaultFont
if val, ok := fl.fontMapData.FMap.FontMap[script]; ok {
fontFile = val
}
if fontFile != "" && !filepath.IsAbs(fontFile) {
fontFile = filepath.Join(filepath.Dir(fl.fontMapData.FLoc), fontFile)
}
// if the face for the script isn't cached, load the font file and create a new face
cachedFace := fl.faceCache.Load(fontFile)
if cachedFace != nil {
return cachedFace.V.(*LoadedFace), nil
}
if fontFile == "" {
parsedFont, parsingError = opentype.Parse(goregular.TTF)
if parsingError != nil {
return nil, parsingError
}
} else {
// opentype.ParseReaderAt seems to require to keep the file opened
// so read the font file into memory
data, err := os.ReadFile(fontFile)
if err != nil {
return nil, err
}
parsedFont, parsingError = opentype.Parse(data)
if parsingError != nil {
return nil, parsingError
}
}
face, err := opentype.NewFace(parsedFont, fl.faceOpts)
if err != nil {
return nil, err
}
loadedFace := &LoadedFace{
FontFile: fontFile,
Face: face,
}
fl.faceCache.Store(fontFile, loadedFace, time.Now().Add(10*time.Minute))
return loadedFace, nil
}
// GetFaceOptSize returns face opt size
func (fl *FontLoader) GetFaceOptSize() float64 {
return fl.faceOpts.Size
}
// GetFaceOptDPI returns face opt DPI
func (fl *FontLoader) GetFaceOptDPI() float64 {
return fl.faceOpts.DPI
}
// GetScriptList returns script list
func (fl *FontLoader) GetScriptList() []string {
fontMap := fl.fontMapData.FMap.FontMap
arePresent := map[string]bool{
"Common": false,
"Inherited": false,
}
listSize := len(fontMap)
for key := range arePresent {
if _, inFontMap := fontMap[key]; inFontMap {
arePresent[key] = true
} else {
listSize++
}
}
keys := make([]string, listSize)
i := 0
for k := range fontMap {
keys[i] = k
i++
}
for script, isPresent := range arePresent {
if !isPresent {
keys[i] = script
i++
}
}
return keys
}
@@ -0,0 +1,350 @@
package preprocessor
import (
"archive/zip"
"bufio"
"bytes"
"encoding/base64"
"encoding/json"
"image"
"image/draw"
"image/gif"
"io"
"math"
"mime"
"strings"
"github.com/pkg/errors"
"golang.org/x/image/font"
"golang.org/x/image/font/opentype"
"golang.org/x/image/math/fixed"
"github.com/dhowden/tag"
thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// FileConverter is the interface for the file converter
type FileConverter interface {
Convert(r io.Reader) (any, error)
}
// GifDecoder is a converter for the gif file
type GifDecoder struct{}
// Convert reads the gif file and returns the thumbnail image
func (i GifDecoder) Convert(r io.Reader) (any, error) {
img, err := gif.DecodeAll(r)
if err != nil {
return nil, errors.Wrap(err, `could not decode the image`)
}
return img, nil
}
// GgsDecoder is a converter for the geogebra slides file
type GgsDecoder struct{ thumbnailpath string }
// Convert reads the ggs file and returns the thumbnail image
func (g GgsDecoder) Convert(r io.Reader) (any, error) {
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
if err != nil {
return nil, err
}
zipReader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
if err != nil {
return nil, err
}
for _, file := range zipReader.File {
if file.Name == g.thumbnailpath {
thumbnail, err := file.Open()
if err != nil {
return nil, err
}
converter := ForType("image/png", nil)
if converter == nil {
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromGgsFile
}
img, err := converter.Convert(thumbnail)
if err != nil {
return nil, errors.Wrap(err, `could not decode the image`)
}
return img, nil
}
}
return nil, errors.Errorf("%s not found", g.thumbnailpath)
}
// AudioDecoder is a converter for the audio file
type AudioDecoder struct{}
// Convert reads the audio file and extracts the thumbnail image from the id3 tag
func (i AudioDecoder) Convert(r io.Reader) (any, error) {
b, err := io.ReadAll(r)
if err != nil {
return nil, err
}
m, err := tag.ReadFrom(bytes.NewReader(b))
if err != nil {
return nil, err
}
picture := m.Picture()
if picture == nil {
return nil, thumbnailerErrors.ErrNoImageFromAudioFile
}
converter := ForType(picture.MIMEType, nil)
if converter == nil {
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
}
return converter.Convert(bytes.NewReader(picture.Data))
}
// TxtToImageConverter is a converter for the text file
type TxtToImageConverter struct {
fontLoader *FontLoader
}
// Convert reads the text file and renders it into a thumbnail image
func (t TxtToImageConverter) Convert(r io.Reader) (any, error) {
img := image.NewRGBA(image.Rect(0, 0, 640, 480))
imgBounds := img.Bounds()
draw.Draw(img, imgBounds, image.White, image.Point{}, draw.Src)
fontSizeAsInt := int(math.Ceil(t.fontLoader.GetFaceOptSize()))
margin := 10
minX := fixed.I(imgBounds.Min.X + margin)
maxX := fixed.I(imgBounds.Max.X - margin)
maxY := fixed.I(imgBounds.Max.Y - margin)
initialPoint := fixed.P(imgBounds.Min.X+margin, imgBounds.Min.Y+margin+fontSizeAsInt)
canvas := &font.Drawer{
Dst: img,
Src: image.Black,
Dot: initialPoint,
}
scriptList := t.fontLoader.GetScriptList()
textAnalyzer := NewTextAnalyzer(scriptList)
taOpts := AnalysisOpts{
UseMergeMap: true,
MergeMap: DefaultMergeMap,
}
scanner := bufio.NewScanner(r)
Scan: // Label for the scanner loop, so we can break it easily
for scanner.Scan() {
txt := scanner.Text()
height := fixed.I(fontSizeAsInt) // reset to default height
textResult := textAnalyzer.AnalyzeString(txt, taOpts)
textResult.MergeCommon(DefaultMergeMap)
for _, sRange := range textResult.ScriptRanges {
targetFontFace, err := t.fontLoader.LoadFaceForScript(sRange.TargetScript)
if err != nil {
return nil, err
}
// if the target script is "_unknown" it's expected that the loaded face
// uses the default font
faceHeight := targetFontFace.Face.Metrics().Height
if faceHeight > height {
height = faceHeight
}
canvas.Face = targetFontFace.Face
initialByte := sRange.Low
for _, sRangeSpace := range sRange.Spaces {
if canvas.Dot.Y > maxY {
break Scan
}
drawWord(canvas, textResult.Text[initialByte:sRangeSpace], minX, maxX, height, maxY)
initialByte = sRangeSpace
}
if initialByte <= sRange.High {
// some bytes left to be written
if canvas.Dot.Y > maxY {
break Scan
}
drawWord(canvas, textResult.Text[initialByte:sRange.High+1], minX, maxX, height, maxY)
}
}
canvas.Dot.X = minX
canvas.Dot.Y += height.Mul(fixed.Int26_6(1<<6 + 1<<5)) // height * 1.5
if canvas.Dot.Y > maxY {
break
}
}
return img, scanner.Err()
}
// GGPStruct is the layout of a ggp file (which is basically json)
type GGPStruct struct {
Sections []struct {
Cards []struct {
Element struct {
Image struct {
Base64Image string
}
}
}
}
}
// GgpDecoder is a converter for the geogebra pinboard file
type GgpDecoder struct{}
// Convert reads the ggp file and returns the first thumbnail image
func (j GgpDecoder) Convert(r io.Reader) (any, error) {
ggp := &GGPStruct{}
err := json.NewDecoder(r).Decode(ggp)
if err != nil {
return nil, err
}
elem, err := extractBase64ImageFromGGP(ggp)
if err != nil {
return nil, err
}
b, err := base64.StdEncoding.DecodeString(elem)
if err != nil {
return nil, err
}
img, _, err := image.Decode(bytes.NewReader(b))
return img, err
}
func extractBase64ImageFromGGP(ggp *GGPStruct) (string, error) {
if len(ggp.Sections) < 1 || len(ggp.Sections[0].Cards) < 1 {
return "", errors.New("cant find thumbnail in ggp file")
}
raw := strings.Split(ggp.Sections[0].Cards[0].Element.Image.Base64Image, "base64,")
if len(raw) < 2 {
return "", errors.New("cant decode ggp thumbnail")
}
return raw[1], nil
}
// Draw the word in the canvas. The mixX and maxX defines the drawable range
// (X axis) where the word can be drawn (in case the word is too big and doesn't
// fit in the canvas), and the incY defines the increment in the Y axis if we
// need to draw the word in a new line
//
// Note that the word will likely start with a white space char
func drawWord(canvas *font.Drawer, word string, minX, maxX, incY, maxY fixed.Int26_6) {
// calculate the actual measurement of the string at a given X position
measure := func(s string, dotX fixed.Int26_6) (min, max fixed.Int26_6) {
bbox, _ := canvas.BoundString(s)
return dotX + bbox.Min.X, dotX + bbox.Max.X
}
// first try to draw the whole word
absMin, absMax := measure(word, canvas.Dot.X)
if absMin >= minX && absMax <= maxX {
canvas.DrawString(word)
return
}
// try to draw the trimmed word in a new line
trimmed := strings.TrimSpace(word)
oldDot := canvas.Dot
canvas.Dot.X = minX
canvas.Dot.Y += incY
if canvas.Dot.Y <= maxY {
tMin, tMax := measure(trimmed, canvas.Dot.X)
if tMin >= minX && tMax <= maxX {
canvas.DrawString(trimmed)
return
}
}
// if the trimmed word is still too big, draw it char by char
canvas.Dot = oldDot
for _, char := range trimmed {
s := string(char)
_, cMax := measure(s, canvas.Dot.X)
if cMax > maxX {
canvas.Dot.X = minX
canvas.Dot.Y += incY
}
// stop drawing if we exceed maxY
if canvas.Dot.Y > maxY {
return
}
// ensure that we don't start drawing before minX
cMin, _ := measure(s, canvas.Dot.X)
if cMin < minX {
canvas.Dot.X += minX - cMin
}
canvas.DrawString(s)
}
}
// ForType returns the converter for the specified mimeType
func ForType(mimeType string, opts map[string]any) FileConverter {
// We can ignore the error here because we parse it in IsMimeTypeSupported before and if it fails
// return the service call. So we should only get here when the mimeType parses fine.
mimeType, _, _ = mime.ParseMediaType(mimeType)
switch mimeType {
case "text/plain":
fontFileMap := ""
fontFaceOpts := &opentype.FaceOptions{
Size: 12,
DPI: 72,
Hinting: font.HintingNone,
}
if optedFontFileMap, ok := opts["fontFileMap"]; ok {
if stringFontFileMap, ok := optedFontFileMap.(string); ok {
fontFileMap = stringFontFileMap
}
}
if optedFontFaceOpts, ok := opts["fontFaceOpts"]; ok {
if typedFontFaceOpts, ok := optedFontFaceOpts.(*opentype.FaceOptions); ok {
fontFaceOpts = typedFontFaceOpts
}
}
fontLoader, err := NewFontLoader(fontFileMap, fontFaceOpts)
if err != nil {
// if it couldn't create the FontLoader with the specified fontFileMap,
// try to use the default font
fontLoader, _ = NewFontLoader("", fontFaceOpts)
}
return TxtToImageConverter{
fontLoader: fontLoader,
}
case "application/vnd.geogebra.slides":
return GgsDecoder{"_slide0/geogebra_thumbnail.png"}
case "application/vnd.geogebra.pinboard":
return GgpDecoder{}
case "image/gif":
return GifDecoder{}
case "audio/flac":
fallthrough
case "audio/mpeg":
fallthrough
case "audio/ogg":
return AudioDecoder{}
default:
return ImageDecoder{}
}
}
@@ -0,0 +1,22 @@
//go:build !enable_vips
package preprocessor
import (
"io"
"github.com/kovidgoyal/imaging"
"github.com/pkg/errors"
)
// ImageDecoder is a converter for the image file
type ImageDecoder struct{}
// Convert reads the image file and returns the thumbnail image
func (i ImageDecoder) Convert(r io.Reader) (any, error) {
img, err := imaging.Decode(r, imaging.AutoOrientation(true))
if err != nil {
return nil, errors.Wrap(err, `could not decode the image`)
}
return img, nil
}
@@ -0,0 +1,197 @@
package preprocessor
import (
"bytes"
"io"
"os"
"testing"
"golang.org/x/image/font"
"golang.org/x/image/font/opentype"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestImageDecoder(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "ImageDecoder Suite")
}
var _ = Describe("ImageDecoder", func() {
Describe("ImageDecoder", func() {
var fileReader io.Reader
BeforeEach(func() {
fileContent, err := os.ReadFile("test_assets/noise.png")
if err != nil {
panic(err)
}
fileReader = bytes.NewReader(fileContent)
})
It("should decode an image", func() {
decoder := ImageDecoder{}
img, err := decoder.Convert(fileReader)
Expect(err).ToNot(HaveOccurred())
Expect(img).ToNot(BeNil())
})
It("should return an error if the image is invalid", func() {
decoder := ImageDecoder{}
img, err := decoder.Convert(bytes.NewReader([]byte("not an image")))
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
})
Describe("GifDecoder", func() {
var fileReader io.Reader
BeforeEach(func() {
fileContent, err := os.ReadFile("test_assets/noise.gif")
if err != nil {
panic(err)
}
fileReader = bytes.NewReader(fileContent)
})
It("should decode a gif", func() {
decoder := GifDecoder{}
img, err := decoder.Convert(fileReader)
Expect(err).ToNot(HaveOccurred())
Expect(img).ToNot(BeNil())
})
It("should return an error if the gif is invalid", func() {
decoder := GifDecoder{}
img, err := decoder.Convert(bytes.NewReader([]byte("not a gif")))
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
})
Describe("GgsDecoder", func() {
var fileReader io.Reader
BeforeEach(func() {
fileContent, err := os.ReadFile("test_assets/ggs_test.ggs")
if err != nil {
panic(err)
}
fileReader = bytes.NewReader(fileContent)
})
It("should decode a ggs", func() {
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
img, err := decoder.Convert(fileReader)
Expect(err).ToNot(HaveOccurred())
Expect(img).ToNot(BeNil())
})
It("should return an error if the ggs is invalid", func() {
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
img, err := decoder.Convert(bytes.NewReader([]byte("not a ggs")))
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
})
Describe("should decode audio", func() {
var fileReader io.Reader
It("should decode an audio", func() {
fileContent, err := os.ReadFile("test_assets/empty.mp3")
if err != nil {
panic(err)
}
fileReader = bytes.NewReader(fileContent)
decoder := AudioDecoder{}
img, err := decoder.Convert(fileReader)
Expect(err).ToNot(HaveOccurred())
Expect(img).ToNot(BeNil())
})
It("should decode an audio", func() {
fileContent, err := os.ReadFile("test_assets/empty_no_image.mp3")
if err != nil {
panic(err)
}
fileReader = bytes.NewReader(fileContent)
decoder := AudioDecoder{}
img, err := decoder.Convert(fileReader)
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
It("should return an error if the audio is invalid", func() {
decoder := AudioDecoder{}
img, err := decoder.Convert(bytes.NewReader([]byte("not an audio")))
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
})
Describe("should decode text", func() {
var decoder TxtToImageConverter
BeforeEach(func() {
fontFaceOpts := &opentype.FaceOptions{
Size: 12,
DPI: 72,
Hinting: font.HintingNone,
}
fontLoader, err := NewFontLoader("", fontFaceOpts)
if err != nil {
fontLoader, _ = NewFontLoader("", fontFaceOpts)
}
decoder = TxtToImageConverter{
fontLoader: fontLoader,
}
})
It("should decode a text", func() {
img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
Expect(err).ToNot(HaveOccurred())
Expect(img).ToNot(BeNil())
})
It("fails if the font is missing", func() {
decoder.fontLoader.fontMapData = &FontMapData{
FMap: &FontMap{
DefaultFont: "/some/unknown/font.otf",
},
}
img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
Expect(err).To(HaveOccurred())
Expect(img).To(BeNil())
})
})
Describe("test ForType", func() {
It("should return an ImageDecoder for image types", func() {
decoder := ForType("image/png", nil)
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
})
It("should return an GifDecoder for gif types", func() {
decoder := ForType("image/gif", nil)
Expect(decoder).To(BeAssignableToTypeOf(GifDecoder{}))
})
It("should return an GgsDecoder for ggs types", func() {
decoder := ForType("application/vnd.geogebra.ggs", nil)
// This will not return the expected ggsDecoder, but an ImageDecoder since ggs contains an embedded png.
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
})
It("should return an AudioDecoder for audio types", func() {
decoder := ForType("audio/mpeg", nil)
Expect(decoder).To(BeAssignableToTypeOf(AudioDecoder{}))
})
It("should return an TxtToImageConverter for text types", func() {
decoder := ForType("text/plain", nil)
Expect(decoder).To(BeAssignableToTypeOf(TxtToImageConverter{}))
})
It("should return an ImageDecoder for unknown types", func() {
decoder := ForType("unknown", nil)
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
})
})
})
@@ -0,0 +1,20 @@
//go:build enable_vips
package preprocessor
import (
"io"
"github.com/davidbyttow/govips/v2/vips"
)
func init() {
vips.LoggingSettings(nil, vips.LogLevelError)
}
type ImageDecoder struct{}
func (v ImageDecoder) Convert(r io.Reader) (interface{}, error) {
img, err := vips.NewImageFromReader(r)
return img, err
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 181 KiB

@@ -0,0 +1,309 @@
package preprocessor
import (
"unicode"
"unicode/utf8"
)
// Default list of scripts to be analyzed within the string.
//
// Scripts that aren't present in the list will be considered as part
// of the last "known" script. For example, if "Avestan" script (which isn't
// present) is preceeded by "Arabic" script, then the "Avestan" script will
// be considered as "Arabic"
//
// Punctuation symbols are usually considered part of the "Common" script
var DefaultScripts = []string{
"Arabic",
"Common",
"Devanagari",
"Han",
"Hangul",
"Hiragana",
"Inherited",
"Katakana",
"Latin",
}
// Convenient map[string]map[string]string type used to merge multiple
// scripts into one. This is mainly used for japanese language which uses
// "Han", "Hiragana" and "Katakana" scripts.
//
// The map contains the expected previous script as first key, the expected
// current script as second key, and the resulting script (if both keys
// match) as value
type MergeMap map[string]map[string]string
// The default mergeMap containing info for the japanese scripts
var DefaultMergeMap = MergeMap{
"Han": map[string]string{
"Hiragana": "Hiragana",
"Katakana": "Katakana",
},
"Hiragana": map[string]string{
"Han": "Hiragana",
"Katakana": "Hiragana",
},
"Katakana": map[string]string{
"Han": "Katakana",
"Hiragana": "Hiragana",
},
}
// Analysis options.
type AnalysisOpts struct {
UseMergeMap bool
MergeMap MergeMap
}
// A script range. The range should be attached to a string which could contain
// multiple scripts. The "TargetScript" will go from bytes "Low" to "High"
// (both inclusive), and contains a "RuneCount" number of runes or chars
// (mostly for debugging purposes).
// The Space contains the bytes (inside the range) that are considered as
// white space.
type ScriptRange struct {
Low, High int
Spaces []int
TargetScript string
RuneCount int
}
// The result of a text analysis. It contains the analyzed text, a list of
// script ranges (see the ScriptRange type) and a map containing how many
// runes have been detected for a particular script.
type TextAnalysis struct {
ScriptRanges []ScriptRange
RuneCount map[string]int
Text string
}
// The TextAnalyzer object contains private members. It should be created via
// "NewTextAnalyzer" function.
type TextAnalyzer struct {
scripts map[string]*unicode.RangeTable
scriptListCache []string
}
// Create a new TextAnalyzer. A list of scripts must be provided.
// You can use the "DefaultScripts" variable for a default list,
// although it doesn't contain all the available scripts.
// See the unicode.Scripts variable (in the unicode package) for a
// full list. Note that using invalid scripts will cause an undefined
// behavior
func NewTextAnalyzer(scriptList []string) TextAnalyzer {
scriptRanges := make(map[string]*unicode.RangeTable, len(scriptList))
for _, script := range scriptList {
scriptRanges[script] = unicode.Scripts[script]
}
return TextAnalyzer{
scripts: scriptRanges,
scriptListCache: scriptList,
}
}
// Analyze the target string using the specified options.
// A TextAnalysis will be returned with the result of the analysis.
func (ta *TextAnalyzer) AnalyzeString(word string, opts AnalysisOpts) TextAnalysis {
analysis := TextAnalysis{
ScriptRanges: []ScriptRange{},
RuneCount: make(map[string]int),
Text: word,
}
if len(word) < 1 {
return analysis
}
firstRune, runeLen := utf8.DecodeRuneInString(word)
lastRange := &ScriptRange{
Low: 0,
Spaces: make([]int, 0),
TargetScript: ta.chooseScriptFor(firstRune),
}
firstRuneIsWhiteSpace := unicode.Is(unicode.White_Space, firstRune)
if firstRuneIsWhiteSpace {
lastRange.Spaces = append(lastRange.Spaces, 0)
}
runeCount := 1
for wordIndex, char := range word[runeLen:] {
wordIndex += runeLen // shifted from the original string
script := ta.chooseScriptFor(char)
isWhiteSpace := unicode.Is(unicode.White_Space, char)
if script != lastRange.TargetScript {
if mapScript, isOk := ta.getMergeMapValue(opts, lastRange.TargetScript, script); isOk {
lastRange.TargetScript = mapScript
if isWhiteSpace {
// TODO: Check if this is dead code.
// whitespace should be part of the "Common" script, and the Common
// script shouldn't be part of a mergeMap
lastRange.Spaces = append(lastRange.Spaces, wordIndex)
}
runeCount++
continue
}
lastRange.High = wordIndex - 1
lastRange.RuneCount = runeCount
analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
if _, exists := analysis.RuneCount[lastRange.TargetScript]; !exists {
analysis.RuneCount[lastRange.TargetScript] = 0
}
analysis.RuneCount[lastRange.TargetScript] += runeCount
lastRange = &ScriptRange{
Low: wordIndex,
Spaces: make([]int, 0),
TargetScript: script,
}
runeCount = 0
}
runeCount++
if isWhiteSpace {
lastRange.Spaces = append(lastRange.Spaces, wordIndex)
}
}
// close the last range
lastRange.High = len(word) - 1
lastRange.RuneCount = runeCount
analysis.RuneCount[lastRange.TargetScript] += runeCount
analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
return analysis
}
func (ta *TextAnalyzer) chooseScriptFor(char rune) string {
script := "_unknown"
for scriptIndex, scriptFound := range ta.scriptListCache {
// if we can't match with a known script, do nothing and jump to the next char
if unicode.Is(ta.scripts[scriptFound], char) {
if scriptIndex > 3 {
// we might expect more chars with the same script
// so move the script first to match it faster next time
ta.reorderScriptList(scriptFound)
}
return scriptFound
}
}
return script
}
// Reorder the scriptListCache in the TextAnalyzer in order to speed up
// the next script searches. A "Latin" script is expected to be surrounded
// by "Latin" chars, although "Common" script chars might be present too
func (ta *TextAnalyzer) reorderScriptList(matchedScript string) {
for index, script := range ta.scriptListCache {
if script == matchedScript {
if index != 0 {
// move the script to the first position for a faster matching
newList := append([]string{script}, ta.scriptListCache[:index]...)
ta.scriptListCache = append(newList, ta.scriptListCache[index+1:]...)
}
// if index == 0 there is nothing to do: the element is already the first
break
}
}
}
// Get the value from the merge map based on the previous and current scripts.
// The information about using the merge map and the actual merge map will be
// gotten from the AnalysisOpts passed as parameter
func (ta *TextAnalyzer) getMergeMapValue(opts AnalysisOpts, previous, current string) (string, bool) {
if opts.UseMergeMap {
// This option mainly target japanese chars; multiple scripts can be used
// in the same piece of text (Han, Hiragana and Katakana)
// Instead of starting a new range, adjust the target script of the last range
if expCurrent, currentOk := opts.MergeMap[previous]; currentOk {
if expFinal, finalOk := expCurrent[current]; finalOk {
return expFinal, finalOk
}
}
}
return "", false
}
// Change the "Common" script to the one used in the previous script range.
// The ranges will be readjusted and merged if they're adjacent.
// This naive approach should be good enough for normal use cases
//
// The MergeMap is needed in case of the japanese language: the ranges
// "Han"-"Common"-"Katakana" might be replaced to "Han"-"Hiragana"-"Katakana"
// However, the ranges should be merged together into a big "Hiragana" range.
// If the MergeMap isn't needed, use an empty one
func (tr *TextAnalysis) MergeCommon(mergeMap MergeMap) {
var finalRanges []ScriptRange
if len(tr.ScriptRanges) < 1 {
// no ranges -> nothing to do
return
}
previousRange := &ScriptRange{}
*previousRange = tr.ScriptRanges[0]
for _, sRange := range tr.ScriptRanges[1:] {
if previousRange.TargetScript == sRange.TargetScript {
previousRange.High = sRange.High
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
previousRange.RuneCount += sRange.RuneCount
} else if sRange.TargetScript == "Common" || sRange.TargetScript == "Inherited" {
// new range will be absorbed into the previous one
previousRange.High = sRange.High
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
previousRange.RuneCount += sRange.RuneCount
tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
} else if previousRange.TargetScript == "Common" || previousRange.TargetScript == "Inherited" {
// might happen if the text starts with a Common script
previousRange.High = sRange.High
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
previousRange.RuneCount += sRange.RuneCount
previousRange.TargetScript = sRange.TargetScript
} else {
if mapScript, isOk := tr.getMergeMapValue(mergeMap, previousRange.TargetScript, sRange.TargetScript); isOk {
if sRange.TargetScript == mapScript {
// the previous range has changed the target script
tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
} else {
// new range has been absorbed
tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
}
previousRange.TargetScript = mapScript
previousRange.High = sRange.High
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
previousRange.RuneCount += sRange.RuneCount
continue
}
finalRanges = append(finalRanges, *previousRange)
*previousRange = sRange
}
}
finalRanges = append(finalRanges, *previousRange)
tr.ScriptRanges = finalRanges
delete(tr.RuneCount, "Common")
delete(tr.RuneCount, "Inherited")
for index, rCount := range tr.RuneCount {
if rCount == 0 {
delete(tr.RuneCount, index)
}
}
}
func (tr *TextAnalysis) getMergeMapValue(mMap MergeMap, previous, current string) (string, bool) {
// This option mainly target japanese chars; multiple scripts can be used
// in the same piece of text (Han, Hiragana and Katakana)
// Instead of starting a new range, adjust the target script of the last range
if expCurrent, currentOk := mMap[previous]; currentOk {
if expFinal, finalOk := expCurrent[current]; finalOk {
return expFinal, finalOk
}
}
return "", false
}
@@ -0,0 +1,614 @@
package preprocessor
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
var (
inputs = [18]string{
"basic latin",
"trailing tab ",
"Small text. \"$\", \"£\" and \"¥\" are currencies.",
"latin with 🖖",
"기본 한국어",
"基本的な日本語",
"ウーロン茶",
"私はエンジニアです",
"ティー私はエンジニアです",
"私はエンジニアです ティー",
"आधारभूत देवनागरी",
"mixed 언어 传入 🚀!",
"/k͜p/",
// ä and a + ¨
"ä ä",
"базовый русский", // cyrillic script isn't part of our default
"latin русский", // latin + cyrillic (cyrillic not supported)
" space justified ",
"",
}
)
func TestAnalyzeString(t *testing.T) {
defaultOpts := AnalysisOpts{
UseMergeMap: true,
MergeMap: DefaultMergeMap,
}
tables := []struct {
input string
opts AnalysisOpts
eOut TextAnalysis
}{
{
input: inputs[0],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 10, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 11},
},
RuneCount: map[string]int{
"Latin": 11,
},
Text: inputs[0],
},
},
{
input: inputs[1],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 12, Spaces: []int{8, 12}, TargetScript: "Latin", RuneCount: 13},
},
RuneCount: map[string]int{
"Latin": 13,
},
Text: inputs[1],
},
},
{
input: inputs[2],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 45, Spaces: []int{5, 11, 16, 21, 25, 30, 34}, TargetScript: "Latin", RuneCount: 44},
},
RuneCount: map[string]int{
"Latin": 44,
},
Text: inputs[2],
},
},
{
input: inputs[3],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 14, Spaces: []int{5, 10}, TargetScript: "Latin", RuneCount: 12},
},
RuneCount: map[string]int{
"Latin": 12,
},
Text: inputs[3],
},
},
{
input: inputs[4],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 15, Spaces: []int{6}, TargetScript: "Hangul", RuneCount: 6},
},
RuneCount: map[string]int{
"Hangul": 6,
},
Text: inputs[4],
},
},
{
input: inputs[5],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 7},
},
RuneCount: map[string]int{
"Hiragana": 7,
},
Text: inputs[5],
},
},
{
input: inputs[6],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 14, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
},
RuneCount: map[string]int{
"Katakana": 5,
},
Text: inputs[6],
},
},
{
input: inputs[7],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 9},
},
RuneCount: map[string]int{
"Hiragana": 9,
},
Text: inputs[7],
},
},
{
input: inputs[8],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 12},
},
RuneCount: map[string]int{
"Hiragana": 12,
},
Text: inputs[8],
},
},
{
input: inputs[9],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 36, Spaces: []int{27}, TargetScript: "Hiragana", RuneCount: 13},
},
RuneCount: map[string]int{
"Hiragana": 13,
},
Text: inputs[9],
},
},
{
input: inputs[10],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 45, Spaces: []int{21}, TargetScript: "Devanagari", RuneCount: 16},
},
RuneCount: map[string]int{
"Devanagari": 16,
},
Text: inputs[10],
},
},
{
input: inputs[11],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
{Low: 6, High: 12, Spaces: []int{12}, TargetScript: "Hangul", RuneCount: 3},
{Low: 13, High: 24, Spaces: []int{19}, TargetScript: "Han", RuneCount: 5}, // 🚀 and ! are "Common" script and will be merged with "Han"
},
RuneCount: map[string]int{
"Latin": 6,
"Hangul": 3,
"Han": 5,
},
Text: inputs[11],
},
},
{
input: inputs[12],
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
},
RuneCount: map[string]int{
"Latin": 5,
},
Text: inputs[12],
},
},
{
input: inputs[13], // ä and a + ¨
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{2}, TargetScript: "Latin", RuneCount: 4},
},
RuneCount: map[string]int{
"Latin": 4,
},
Text: inputs[13],
},
},
{
input: inputs[14], // cyrillic script isn't part of our default
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 28, Spaces: []int{14}, TargetScript: "_unknown", RuneCount: 15},
},
RuneCount: map[string]int{
"_unknown": 15,
},
Text: inputs[14],
},
},
{
input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
},
RuneCount: map[string]int{
"Latin": 6,
"_unknown": 7,
},
Text: inputs[15],
},
},
{
input: inputs[16], // latin + cyrillic (cyrillic script isn't part of our default)
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 16, Spaces: []int{0, 6, 16}, TargetScript: "Latin", RuneCount: 17},
},
RuneCount: map[string]int{
"Latin": 17,
},
Text: inputs[16],
},
},
{
input: inputs[17], // latin + cyrillic (cyrillic script isn't part of our default)
opts: defaultOpts,
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{},
RuneCount: map[string]int{},
Text: inputs[17],
},
},
}
for _, table := range tables {
testname := fmt.Sprintf("Analyzing \"%s\" string", table.input)
t.Run(testname, func(t *testing.T) {
ta := NewTextAnalyzer(DefaultScripts)
result := ta.AnalyzeString(table.input, table.opts)
if table.opts.UseMergeMap {
result.MergeCommon(table.opts.MergeMap)
} else {
result.MergeCommon(MergeMap{})
}
assert.Equal(t, table.eOut, result)
})
}
}
func TestAnalyzeStringRaw(t *testing.T) {
tables := []struct {
input string
eOut TextAnalysis
}{
{
input: inputs[0],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
{Low: 6, High: 10, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
},
RuneCount: map[string]int{
"Latin": 10,
"Common": 1,
},
Text: inputs[0],
},
},
{
input: inputs[1],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 7, Spaces: []int{}, TargetScript: "Latin", RuneCount: 8},
{Low: 8, High: 8, Spaces: []int{8}, TargetScript: "Common", RuneCount: 1},
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
},
RuneCount: map[string]int{
"Latin": 11,
"Common": 2,
},
Text: inputs[1],
},
},
{
input: inputs[2],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
{Low: 10, High: 21, Spaces: []int{11, 16, 21}, TargetScript: "Common", RuneCount: 11}, // £ takes 2 bytes
{Low: 22, High: 24, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
{Low: 25, High: 30, Spaces: []int{25, 30}, TargetScript: "Common", RuneCount: 5}, // ¥ takes 2 bytes
{Low: 31, High: 33, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
{Low: 34, High: 34, Spaces: []int{34}, TargetScript: "Common", RuneCount: 1},
{Low: 35, High: 44, Spaces: []int{}, TargetScript: "Latin", RuneCount: 10},
{Low: 45, High: 45, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
},
RuneCount: map[string]int{
"Latin": 25,
"Common": 19,
},
Text: inputs[2],
},
},
{
input: inputs[3],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
{Low: 10, High: 14, Spaces: []int{10}, TargetScript: "Common", RuneCount: 2},
},
RuneCount: map[string]int{
"Latin": 9,
"Common": 3,
},
Text: inputs[3],
},
},
{
input: inputs[4],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 3},
},
RuneCount: map[string]int{
"Hangul": 5,
"Common": 1,
},
Text: inputs[4],
},
},
{
input: inputs[5],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 8, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
{Low: 12, High: 20, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
},
RuneCount: map[string]int{
"Hiragana": 1,
"Han": 6,
},
Text: inputs[5],
},
},
{
input: inputs[6],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 1},
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1}, // ー U+30FC (KATAKANA-HIRAGANA PROLONGED SOUND MARK) seems to be counted as Common
{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
},
RuneCount: map[string]int{
"Katakana": 3,
"Common": 1,
"Han": 1,
},
Text: inputs[6],
},
},
{
input: inputs[7],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
},
RuneCount: map[string]int{
"Han": 1,
"Hiragana": 3,
"Katakana": 5,
},
Text: inputs[7],
},
},
{
input: inputs[8],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
{Low: 6, High: 8, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
{Low: 15, High: 29, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
{Low: 30, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
},
RuneCount: map[string]int{
"Han": 1,
"Hiragana": 3,
"Katakana": 7,
"Common": 1,
},
Text: inputs[8],
},
},
{
input: inputs[9],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
{Low: 27, High: 27, Spaces: []int{27}, TargetScript: "Common", RuneCount: 1},
{Low: 28, High: 33, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
{Low: 34, High: 36, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
},
RuneCount: map[string]int{
"Han": 1,
"Hiragana": 3,
"Katakana": 7,
"Common": 2,
},
Text: inputs[9],
},
},
{
input: inputs[10],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 7},
{Low: 21, High: 21, Spaces: []int{21}, TargetScript: "Common", RuneCount: 1},
{Low: 22, High: 45, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 8},
},
RuneCount: map[string]int{
"Devanagari": 15,
"Common": 1,
},
Text: inputs[10],
},
},
{
input: inputs[11],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
{Low: 13, High: 18, Spaces: []int{}, TargetScript: "Han", RuneCount: 2},
{Low: 19, High: 24, Spaces: []int{19}, TargetScript: "Common", RuneCount: 3},
},
RuneCount: map[string]int{
"Latin": 5,
"Hangul": 2,
"Han": 2,
"Common": 5,
},
Text: inputs[11],
},
},
{
input: inputs[12],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 0, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
{Low: 1, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
{Low: 2, High: 3, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
{Low: 4, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
{Low: 5, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
},
RuneCount: map[string]int{
"Latin": 2,
"Common": 2,
"Inherited": 1,
},
Text: inputs[12],
},
},
{
input: inputs[13], // ä and a + ¨
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
{Low: 2, High: 2, Spaces: []int{2}, TargetScript: "Common", RuneCount: 1},
{Low: 3, High: 3, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
{Low: 4, High: 5, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
},
RuneCount: map[string]int{
"Latin": 2,
"Common": 1,
"Inherited": 1,
},
Text: inputs[13],
},
},
{
input: inputs[14], // cyrillic script isn't part of our default
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 13, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
{Low: 14, High: 14, Spaces: []int{14}, TargetScript: "Common", RuneCount: 1},
{Low: 15, High: 28, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
},
RuneCount: map[string]int{
"_unknown": 14,
"Common": 1,
},
Text: inputs[14],
},
},
{
input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
},
RuneCount: map[string]int{
"Latin": 5,
"Common": 1,
"_unknown": 7,
},
Text: inputs[15],
},
},
{
input: inputs[16],
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{
{Low: 0, High: 0, Spaces: []int{0}, TargetScript: "Common", RuneCount: 1},
{Low: 1, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Latin", RuneCount: 9},
{Low: 16, High: 16, Spaces: []int{16}, TargetScript: "Common", RuneCount: 1},
},
RuneCount: map[string]int{
"Latin": 14,
"Common": 3,
},
Text: inputs[16],
},
},
{
input: inputs[17], // empty string
eOut: TextAnalysis{
ScriptRanges: []ScriptRange{},
RuneCount: map[string]int{},
Text: inputs[17],
},
},
}
for _, table := range tables {
testname := fmt.Sprintf("Raw-Analyzing \"%s\" string", table.input)
t.Run(testname, func(t *testing.T) {
ta := NewTextAnalyzer(DefaultScripts)
result := ta.AnalyzeString(table.input, AnalysisOpts{})
assert.Equal(t, table.eOut, result)
})
}
}
@@ -0,0 +1,52 @@
package debug
import (
"context"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Address string
Logger log.Logger
Context context.Context
Config *config.Config
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
@@ -0,0 +1,38 @@
package debug
import (
"net/http"
"github.com/qsfera/server/pkg/checks"
"github.com/qsfera/server/pkg/handlers"
"github.com/qsfera/server/pkg/service/debug"
"github.com/qsfera/server/pkg/version"
)
// Server initializes the debug service and server.
func Server(opts ...Option) (*http.Server, error) {
options := newOptions(opts...)
checkHandler := handlers.NewCheckHandler(
handlers.NewCheckHandlerConfiguration().
WithLogger(options.Logger).
WithCheck("web reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr)).
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr)),
)
return debug.NewService(
debug.Logger(options.Logger),
debug.Name(options.Config.Service.Name),
debug.Version(version.GetString()),
debug.Address(options.Config.Debug.Addr),
debug.Token(options.Config.Debug.Token),
debug.Pprof(options.Config.Debug.Pprof),
debug.Zpages(options.Config.Debug.Zpages),
debug.Health(checkHandler),
debug.Ready(checkHandler),
debug.CorsAllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins),
debug.CorsAllowedMethods(options.Config.HTTP.CORS.AllowedMethods),
debug.CorsAllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders),
debug.CorsAllowCredentials(options.Config.HTTP.CORS.AllowCredentials),
), nil
}
@@ -0,0 +1,100 @@
package grpc
import (
"context"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
"go.opentelemetry.io/otel/trace"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Address string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Namespace string
TraceProvider trace.TracerProvider
MaxConcurrentRequests int
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Name provides a name for the service.
func Name(val string) Option {
return func(o *Options) {
o.Name = val
}
}
// Address provides an address for the service.
func Address(val string) Option {
return func(o *Options) {
o.Address = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
return func(o *Options) {
o.Metrics = val
}
}
// Namespace provides a function to set the namespace option.
func Namespace(val string) Option {
return func(o *Options) {
o.Namespace = val
}
}
// TraceProvider provides a function to set the TraceProvider option
func TraceProvider(val trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = val
}
}
// MaxConcurrentRequests provides a function to set the MaxConcurrentRequests option.
func MaxConcurrentRequests(val int) Option {
return func(o *Options) {
o.MaxConcurrentRequests = val
}
}
@@ -0,0 +1,92 @@
package grpc
import (
"github.com/qsfera/server/pkg/registry"
"github.com/qsfera/server/pkg/service/grpc"
"github.com/qsfera/server/pkg/service/grpc/handler/ratelimiter"
"github.com/qsfera/server/pkg/version"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
svc "github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0"
"github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0/decorators"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
// NewService initializes the grpc service and server.
func NewService(opts ...Option) grpc.Service {
options := newOptions(opts...)
service, err := grpc.NewServiceWithClient(
options.Config.GrpcClient,
grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled),
grpc.TLSCert(
options.Config.GRPC.TLS.Cert,
options.Config.GRPC.TLS.Key,
),
grpc.Logger(options.Logger),
grpc.Namespace(options.Namespace),
grpc.Name(options.Name),
grpc.Version(version.GetString()),
grpc.Address(options.Address),
grpc.Context(options.Context),
grpc.Version(version.GetString()),
grpc.TraceProvider(options.TraceProvider),
grpc.HandlerWrappers(ratelimiter.NewHandlerWrapper(options.MaxConcurrentRequests)),
)
if err != nil {
options.Logger.Fatal().Err(err).Msg("Error creating thumbnail service")
return grpc.Service{}
}
tconf := options.Config.Thumbnail
tm, err := pool.StringToTLSMode(options.Config.GRPCClientTLS.Mode)
if err != nil {
options.Logger.Error().Err(err).Msg("could not get gateway client tls mode")
return grpc.Service{}
}
gatewaySelector, err := pool.GatewaySelector(tconf.RevaGateway,
pool.WithTLSCACert(options.Config.GRPCClientTLS.CACert),
pool.WithTLSMode(tm),
pool.WithRegistry(registry.GetRegistry()),
pool.WithTracerProvider(options.TraceProvider),
)
if err != nil {
options.Logger.Error().Err(err).Msg("could not get gateway selector")
return grpc.Service{}
}
b, err := bytesize.Parse(tconf.MaxInputImageFileSize)
if err != nil {
options.Logger.Error().Err(err).Msg("could not parse MaxInputImageFileSize")
return grpc.Service{}
}
var thumbnail decorators.DecoratedService
{
thumbnail = svc.NewService(
svc.Config(options.Config),
svc.Logger(options.Logger),
svc.ThumbnailSource(imgsource.NewWebDavSource(tconf, b)),
svc.ThumbnailStorage(
storage.NewFileSystemStorage(
tconf.FileSystemStorage,
options.Logger,
),
),
svc.CS3Source(imgsource.NewCS3Source(tconf, gatewaySelector, b)),
svc.GatewaySelector(gatewaySelector),
)
thumbnail = decorators.NewInstrument(thumbnail, options.Metrics)
thumbnail = decorators.NewLogging(thumbnail, options.Logger)
thumbnail = decorators.NewTracing(thumbnail, options.TraceProvider)
}
_ = thumbnailssvc.RegisterThumbnailServiceHandler(
service.Server(),
thumbnail,
)
return service
}
@@ -0,0 +1,91 @@
package http
import (
"context"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
"github.com/spf13/pflag"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Namespace string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Flags []pflag.Flag
TraceProvider trace.TracerProvider
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
return func(o *Options) {
o.Metrics = val
}
}
// Namespace provides a function to set the Namespace option.
func Namespace(val string) Option {
return func(o *Options) {
o.Namespace = val
}
}
// TraceProvider provides a function to configure the trace provider
func TraceProvider(traceProvider trace.TracerProvider) Option {
return func(o *Options) {
if traceProvider != nil {
o.TraceProvider = traceProvider
} else {
o.TraceProvider = noop.NewTracerProvider()
}
}
}
// Flags provides a function to set the flags option.
func Flags(flags ...pflag.Flag) Option {
return func(o *Options) {
o.Flags = append(o.Flags, flags...)
}
}
@@ -0,0 +1,74 @@
package http
import (
"fmt"
"github.com/go-chi/chi/v5/middleware"
"github.com/qsfera/server/pkg/cors"
qsferamiddleware "github.com/qsfera/server/pkg/middleware"
"github.com/qsfera/server/pkg/service/http"
"github.com/qsfera/server/pkg/version"
svc "github.com/qsfera/server/services/thumbnails/pkg/service/http/v0"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
"go-micro.dev/v4"
)
// Server initializes the http service and server.
func Server(opts ...Option) (http.Service, error) {
options := newOptions(opts...)
service, err := http.NewService(
http.TLSConfig(options.Config.HTTP.TLS),
http.Logger(options.Logger),
http.Name(options.Config.Service.Name),
http.Version(version.GetString()),
http.Namespace(options.Config.HTTP.Namespace),
http.Address(options.Config.HTTP.Addr),
http.Context(options.Context),
http.TraceProvider(options.TraceProvider),
)
if err != nil {
options.Logger.Error().
Err(err).
Msg("Error initializing http service")
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
}
handle := svc.NewService(
svc.Logger(options.Logger),
svc.Config(options.Config),
svc.Middleware(
middleware.RealIP,
middleware.RequestID,
qsferamiddleware.Cors(
cors.Logger(options.Logger),
cors.AllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins),
cors.AllowedMethods(options.Config.HTTP.CORS.AllowedMethods),
cors.AllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders),
cors.AllowCredentials(options.Config.HTTP.CORS.AllowCredentials),
),
qsferamiddleware.Version(
options.Config.Service.Name,
version.GetString(),
),
qsferamiddleware.Logger(options.Logger),
),
svc.ThumbnailStorage(
storage.NewFileSystemStorage(
options.Config.Thumbnail.FileSystemStorage,
options.Logger,
),
),
)
{
handle = svc.NewInstrument(handle, options.Metrics)
handle = svc.NewLogging(handle, options.Logger)
}
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
return http.Service{}, err
}
return service, nil
}
@@ -0,0 +1,63 @@
package decorators
import (
"context"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
)
// DecoratedService is an interface acting as facade, holding all the interfaces that this
// thumbnails microservice is expecting to implement.
// For now, only the thumbnailssvc.ThumbnailServiceHandler is present,
// but a future configsvc.ConfigServiceHandler is expected to be added here
//
// This interface will also act as the base interface to implement
// a decorator pattern.
type DecoratedService interface {
thumbnailssvc.ThumbnailServiceHandler
}
// Decorator is the base type to implement the decorators. It will provide a basic implementation
// by delegating to the decoratedService
//
// Expected implementations will be like:
// ```
//
// type MyDecorator struct {
// Decorator
// myCustomOpts *opts
// additionalSrv *srv
// }
//
// func NewMyDecorator(next DecoratedService, customOpts *customOpts) DecoratedService {
// .....
// return MyDecorator{
// Decorator: Decorator{next: next},
// myCustomOpts: opts,
// additionalSrv: srv,
// }
// }
//
// ```
type Decorator struct {
next DecoratedService
}
// GetThumbnail is the base implementation for the thumbnailssvc.GetThumbnail.
// It will just delegate to the underlying decoratedService
//
// Your custom decorator is expected to overwrite this function,
// but it MUST call the underlying decoratedService at some point
// ```
//
// func (d MyDecorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
// doSomething()
// err := d.next.GetThumbnail(ctx, req, resp)
// doAnotherThing()
// return err
// }
//
// ```
func (deco Decorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
return deco.next.GetThumbnail(ctx, req, resp)
}
@@ -0,0 +1,39 @@
package decorators
import (
"context"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
"github.com/prometheus/client_golang/prometheus"
)
// NewInstrument returns a service that instruments metrics.
func NewInstrument(next DecoratedService, metrics *metrics.Metrics) DecoratedService {
return instrument{
Decorator: Decorator{next: next},
metrics: metrics,
}
}
type instrument struct {
Decorator
metrics *metrics.Metrics
}
// GetThumbnail implements the ThumbnailServiceHandler interface.
func (i instrument) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
us := v * 1000_000
i.metrics.Latency.WithLabelValues().Observe(us)
i.metrics.Duration.WithLabelValues().Observe(v)
}))
defer timer.ObserveDuration()
err := i.next.GetThumbnail(ctx, req, rsp)
if err != nil {
i.metrics.Counter.WithLabelValues().Inc()
}
return err
}
@@ -0,0 +1,53 @@
package decorators
import (
"context"
"net/http"
"time"
"github.com/qsfera/server/pkg/log"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
merrors "go-micro.dev/v4/errors"
)
// NewLogging returns a service that logs messages.
func NewLogging(next DecoratedService, logger log.Logger) DecoratedService {
return logging{
Decorator: Decorator{next: next},
logger: logger,
}
}
type logging struct {
Decorator
logger log.Logger
}
// GetThumbnail implements the ThumbnailServiceHandler interface.
func (l logging) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
start := time.Now()
err := l.next.GetThumbnail(ctx, req, rsp)
logger := l.logger.With().
Str("method", "Thumbnails.GetThumbnail").
Dur("duration", time.Since(start)).
Logger()
if err != nil {
fromError := merrors.FromError(err)
switch fromError.GetCode() {
case http.StatusNotFound:
logger.Debug().
Str("error_detail", fromError.GetDetail()).
Msg("no thumbnail found")
default:
logger.Warn().
Err(err).
Msg("Failed to execute")
}
} else {
logger.Debug().
Msg("")
}
return err
}
@@ -0,0 +1,46 @@
package decorators
import (
"context"
"go.opentelemetry.io/otel/trace"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
"go.opentelemetry.io/otel/attribute"
)
// NewTracing returns a service that instruments traces.
func NewTracing(next DecoratedService, tp trace.TracerProvider) DecoratedService {
return tracing{
Decorator: Decorator{next: next},
tp: tp,
}
}
type tracing struct {
Decorator
tp trace.TracerProvider
}
// GetThumbnail implements the ThumbnailServiceHandler interface.
func (t tracing) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
var span trace.Span
if t.tp != nil {
tracer := t.tp.Tracer("thumbnails")
spanOpts := []trace.SpanStartOption{
trace.WithSpanKind(trace.SpanKindServer),
}
ctx, span = tracer.Start(ctx, "Thumbnails.GetThumbnail", spanOpts...)
defer span.End()
span.SetAttributes(
attribute.KeyValue{Key: "filepath", Value: attribute.StringValue(req.GetFilepath())},
attribute.KeyValue{Key: "thumbnail_type", Value: attribute.StringValue(req.GetThumbnailType().String())},
attribute.KeyValue{Key: "width", Value: attribute.IntValue(int(req.GetWidth()))},
attribute.KeyValue{Key: "height", Value: attribute.IntValue(int(req.GetHeight()))},
)
}
return t.next.GetThumbnail(ctx, req, rsp)
}
@@ -0,0 +1,79 @@
package svc
import (
"net/http"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Config *config.Config
Middleware []func(http.Handler) http.Handler
ThumbnailStorage storage.Storage
ImageSource imgsource.Source
CS3Source imgsource.Source
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// ThumbnailStorage provides a function to set the thumbnail storage option.
func ThumbnailStorage(val storage.Storage) Option {
return func(o *Options) {
o.ThumbnailStorage = val
}
}
// ThumbnailSource provides a function to set the image source option.
func ThumbnailSource(val imgsource.Source) Option {
return func(o *Options) {
o.ImageSource = val
}
}
// CS3Source provides a function to set the CS3Source option
func CS3Source(val imgsource.Source) Option {
return func(o *Options) {
o.CS3Source = val
}
}
// GatewaySelector adds a grpc client selector for the gateway service
func GatewaySelector(val pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = val
}
}
@@ -0,0 +1,331 @@
package svc
import (
"context"
"net/http"
"net/url"
"path"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/golang-jwt/jwt/v5"
"github.com/pkg/errors"
merrors "go-micro.dev/v4/errors"
"google.golang.org/grpc/metadata"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/qsfera/server/pkg/log"
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
terrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
"github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0/decorators"
tjwt "github.com/qsfera/server/services/thumbnails/pkg/service/jwt"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
)
// NewService returns a service implementation for Service.
func NewService(opts ...Option) decorators.DecoratedService {
options := newOptions(opts...)
logger := options.Logger
resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions)
if err != nil {
logger.Fatal().Err(err).Msg("resolutions not configured correctly")
}
svc := Thumbnail{
serviceID: options.Config.GRPC.Namespace + "." + options.Config.Service.Name,
manager: thumbnail.NewSimpleManager(
resolutions,
options.ThumbnailStorage,
logger,
options.Config.Thumbnail.MaxInputWidth,
options.Config.Thumbnail.MaxInputHeight,
),
webdavSource: options.ImageSource,
cs3Source: options.CS3Source,
logger: logger,
selector: options.GatewaySelector,
preprocessorOpts: PreprocessorOpts{
TxtFontFileMap: options.Config.Thumbnail.FontMapFile,
},
dataEndpoint: options.Config.Thumbnail.DataEndpoint,
transferSecret: options.Config.Thumbnail.TransferSecret,
}
return svc
}
// Thumbnail implements the GRPC handler.
type Thumbnail struct {
serviceID string
dataEndpoint string
transferSecret string
manager thumbnail.Manager
webdavSource imgsource.Source
cs3Source imgsource.Source
logger log.Logger
selector pool.Selectable[gateway.GatewayAPIClient]
preprocessorOpts PreprocessorOpts
}
// PreprocessorOpts holds the options for the preprocessor
type PreprocessorOpts struct {
TxtFontFileMap string
}
// GetThumbnail retrieves a thumbnail for an image
func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
var err error
var key string
switch {
case req.GetWebdavSource() != nil:
key, err = g.handleWebdavSource(ctx, req)
case req.GetCs3Source() != nil:
key, err = g.handleCS3Source(ctx, req)
default:
g.logger.Error().Msg("no image source provided")
return merrors.BadRequest(g.serviceID, "image source is missing")
}
if err != nil {
return err
}
claims := tjwt.ThumbnailClaims{
Key: key,
RegisteredClaims: jwt.RegisteredClaims{
IssuedAt: jwt.NewNumericDate(time.Now()),
ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Minute)),
},
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
transferToken, err := token.SignedString([]byte(g.transferSecret))
if err != nil {
g.logger.Error().
Err(err).
Msg("GetThumbnail: failed to sign token")
return merrors.InternalServerError(g.serviceID, "couldn't finish request")
}
rsp.DataEndpoint = g.dataEndpoint
rsp.TransferToken = transferToken
return nil
}
func (g Thumbnail) checkThumbnail(req *thumbnailssvc.GetThumbnailRequest, sRes *provider.StatResponse) (string, thumbnail.Request, error) {
tr := thumbnail.Request{}
if !sRes.GetInfo().GetPermissionSet().GetInitiateFileDownload() {
return "", tr, merrors.Forbidden(g.serviceID, "no download permission")
}
tType := thumbnail.GetExtForMime(sRes.GetInfo().GetMimeType())
if tType == "" {
tType = req.GetThumbnailType().String()
}
tr, err := thumbnail.PrepareRequest(int(req.GetWidth()), int(req.GetHeight()), tType, sRes.GetInfo().GetChecksum().GetSum(), req.GetProcessor())
if err != nil {
return "", tr, merrors.BadRequest(g.serviceID, "%s", err.Error())
}
if key, exists := g.manager.CheckThumbnail(tr); exists {
return key, tr, nil
}
return "", tr, nil
}
func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
src := req.GetCs3Source()
sRes, err := g.stat(src.GetPath(), src.GetAuthorization())
if err != nil {
return "", err
}
key, tr, err := g.checkThumbnail(req, sRes)
switch {
case err != nil:
return "", err
case key != "":
// we have matching thumbnail already, use that
return key, nil
}
ctx = imgsource.ContextSetAuthorization(ctx, src.GetAuthorization())
r, err := g.cs3Source.Get(ctx, src.GetPath())
switch {
case errors.Is(err, terrors.ErrImageTooLarge):
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
case err != nil:
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
defer r.Close()
ppOpts := map[string]any{
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
}
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
img, err := pp.Convert(r)
if err != nil {
g.logger.Error().Err(err).Msg("failed to convert image")
}
if img == nil || err != nil {
return "", merrors.NotFound(g.serviceID, "could not get image")
}
key, err = g.manager.Generate(tr, img)
if errors.Is(err, terrors.ErrImageTooLarge) {
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
}
return key, err
}
func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
src := req.GetWebdavSource()
imgURL, err := url.Parse(src.GetUrl())
if err != nil {
return "", errors.Wrap(err, "source url is invalid")
}
var auth, statPath string
if src.GetIsPublicLink() {
q := imgURL.Query()
var rsp *gateway.AuthenticateResponse
client, err := g.selector.Next()
if err != nil {
return "", merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
}
if q.Get("signature") != "" && q.Get("expiration") != "" {
// Handle pre-signed public links
sig := q.Get("signature")
exp := q.Get("expiration")
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
Type: "publicshares",
ClientId: src.GetPublicLinkToken(),
ClientSecret: strings.Join([]string{"signature", sig, exp}, "|"),
})
} else {
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
Type: "publicshares",
ClientId: src.GetPublicLinkToken(),
// We pass an empty password because we expect non pre-signed public links
// to not be password protected
ClientSecret: "password|",
})
}
if err != nil {
return "", merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error())
}
auth = rsp.GetToken()
statPath = path.Join("/public", src.GetPublicLinkToken(), req.GetFilepath())
} else {
auth = src.GetRevaAuthorization()
statPath = req.GetFilepath()
}
sRes, err := g.stat(statPath, auth)
if err != nil {
return "", err
}
key, tr, err := g.checkThumbnail(req, sRes)
switch {
case err != nil:
return "", err
case key != "":
// we have matching thumbnail already, use that
return key, nil
}
if src.GetWebdavAuthorization() != "" {
ctx = imgsource.ContextSetAuthorization(ctx, src.GetWebdavAuthorization())
}
// add signature and expiration to webdav url
signature, expiration := imgURL.Query().Get("signature"), imgURL.Query().Get("expiration")
params := url.Values{}
params.Add("signature", signature)
params.Add("expiration", expiration)
imgURL.RawQuery = params.Encode()
r, err := g.webdavSource.Get(ctx, imgURL.String())
switch {
case errors.Is(err, terrors.ErrImageTooLarge):
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
case err != nil:
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
}
defer r.Close()
ppOpts := map[string]any{
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
}
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
img, err := pp.Convert(r)
if img == nil || err != nil {
return "", merrors.NotFound(g.serviceID, "could not get image")
}
key, err = g.manager.Generate(tr, img)
if errors.Is(err, terrors.ErrImageTooLarge) {
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
}
return key, err
}
func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
ctx := metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
ref, err := storagespace.ParseReference(path)
if err != nil {
// If the path is not a spaces reference try to handle it like a plain
// path reference.
ref = provider.Reference{
Path: path,
}
}
client, err := g.selector.Next()
if err != nil {
return nil, merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
}
req := &provider.StatRequest{Ref: &ref}
rsp, err := client.Stat(ctx, req)
if err != nil {
g.logger.Error().Err(err).Str("path", path).Msg("could not stat file")
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", err.Error())
}
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
switch rsp.GetStatus().GetCode() {
case rpc.Code_CODE_NOT_FOUND:
return nil, merrors.NotFound(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
default:
g.logger.Error().Str("status_message", rsp.GetStatus().GetMessage()).Str("path", path).Msg("could not stat file")
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
}
}
if rsp.GetInfo().GetType() != provider.ResourceType_RESOURCE_TYPE_FILE {
return nil, merrors.BadRequest(g.serviceID, "Unsupported file type")
}
if utils.ReadPlainFromOpaque(rsp.GetInfo().GetOpaque(), "status") == "processing" {
return nil, &merrors.Error{
Id: g.serviceID,
Code: http.StatusTooEarly,
Detail: "File Processing",
Status: http.StatusText(http.StatusTooEarly),
}
}
if rsp.GetInfo().GetChecksum().GetSum() == "" {
g.logger.Error().Msg("resource info is missing checksum")
return nil, merrors.NotFound(g.serviceID, "resource info is missing a checksum")
}
if !thumbnail.IsMimeTypeSupported(rsp.GetInfo().GetMimeType()) {
return nil, merrors.NotFound(g.serviceID, "Unsupported file type")
}
return rsp, nil
}
@@ -0,0 +1,30 @@
package svc
import (
"net/http"
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
)
// NewInstrument returns a service that instruments metrics.
func NewInstrument(next Service, metrics *metrics.Metrics) Service {
return instrument{
next: next,
metrics: metrics,
}
}
type instrument struct {
next Service
metrics *metrics.Metrics
}
// ServeHTTP implements the Service interface.
func (i instrument) ServeHTTP(w http.ResponseWriter, r *http.Request) {
i.next.ServeHTTP(w, r)
}
// GetThumbnail implements the Service interface.
func (i instrument) GetThumbnail(w http.ResponseWriter, r *http.Request) {
i.next.GetThumbnail(w, r)
}
@@ -0,0 +1,30 @@
package svc
import (
"net/http"
"github.com/qsfera/server/pkg/log"
)
// NewLogging returns a service that logs messages.
func NewLogging(next Service, logger log.Logger) Service {
return logging{
next: next,
logger: logger,
}
}
type logging struct {
next Service
logger log.Logger
}
// ServeHTTP implements the Service interface.
func (l logging) ServeHTTP(w http.ResponseWriter, r *http.Request) {
l.next.ServeHTTP(w, r)
}
// GetThumbnail implements the Service interface.
func (l logging) GetThumbnail(w http.ResponseWriter, r *http.Request) {
l.next.GetThumbnail(w, r)
}
@@ -0,0 +1,61 @@
package svc
import (
"net/http"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
"go.opentelemetry.io/otel/trace"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Config *config.Config
Middleware []func(http.Handler) http.Handler
ThumbnailStorage storage.Storage
TraceProvider trace.TracerProvider
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// Middleware provides a function to set the middleware option.
func Middleware(val ...func(http.Handler) http.Handler) Option {
return func(o *Options) {
o.Middleware = val
}
}
// ThumbnailStorage provides a function to set the ThumbnailStorage option.
func ThumbnailStorage(storage storage.Storage) Option {
return func(o *Options) {
o.ThumbnailStorage = storage
}
}
@@ -0,0 +1,150 @@
package svc
import (
"context"
"fmt"
"net/http"
"strconv"
"github.com/go-chi/chi/v5"
"github.com/golang-jwt/jwt/v5"
"github.com/riandyrn/otelchi"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/pkg/tracing"
"github.com/qsfera/server/services/thumbnails/pkg/config"
tjwt "github.com/qsfera/server/services/thumbnails/pkg/service/jwt"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
)
type contextKey string
const (
keyContextKey contextKey = "key"
)
// Service defines the service handlers.
type Service interface {
ServeHTTP(w http.ResponseWriter, r *http.Request)
GetThumbnail(w http.ResponseWriter, r *http.Request)
}
// NewService returns a service implementation for Service.
func NewService(opts ...Option) Service {
options := newOptions(opts...)
m := chi.NewMux()
m.Use(options.Middleware...)
m.Use(
otelchi.Middleware(
"thumbnails",
otelchi.WithChiRoutes(m),
otelchi.WithTracerProvider(options.TraceProvider),
otelchi.WithPropagators(tracing.GetPropagator()),
),
)
logger := options.Logger
resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions)
if err != nil {
logger.Fatal().Err(err).Msg("resolutions not configured correctly")
}
svc := Thumbnails{
config: options.Config,
mux: m,
logger: options.Logger,
manager: thumbnail.NewSimpleManager(
resolutions,
options.ThumbnailStorage,
logger,
options.Config.Thumbnail.MaxInputWidth,
options.Config.Thumbnail.MaxInputHeight,
),
}
m.Route(options.Config.HTTP.Root, func(r chi.Router) {
r.Use(svc.TransferTokenValidator)
r.Get("/data", svc.GetThumbnail)
})
_ = chi.Walk(m, func(method string, route string, _ http.Handler, middlewares ...func(http.Handler) http.Handler) error {
options.Logger.Debug().Str("method", method).Str("route", route).Int("middlewares", len(middlewares)).Msg("serving endpoint")
return nil
})
return svc
}
// Thumbnails implements the business logic for Service.
type Thumbnails struct {
config *config.Config
logger log.Logger
mux *chi.Mux
manager thumbnail.Manager
}
// ServeHTTP implements the Service interface.
func (s Thumbnails) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
}
// GetThumbnail implements the Service interface.
func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) {
logger := s.logger.SubloggerWithRequestID(r.Context())
key := r.Context().Value(keyContextKey).(string)
thumbnailBytes, err := s.manager.GetThumbnail(key)
if err != nil {
logger.Debug().
Err(err).
Str("key", key).
Msg("could not get the thumbnail")
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Length", strconv.Itoa(len(thumbnailBytes)))
if _, err = w.Write(thumbnailBytes); err != nil {
logger.Error().
Err(err).
Str("key", key).
Msg("could not write the thumbnail response")
}
}
// TransferTokenValidator validates a transfer token
func (s Thumbnails) TransferTokenValidator(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tokenString := r.Header.Get("Transfer-Token")
if tokenString == "" {
next.ServeHTTP(w, r)
return
}
logger := s.logger.SubloggerWithRequestID(r.Context())
token, err := jwt.ParseWithClaims(tokenString, &tjwt.ThumbnailClaims{}, func(token *jwt.Token) (any, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
}
return []byte(s.config.Thumbnail.TransferSecret), nil
})
if err != nil {
logger.Debug().
Err(err).
Str("transfer-token", tokenString).
Msg("failed to parse transfer token")
w.WriteHeader(http.StatusUnauthorized)
return
}
if claims, ok := token.Claims.(*tjwt.ThumbnailClaims); ok && token.Valid {
ctx := context.WithValue(r.Context(), keyContextKey, claims.Key)
next.ServeHTTP(w, r.WithContext(ctx))
return
}
logger.Debug().Msg("invalid transfer token")
w.WriteHeader(http.StatusUnauthorized)
})
}
@@ -0,0 +1,9 @@
package jwt
import "github.com/golang-jwt/jwt/v5"
// ThumbnailClaims defines the claims for thumb-nailing
type ThumbnailClaims struct {
jwt.RegisteredClaims
Key string `json:"key"`
}
@@ -0,0 +1,81 @@
package thumbnail
import (
"image/gif"
"io"
"strings"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
const (
typePng = "png"
typeJpg = "jpg"
typeJpeg = "jpeg"
typeGif = "gif"
typeGgs = "ggs"
typeGgp = "ggp"
typeWebp = "webp"
)
// Encoder encodes the thumbnail to a specific format.
type Encoder interface {
// Encode encodes the image to a format.
Encode(w io.Writer, img any) error
// Types returns the formats suffixes.
Types() []string
// MimeType returns the mimetype used by the encoder.
MimeType() string
}
// GifEncoder encodes to gif
type GifEncoder struct{}
// Encode encodes the image to a gif format
func (e GifEncoder) Encode(w io.Writer, img any) error {
g, ok := img.(*gif.GIF)
if !ok {
return errors.ErrInvalidType
}
return gif.EncodeAll(w, g)
}
// Types returns the supported types of the GifEncoder
func (e GifEncoder) Types() []string {
return []string{typeGif}
}
// MimeType returns the mimetype used by the encoder.
func (e GifEncoder) MimeType() string {
return "image/gif"
}
// EncoderForType returns the encoder for a given file type
// or nil if the type is not supported.
func EncoderForType(fileType string) (Encoder, error) {
switch strings.ToLower(fileType) {
case typePng, typeGgs, typeGgp:
return PngEncoder{}, nil
case typeJpg, typeJpeg, typeWebp:
return JpegEncoder{}, nil
case typeGif:
return GifEncoder{}, nil
default:
return nil, errors.ErrNoEncoderForType
}
}
// GetExtForMime return the supported extension by mime
func GetExtForMime(fileType string) string {
ext := strings.TrimPrefix(strings.TrimSpace(strings.ToLower(fileType)), "image/")
switch ext {
case typeJpg, typeJpeg, typePng, typeGif, typeWebp:
return ext
case "application/vnd.geogebra.slides":
return typeGgs
case "application/vnd.geogebra.pinboard":
return typeGgp
default:
return ""
}
}
@@ -0,0 +1,56 @@
//go:build !enable_vips
package thumbnail
import (
"image"
"image/jpeg"
"image/png"
"io"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// PngEncoder encodes to png
type PngEncoder struct{}
// Encode encodes to png format
func (e PngEncoder) Encode(w io.Writer, img any) error {
m, ok := img.(image.Image)
if !ok {
return errors.ErrInvalidType
}
return png.Encode(w, m)
}
// Types returns the png suffix
func (e PngEncoder) Types() []string {
return []string{typePng}
}
// MimeType returns the mimetype for png files.
func (e PngEncoder) MimeType() string {
return "image/png"
}
// JpegEncoder encodes to jpg
type JpegEncoder struct{}
// Encode encodes to jpg
func (e JpegEncoder) Encode(w io.Writer, img any) error {
m, ok := img.(image.Image)
if !ok {
return errors.ErrInvalidType
}
return jpeg.Encode(w, m, nil)
}
// Types returns the jpg suffixes.
func (e JpegEncoder) Types() []string {
return []string{typeJpeg, typeJpg}
}
// MimeType returns the mimetype for jpg files.
func (e JpegEncoder) MimeType() string {
return "image/jpeg"
}
@@ -0,0 +1,22 @@
package thumbnail
import "testing"
func TestEncoderForType(t *testing.T) {
table := map[string]Encoder{
"jpg": JpegEncoder{},
"JPG": JpegEncoder{},
"jpeg": JpegEncoder{},
"JPEG": JpegEncoder{},
"png": PngEncoder{},
"PNG": PngEncoder{},
"invalid": nil,
}
for k, v := range table {
e, _ := EncoderForType(k)
if e != v {
t.Fail()
}
}
}
@@ -0,0 +1,66 @@
//go:build enable_vips
package thumbnail
import (
"io"
"github.com/davidbyttow/govips/v2/vips"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// PngEncoder encodes to png
type PngEncoder struct{}
// Encode encodes to png format
func (e PngEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(*vips.ImageRef)
if !ok {
return errors.ErrInvalidType
}
buf, _, err := m.ExportPng(vips.NewPngExportParams())
if err != nil {
return err
}
_, err = w.Write(buf)
return err
}
// Types returns the png suffix
func (e PngEncoder) Types() []string {
return []string{typePng}
}
// MimeType returns the mimetype for png files.
func (e PngEncoder) MimeType() string {
return "image/png"
}
// JpegEncoder encodes to jpg
type JpegEncoder struct{}
// Encode encodes to jpg
func (e JpegEncoder) Encode(w io.Writer, img interface{}) error {
m, ok := img.(*vips.ImageRef)
if !ok {
return errors.ErrInvalidType
}
buf, _, err := m.ExportJpeg(vips.NewJpegExportParams())
if err != nil {
return err
}
_, err = w.Write(buf)
return err
}
// Types returns the jpg suffixes.
func (e JpegEncoder) Types() []string {
return []string{typeJpeg, typeJpg}
}
// MimeType returns the mimetype for jpg files.
func (e JpegEncoder) MimeType() string {
return "image/jpeg"
}
@@ -0,0 +1,99 @@
package thumbnail
import (
"image"
"image/color"
"image/draw"
"image/gif"
"strings"
"github.com/kovidgoyal/imaging"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// Generator generates a web friendly file version.
type Generator interface {
Generate(size image.Rectangle, img any) (any, error)
Dimensions(img any) (image.Rectangle, error)
ProcessorID() string
}
// GifGenerator is used to create a web friendly version of the provided gif image.
type GifGenerator struct {
processor Processor
}
func NewGifGenerator(filetype, process string) (GifGenerator, error) {
processor, err := ProcessorFor(process, filetype)
if err != nil {
return GifGenerator{}, err
}
return GifGenerator{processor: processor}, nil
}
// ProcessorID returns the processor identification.
func (g GifGenerator) ProcessorID() string {
return g.processor.ID()
}
// Generate generates a alternative gif version.
func (g GifGenerator) Generate(size image.Rectangle, img any) (any, error) {
// Code inspired by https://github.com/willnorris/gifresize/blob/db93a7e1dcb1c279f7eeb99cc6d90b9e2e23e871/gifresize.go
m, ok := img.(*gif.GIF)
if !ok {
return nil, errors.ErrInvalidType
}
// Create a new RGBA image to hold the incremental frames.
srcX, srcY := m.Config.Width, m.Config.Height
b := image.Rect(0, 0, srcX, srcY)
tmp := image.NewRGBA(b)
for i, frame := range m.Image {
bounds := frame.Bounds()
prev := tmp
draw.Draw(tmp, bounds, frame, bounds.Min, draw.Over)
processed := g.processor.Process(tmp, size.Dx(), size.Dy(), imaging.Lanczos)
m.Image[i] = g.imageToPaletted(processed, frame.Palette)
switch m.Disposal[i] {
case gif.DisposalBackground:
tmp = image.NewRGBA(b)
case gif.DisposalPrevious:
tmp = prev
}
}
m.Config.Width = size.Dx()
m.Config.Height = size.Dy()
return m, nil
}
func (g GifGenerator) Dimensions(img any) (image.Rectangle, error) {
m, ok := img.(*gif.GIF)
if !ok {
return image.Rectangle{}, errors.ErrInvalidType
}
return m.Image[0].Bounds(), nil
}
func (g GifGenerator) imageToPaletted(img image.Image, p color.Palette) *image.Paletted {
b := img.Bounds()
pm := image.NewPaletted(b, p)
draw.FloydSteinberg.Draw(pm, b, img, image.Point{})
return pm
}
// GeneratorFor returns the generator for a given file type
// or nil if the type is not supported.
func GeneratorFor(fileType, processorID string) (Generator, error) {
switch strings.ToLower(fileType) {
case typePng, typeJpg, typeJpeg, typeGgs, typeGgp, typeWebp:
return NewSimpleGenerator(fileType, processorID)
case typeGif:
return NewGifGenerator(fileType, processorID)
default:
return nil, errors.ErrNoGeneratorForType
}
}
@@ -0,0 +1,46 @@
//go:build !enable_vips
package thumbnail
import (
"image"
"github.com/kovidgoyal/imaging"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
)
// SimpleGenerator is the default image generator and is used for all image types expect gif.
type SimpleGenerator struct {
processor Processor
}
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
processor, err := ProcessorFor(process, filetype)
if err != nil {
return SimpleGenerator{}, err
}
return SimpleGenerator{processor: processor}, nil
}
// ProcessorID returns the processor identification.
func (g SimpleGenerator) ProcessorID() string {
return g.processor.ID()
}
// Generate generates a alternative image version.
func (g SimpleGenerator) Generate(size image.Rectangle, img any) (any, error) {
m, ok := img.(image.Image)
if !ok {
return nil, errors.ErrInvalidType
}
return g.processor.Process(m, size.Dx(), size.Dy(), imaging.Lanczos), nil
}
func (g SimpleGenerator) Dimensions(img any) (image.Rectangle, error) {
m, ok := img.(image.Image)
if !ok {
return image.Rectangle{}, errors.ErrInvalidType
}
return m.Bounds(), nil
}
@@ -0,0 +1,84 @@
//go:build enable_vips
package thumbnail
import (
"bytes"
"image"
"strings"
"github.com/davidbyttow/govips/v2/vips"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"golang.org/x/image/bmp"
)
// SimpleGenerator is the default image generator and is used for all image types expect gif.
type SimpleGenerator struct {
crop vips.Interesting
size vips.Size
process string
}
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
switch strings.ToLower(process) {
case "thumbnail":
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
case "fit":
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeBoth}, nil
case "resize":
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeForce}, nil
default:
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
}
}
// ProcessorID returns the processor identification.
func (g SimpleGenerator) ProcessorID() string {
return g.process
}
// Generate generates a alternative image version.
func (g SimpleGenerator) Generate(size image.Rectangle, img interface{}) (interface{}, error) {
var m *vips.ImageRef
var err error
switch img.(type) {
case *image.RGBA:
// This comes from the txt preprocessor
var buf bytes.Buffer
if err = bmp.Encode(&buf, img.(*image.RGBA)); err != nil {
return nil, err
}
m, err = vips.NewImageFromReader(&buf)
if err != nil {
return nil, err
}
case *vips.ImageRef:
m = img.(*vips.ImageRef)
default:
return nil, errors.ErrInvalidType
}
if err := m.ThumbnailWithSize(size.Dx(), 0, g.crop, g.size); err != nil {
return nil, err
}
if err := m.RemoveMetadata(); err != nil {
return nil, err
}
return m, nil
}
func (g SimpleGenerator) Dimensions(img interface{}) (image.Rectangle, error) {
switch img.(type) {
case *image.RGBA:
m := img.(*image.RGBA)
return m.Bounds(), nil
case *vips.ImageRef:
m := img.(*vips.ImageRef)
return image.Rect(0, 0, m.Width(), m.Height()), nil
default:
return image.Rectangle{}, errors.ErrInvalidType
}
}
@@ -0,0 +1,132 @@
package imgsource
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/qsfera/server/services/thumbnails/pkg/config"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/rhttp"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"google.golang.org/grpc/metadata"
)
const (
// TokenTransportHeader holds the header key for the reva transfer token
// "github.com/opencloud-eu/reva/v2/internal/http/services/datagateway" is internal so we redeclare it here
TokenTransportHeader = "X-Reva-Transfer"
)
// CS3 implements a CS3 image source
type CS3 struct {
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
insecure bool
maxImageFileSize uint64
}
// NewCS3Source configures a new CS3 image source
func NewCS3Source(cfg config.Thumbnail, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], b bytesize.ByteSize) CS3 {
return CS3{
gatewaySelector: gatewaySelector,
insecure: cfg.CS3AllowInsecure,
maxImageFileSize: b.Bytes(),
}
}
// Get downloads the file from a cs3 service
// The caller MUST make sure to close the returned ReadCloser
func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
auth, ok := ContextGetAuthorization(ctx)
if !ok {
return nil, errors.ErrCS3AuthorizationMissing
}
ref, err := storagespace.ParseReference(path)
if err != nil {
// If the path is not a spaces reference try to handle it like a plain
// path reference.
ref = provider.Reference{
Path: path,
}
}
ctx = metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
err = s.checkImageFileSize(ctx, ref)
if err != nil {
return nil, err
}
gwc, err := s.gatewaySelector.Next()
if err != nil {
return nil, err
}
rsp, err := gwc.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &ref})
if err != nil {
return nil, err
}
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load image: %s", rsp.GetStatus().GetMessage())
}
var ep, tk string
for _, p := range rsp.GetProtocols() {
if p.GetProtocol() == "spaces" {
ep, tk = p.GetDownloadEndpoint(), p.GetToken()
break
}
}
if (ep == "" || tk == "") && len(rsp.GetProtocols()) > 0 {
ep, tk = rsp.GetProtocols()[0].GetDownloadEndpoint(), rsp.GetProtocols()[0].GetToken()
}
httpReq, err := rhttp.NewRequest(ctx, "GET", ep, nil)
if err != nil {
return nil, err
}
httpReq.Header.Set(revactx.TokenHeader, auth)
httpReq.Header.Set(TokenTransportHeader, tk)
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: s.insecure, //nolint:gosec
}
client := &http.Client{}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", path, resp.StatusCode)
}
return resp.Body, nil
}
func (s CS3) checkImageFileSize(ctx context.Context, ref provider.Reference) error {
gwc, err := s.gatewaySelector.Next()
if err != nil {
return err
}
stat, err := gwc.Stat(ctx, &provider.StatRequest{Ref: &ref})
if err != nil {
return err
}
if stat.GetStatus().GetCode() != rpc.Code_CODE_OK {
return fmt.Errorf("could not stat image: %s", stat.GetStatus().GetMessage())
}
if stat.GetInfo().GetSize() > s.maxImageFileSize {
return errors.ErrImageTooLarge
}
return nil
}
@@ -0,0 +1,31 @@
package imgsource
import (
"context"
"io"
)
type key int
const (
auth key = iota
)
// Source defines the interface for image sources
type Source interface {
Get(ctx context.Context, path string) (io.ReadCloser, error)
}
// ContextSetAuthorization puts the authorization in the context.
func ContextSetAuthorization(parent context.Context, authorization string) context.Context {
return context.WithValue(parent, auth, authorization)
}
// ContextGetAuthorization gets the authorization from the context.
func ContextGetAuthorization(ctx context.Context) (string, bool) {
val := ctx.Value(auth)
if val == nil {
return "", false
}
return val.(string), true
}
@@ -0,0 +1,75 @@
package imgsource
import (
"context"
"crypto/tls"
"fmt"
_ "image/gif" // Import the gif package so that image.Decode can understand gifs
_ "image/jpeg" // Import the jpeg package so that image.Decode can understand jpegs
_ "image/png" // Import the png package so that image.Decode can understand pngs
"io"
"net/http"
"strconv"
"github.com/qsfera/server/services/thumbnails/pkg/config"
thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
"github.com/pkg/errors"
)
// NewWebDavSource creates a new webdav instance.
func NewWebDavSource(cfg config.Thumbnail, b bytesize.ByteSize) WebDav {
return WebDav{
insecure: cfg.WebdavAllowInsecure,
maxImageFileSize: b.Bytes(),
}
}
// WebDav implements the Source interface for webdav services
type WebDav struct {
insecure bool
maxImageFileSize uint64
}
// Get downloads the file from a webdav service
// The caller MUST make sure to close the returned ReadCloser
func (s WebDav) Get(ctx context.Context, url string) (io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: s.insecure, //nolint:gosec
}
if auth, ok := ContextGetAuthorization(ctx); ok {
req.Header.Add("Authorization", auth)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", url, resp.StatusCode)
}
contentLength := resp.Header.Get("Content-Length")
if contentLength == "" {
// no size information - let's assume it is too big
return nil, thumbnailerErrors.ErrImageTooLarge
}
c, err := strconv.ParseUint(contentLength, 10, 64)
if err != nil {
return nil, errors.Wrapf(err, `could not parse content length of webdav response "%s"`, url)
}
if c > s.maxImageFileSize {
return nil, thumbnailerErrors.ErrImageTooLarge
}
return resp.Body, nil
}
@@ -0,0 +1,22 @@
//go:build !enable_vips
package thumbnail
var (
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
"image/jpeg": {},
"image/gif": {},
"image/bmp": {},
"image/x-ms-bmp": {},
"image/tiff": {},
"text/plain": {},
"audio/flac": {},
"audio/mpeg": {},
"audio/ogg": {},
"application/vnd.geogebra.slides": {},
"application/vnd.geogebra.pinboard": {},
}
)
@@ -0,0 +1,31 @@
//go:build enable_vips
package thumbnail
import "github.com/davidbyttow/govips/v2/vips"
func init() {
// temporary remove TIFF and JP2K from go-vips' list of supported
// imagetypes
delete(vips.ImageTypes, vips.ImageTypeTIFF)
delete(vips.ImageTypes, vips.ImageTypeJP2K)
}
var (
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
SupportedMimeTypes = map[string]struct{}{
"image/png": {},
"image/jpg": {},
"image/jpeg": {},
"image/gif": {},
"image/bmp": {},
"image/x-ms-bmp": {},
"text/plain": {},
"audio/flac": {},
"audio/mpeg": {},
"audio/ogg": {},
"application/vnd.geogebra.slides": {},
"application/vnd.geogebra.pinboard": {},
"image/webp": {},
}
)
@@ -0,0 +1,51 @@
package thumbnail
import (
"image"
"strings"
"github.com/kovidgoyal/imaging"
)
// Processor processes the thumbnail by applying different transformations to it.
type Processor interface {
ID() string
Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
}
// DefinableProcessor is the simplest processor, it holds a replaceable image converter function.
type DefinableProcessor struct {
Slug string
Converter func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
}
// ID returns the processor identification.
func (p DefinableProcessor) ID() string { return p.Slug }
// Process transforms the given image.
func (p DefinableProcessor) Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
return p.Converter(img, width, height, filter)
}
// ProcessorFor returns a matching Processor
func ProcessorFor(id, fileType string) (DefinableProcessor, error) {
switch strings.ToLower(id) {
case "fit":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Fit}, nil
case "resize":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Resize}, nil
case "fill":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
return imaging.Fill(img, width, height, imaging.Center, filter)
}}, nil
case "thumbnail":
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Thumbnail}, nil
default:
switch strings.ToLower(fileType) {
case typeGif:
return DefinableProcessor{Converter: imaging.Resize}, nil
default:
return DefinableProcessor{Converter: imaging.Thumbnail}, nil
}
}
}
@@ -0,0 +1,97 @@
package thumbnail_test
import (
"testing"
"github.com/kovidgoyal/imaging"
tAssert "github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
)
func TestProcessorFor(t *testing.T) {
tests := []struct {
id string
fileType string
wantP thumbnail.Processor
wantE error
}{
{
id: "fit",
fileType: "",
wantP: thumbnail.DefinableProcessor{Slug: "fit", Converter: imaging.Fit},
wantE: nil,
},
{
id: "fit",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
wantE: nil,
},
{
id: "FIT",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
wantE: nil,
},
{
id: "resize",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
wantE: nil,
},
{
id: "RESIZE",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
wantE: nil,
},
{
id: "fill",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
wantE: nil,
},
{
id: "FILL",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
wantE: nil,
},
{
id: "thumbnail",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
wantE: nil,
},
{
id: "THUMBNAIL",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
wantE: nil,
},
{
id: "",
fileType: "jpg",
wantP: thumbnail.DefinableProcessor{},
wantE: nil,
},
{
id: "",
fileType: "gif",
wantP: thumbnail.DefinableProcessor{},
wantE: nil,
},
}
assert := tAssert.New(t)
for _, tt := range tests {
t.Run("", func(t *testing.T) {
p, e := thumbnail.ProcessorFor(tt.id, tt.fileType)
assert.Equal(p.ID(), tt.wantP.ID())
assert.Equal(e, tt.wantE)
})
}
}
@@ -0,0 +1,101 @@
package thumbnail
import (
"fmt"
"image"
"math"
"strconv"
"strings"
"github.com/pkg/errors"
)
const (
_resolutionSeparator = "x"
)
// ParseResolution returns an image.Rectangle representing the resolution given as a string
func ParseResolution(s string) (image.Rectangle, error) {
parts := strings.Split(s, _resolutionSeparator)
if len(parts) != 2 {
return image.Rectangle{}, fmt.Errorf("failed to parse resolution: %s. Expected format <width>x<height>", s)
}
width, err := strconv.Atoi(parts[0])
if err != nil {
return image.Rectangle{}, fmt.Errorf("width: %s has an invalid value. Expected an integer", parts[0])
}
height, err := strconv.Atoi(parts[1])
if err != nil {
return image.Rectangle{}, fmt.Errorf("height: %s has an invalid value. Expected an integer", parts[1])
}
return image.Rect(0, 0, width, height), nil
}
// Resolutions is a list of image.Rectangle representing resolutions.
type Resolutions []image.Rectangle
// ParseResolutions creates an instance of Resolutions from resolution strings.
func ParseResolutions(strs []string) (Resolutions, error) {
rs := make(Resolutions, 0, len(strs))
for _, s := range strs {
r, err := ParseResolution(s)
if err != nil {
return nil, errors.Wrap(err, "could not parse resolutions")
}
rs = append(rs, r)
}
return rs, nil
}
// ClosestMatch returns the resolution which is closest to the provided resolution.
// If there is no exact match the resolution will be the next higher one.
// If the given resolution is bigger than all available resolutions the biggest available one is used.
func (rs Resolutions) ClosestMatch(requested image.Rectangle, sourceSize image.Rectangle) image.Rectangle {
isLandscape := sourceSize.Dx() > sourceSize.Dy()
sourceLen := dimensionLength(sourceSize, isLandscape)
requestedLen := dimensionLength(requested, isLandscape)
isSourceSmaller := sourceLen < requestedLen
// We don't want to scale images up.
if isSourceSmaller {
return sourceSize
}
if len(rs) == 0 {
return requested
}
var match image.Rectangle
// Since we want to search for the smallest difference we start with the highest possible number
minDiff := math.MaxInt32
for _, current := range rs {
cLen := dimensionLength(current, isLandscape)
diff := requestedLen - cLen
if diff > 0 {
// current is smaller
continue
}
// Convert diff to positive value
// Multiplying by -1 is safe since we aren't getting positive numbers here
// because of the check above
absDiff := diff * -1
if absDiff < minDiff {
minDiff = absDiff
match = current
}
}
if (match == image.Rectangle{}) {
match = rs[len(rs)-1]
}
return match
}
func dimensionLength(rect image.Rectangle, isLandscape bool) int {
if isLandscape {
return rect.Dx()
}
return rect.Dy()
}
@@ -0,0 +1,121 @@
package thumbnail
import (
"image"
"testing"
)
func TestInitWithEmptyArray(t *testing.T) {
rs, err := ParseResolutions([]string{})
if err != nil {
t.Errorf("Init with an empty array should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 0 {
t.Error("Init with an empty array should return an empty Resolutions instance.\n")
}
}
func TestInitWithNil(t *testing.T) {
rs, err := ParseResolutions(nil)
if err != nil {
t.Errorf("Init with nil parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 0 {
t.Error("Init with nil parameter should return an empty Resolutions instance.\n")
}
}
func TestInitWithInvalidValuesInArray(t *testing.T) {
_, err := ParseResolutions([]string{"invalid"})
if err == nil {
t.Error("Init with invalid parameter should fail.\n")
}
}
func TestInit(t *testing.T) {
rs, err := ParseResolutions([]string{"16x16"})
if err != nil {
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != 1 {
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), 1)
}
}
func TestInitWithMultipleResolutions(t *testing.T) {
rStrs := []string{"16x16", "32x32", "64x64", "128x128"}
rs, err := ParseResolutions(rStrs)
if err != nil {
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
}
if len(rs) != len(rStrs) {
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), len(rStrs))
}
}
func TestClosestMatchWithEmptyResolutions(t *testing.T) {
rs, _ := ParseResolutions(nil)
want := image.Rect(0, 0, 24, 24)
imgSize := image.Rect(0, 0, 24, 24)
r := rs.ClosestMatch(want, imgSize)
if r.Dx() != want.Dx() || r.Dy() != want.Dy() {
t.Errorf("ClosestMatch from empty resolutions should return the given resolution")
}
}
func TestClosestMatch(t *testing.T) {
rs, _ := ParseResolutions([]string{"16x16", "24x24", "32x32", "64x64", "128x128"})
testData := [][]image.Rectangle{
{image.Rect(0, 0, 17, 17), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 12, 17), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 24, 24), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 20, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
{image.Rect(0, 0, 20, 80), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 80, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 48, 48), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 64, 64)},
{image.Rect(0, 0, 1024, 1024), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
{image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 256, 36), image.Rect(0, 0, 256, 36)},
}
for _, row := range testData {
given := row[0]
imgSize := row[1]
expected := row[2]
match := rs.ClosestMatch(given, imgSize)
if match != expected {
t.Errorf("Expected resolution %dx%d got %dx%d", expected.Dx(), expected.Dy(), match.Dx(), match.Dy())
}
}
}
func TestParseWithEmptyString(t *testing.T) {
if _, err := ParseResolution(""); err == nil {
t.Error("Parse with empty string should return an error.")
}
}
func TestParseWithInvalidWidth(t *testing.T) {
_, err := ParseResolution("invalidx42")
if err == nil {
t.Error("Parse with invalid width should return an error.")
}
}
func TestParseWithInvalidHeight(t *testing.T) {
_, err := ParseResolution("42xinvalid")
if err == nil {
t.Error("Parse with invalid height should return an error.")
}
}
func TestParseResolution(t *testing.T) {
rStr := "42x23"
r, _ := ParseResolution(rStr)
if r.Dx() != 42 || r.Dy() != 23 {
t.Errorf("Expected resolution %s got %s", rStr, r.String())
}
}
@@ -0,0 +1,118 @@
package storage
import (
"io/fs"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/config"
)
const (
filesDir = "files"
)
// NewFileSystemStorage creates a new instance of FileSystem
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) FileSystem {
return FileSystem{
root: cfg.RootDirectory,
logger: logger,
}
}
// FileSystem represents a storage for the thumbnails using the local file system.
type FileSystem struct {
root string
logger log.Logger
}
// Stat returns if a file for the given key exists on the filesystem
func (s FileSystem) Stat(key string) bool {
img := filepath.Join(s.root, filesDir, key)
if _, err := os.Stat(img); err != nil {
return false
}
return true
}
// Get returns the file content for the given key
func (s FileSystem) Get(key string) ([]byte, error) {
img := filepath.Join(s.root, filesDir, key)
content, err := os.ReadFile(img)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
}
return nil, err
}
return content, nil
}
// Put stores image data in the file system for the given key
func (s FileSystem) Put(key string, img []byte) error {
imgPath := filepath.Join(s.root, filesDir, key)
dir := filepath.Dir(imgPath)
if err := os.MkdirAll(dir, 0700); err != nil {
return errors.Wrapf(err, "error while creating directory %s", dir)
}
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
f, err := os.CreateTemp(dir, "tmpthumb")
if err != nil {
return errors.Wrapf(err, "could not create temporary file for \"%s\"", key)
}
defer f.Close()
// if there was a problem writing, remove the temporary file
if _, writeErr := f.Write(img); writeErr != nil {
if remErr := os.Remove(f.Name()); remErr != nil {
return errors.Wrapf(remErr, "could not cleanup temporary file for \"%s\"", key)
}
return errors.Wrapf(writeErr, "could not write to temporary file for \"%s\"", key)
}
// if there wasn't a problem, ensure the data is written into disk
if synErr := f.Sync(); synErr != nil {
return errors.Wrapf(synErr, "could not sync temporary file data into the disk for \"%s\"", key)
}
// rename the temporary file to the final file
if renErr := os.Rename(f.Name(), imgPath); renErr != nil {
// if we couldn't rename, remove the temporary file
if remErr := os.Remove(f.Name()); remErr != nil {
return errors.Wrapf(remErr, "rename failed and could not cleanup temporary file for \"%s\"", key)
}
return errors.Wrapf(renErr, "could not rename temporary file to \"%s\"", key)
}
}
return nil
}
// BuildKey generate the unique key for a thumbnail.
// The key is structure as follows:
//
// <first two letters of checksum>/<next two letters of checksum>/<rest of checksum>/<width>x<height>.<filetype>
//
// e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png
//
// The key also represents the path to the thumbnail in the filesystem under the configured root directory.
func (s FileSystem) BuildKey(r Request) string {
checksum := r.Checksum
filetype := r.Types[0]
parts := []string{strconv.Itoa(r.Resolution.Dx()), "x", strconv.Itoa(r.Resolution.Dy())}
if r.Characteristic != "" {
parts = append(parts, "-", r.Characteristic)
}
parts = append(parts, ".", filetype)
return filepath.Join(checksum[:2], checksum[2:4], checksum[4:], strings.Join(parts, ""))
}
@@ -0,0 +1,64 @@
package storage_test
import (
"image"
"testing"
tAssert "github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
func TestFileSystem_BuildKey(t *testing.T) {
tests := []struct {
r storage.Request
want string
}{
{
r: storage.Request{
Checksum: "120EA8A25E5D487BF68B5F7096440019",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "",
},
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2.png",
},
{
r: storage.Request{
Checksum: "120EA8A25E5D487BF68B5F7096440019",
Types: []string{"png", "jpg"},
Resolution: image.Rectangle{
Min: image.Point{
X: 1,
Y: 2,
},
Max: image.Point{
X: 3,
Y: 4,
},
},
Characteristic: "fill",
},
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2-fill.png",
},
}
s := storage.FileSystem{}
assert := tAssert.New(t)
for _, tt := range tests {
t.Run("", func(t *testing.T) {
assert.Equal(s.BuildKey(tt.r), tt.want)
})
}
}
@@ -0,0 +1,32 @@
package storage
import (
"image"
)
// Request combines different attributes needed for storage operations.
type Request struct {
// The checksum of the source file
// Will be used to determine if a thumbnail exists
Checksum string
// Types provided by the encoder.
// Contains the mimetypes of the thumbnail.
// In case of jpg/jpeg it will contain both.
Types []string
// The resolution of the thumbnail
Resolution image.Rectangle
// Characteristic defines the different image characteristics,
// for example, if it's scaled up to fit in the bounding box or not,
// is it a chroma version of the image, and so on...
// the main propose for this is to be able to differentiate between images which have
// the same resolution but different characteristics.
Characteristic string
}
// Storage defines the interface for a thumbnail store.
type Storage interface {
Stat(key string) bool
Get(key string) ([]byte, error)
Put(key string, img []byte) error
BuildKey(r Request) string
}
@@ -0,0 +1,131 @@
package thumbnail
import (
"bytes"
"image"
"mime"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
// Request bundles information needed to generate a thumbnail for a file
type Request struct {
Resolution image.Rectangle
Encoder Encoder
Generator Generator
Checksum string
}
// Manager is responsible for generating thumbnails
type Manager interface {
// Generate creates a thumbnail and stores it.
// The function returns a key with which the actual file can be retrieved.
Generate(r Request, img any) (string, error)
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
// The function will return a status if the file exists and the key to the file.
CheckThumbnail(r Request) (string, bool)
// GetThumbnail will load the thumbnail from the storage and return its content.
GetThumbnail(key string) ([]byte, error)
}
// NewSimpleManager creates a new instance of SimpleManager
func NewSimpleManager(resolutions Resolutions, storage storage.Storage, logger log.Logger, maxInputWidth, maxInputHeight int) SimpleManager {
return SimpleManager{
storage: storage,
logger: logger,
resolutions: resolutions,
maxDimension: image.Point{X: maxInputWidth, Y: maxInputHeight},
}
}
// SimpleManager is a simple implementation of Manager
type SimpleManager struct {
storage storage.Storage
logger log.Logger
resolutions Resolutions
maxDimension image.Point
}
// Generate creates a thumbnail and stores it
func (s SimpleManager) Generate(r Request, img any) (string, error) {
var match image.Rectangle
inputDimensions, err := r.Generator.Dimensions(img)
if err != nil {
return "", err
}
match = s.resolutions.ClosestMatch(r.Resolution, inputDimensions)
// validate max input image dimensions - 6016x4000
if inputDimensions.Size().X > s.maxDimension.X || inputDimensions.Size().Y > s.maxDimension.Y {
return "", errors.ErrImageTooLarge
}
thumbnail, err := r.Generator.Generate(match, img)
if err != nil {
return "", err
}
buf := new(bytes.Buffer)
if err := r.Encoder.Encode(buf, thumbnail); err != nil {
return "", err
}
k := s.storage.BuildKey(mapToStorageRequest(r))
if err := s.storage.Put(k, buf.Bytes()); err != nil {
s.logger.Error().Err(err).Msg("could not store thumbnail")
return "", err
}
return k, nil
}
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
func (s SimpleManager) CheckThumbnail(r Request) (string, bool) {
k := s.storage.BuildKey(mapToStorageRequest(r))
return k, s.storage.Stat(k)
}
// GetThumbnail will load the thumbnail from the storage and return its content.
func (s SimpleManager) GetThumbnail(key string) ([]byte, error) {
return s.storage.Get(key)
}
func mapToStorageRequest(r Request) storage.Request {
return storage.Request{
Checksum: r.Checksum,
Resolution: r.Resolution,
Types: r.Encoder.Types(),
Characteristic: r.Generator.ProcessorID(),
}
}
// IsMimeTypeSupported validate if the mime type is supported
func IsMimeTypeSupported(m string) bool {
mimeType, _, err := mime.ParseMediaType(m)
if err != nil {
return false
}
_, supported := SupportedMimeTypes[mimeType]
return supported
}
// PrepareRequest prepare the request based on image parameters
func PrepareRequest(width, height int, tType, checksum, pID string) (Request, error) {
generator, err := GeneratorFor(tType, pID)
if err != nil {
return Request{}, err
}
encoder, err := EncoderForType(tType)
if err != nil {
return Request{}, err
}
return Request{
Resolution: image.Rect(0, 0, width, height),
Generator: generator,
Encoder: encoder,
Checksum: checksum,
}, nil
}
@@ -0,0 +1,198 @@
package thumbnail
import (
"image"
"os"
"path"
"path/filepath"
"reflect"
"testing"
"github.com/qsfera/server/services/thumbnails/pkg/errors"
"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
"github.com/stretchr/testify/assert"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
)
type NoOpManager struct {
storage.Storage
}
func (m NoOpManager) BuildKey(_ storage.Request) string {
return ""
}
func (m NoOpManager) Set(_, _ string, _ []byte) error {
return nil
}
func BenchmarkGet(b *testing.B) {
sut := NewSimpleManager(
Resolutions{},
NoOpManager{},
log.NewLogger(),
6016,
4000,
)
res, _ := ParseResolution("32x32")
req := Request{
Resolution: res,
Checksum: "1872ade88f3013edeb33decd74a4f947",
}
cwd, _ := os.Getwd()
p := filepath.Join(cwd, "../../testdata/test.png")
f, _ := os.Open(p)
defer f.Close()
img, ext, _ := image.Decode(f)
req.Encoder, _ = EncoderForType(ext)
for i := 0; i < b.N; i++ {
_, _ = sut.Generate(req, img)
}
}
func TestPrepareRequest(t *testing.T) {
type args struct {
width int
height int
tType string
checksum string
}
tests := []struct {
name string
args args
want Request
wantErr bool
}{
{
name: "Test successful prepare the request for jpg",
args: args{
width: 32,
height: 32,
tType: "jpg",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: JpegEncoder{},
Generator: SimpleGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test successful prepare the request for png",
args: args{
width: 32,
height: 32,
tType: "png",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: PngEncoder{},
Generator: SimpleGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test successful prepare the request for gif",
args: args{
width: 32,
height: 32,
tType: "gif",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
want: Request{
Resolution: image.Rect(0, 0, 32, 32),
Encoder: GifEncoder{},
Generator: GifGenerator{},
Checksum: "1872ade88f3013edeb33decd74a4f947",
},
},
{
name: "Test error when prepare the request for bmp",
args: args{
width: 32,
height: 32,
tType: "bmp",
checksum: "1872ade88f3013edeb33decd74a4f947",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := PrepareRequest(tt.args.width, tt.args.height, tt.args.tType, tt.args.checksum, "")
if (err != nil) != tt.wantErr {
t.Errorf("PrepareRequest() error = %v, wantErr %v", err, tt.wantErr)
return
}
// funcs are not reflactable, ignore
if diff := cmp.Diff(tt.want, got, cmpopts.IgnoreFields(Request{}, "Generator")); diff != "" {
t.Errorf("PrepareRequest(): %v", diff)
}
if reflect.TypeOf(got.Generator) != reflect.TypeOf(tt.want.Generator) {
t.Errorf("PrepareRequest() = %v, want %v", reflect.TypeOf(got.Generator), reflect.TypeOf(tt.want.Generator))
}
})
}
}
func TestPreviewGenerationTooBigImage(t *testing.T) {
tests := []struct {
name string
fileName string
mimeType string
}{
{name: "png", mimeType: "image/png", fileName: "../../testdata/test.png"},
{name: "jpg", mimeType: "image/jpeg", fileName: "../../testdata/test.jpg"},
{name: "ggs", mimeType: "application/vnd.geogebra.slides", fileName: "../../testdata/test.ggs"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sut := NewSimpleManager(
Resolutions{},
NoOpManager{},
log.NewLogger(),
1024,
768,
)
res, _ := ParseResolution("32x32")
req := Request{
Resolution: res,
Checksum: "1872ade88f3013edeb33decd74a4f947",
}
cwd, _ := os.Getwd()
p := filepath.Join(cwd, tt.fileName)
f, _ := os.Open(p)
defer f.Close()
preproc := preprocessor.ForType(tt.mimeType, nil)
convert, err := preproc.Convert(f)
if err != nil {
return
}
ext := path.Ext(tt.fileName)
req.Encoder, _ = EncoderForType(ext)
req.Generator, err = GeneratorFor(ext, "fit")
if err != nil {
return
}
generate, err := sut.Generate(req, convert)
if err != nil {
return
}
assert.ErrorIs(t, err, errors.ErrImageTooLarge)
assert.Equal(t, "", generate)
})
}
}