Initial QSfera import
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config/parser"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Health is the entrypoint for the health command.
|
||||
func Health(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "health",
|
||||
Short: "check health status",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnError(parser.ParseConfig(cfg))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
|
||||
|
||||
resp, err := http.Get(
|
||||
fmt.Sprintf(
|
||||
"http://%s/healthz",
|
||||
cfg.Debug.Addr,
|
||||
),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
logger.Fatal().
|
||||
Err(err).
|
||||
Msg("Failed to request health check")
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.Fatal().
|
||||
Int("code", resp.StatusCode).
|
||||
Msg("Health seems to be in bad state")
|
||||
}
|
||||
|
||||
logger.Debug().
|
||||
Int("code", resp.StatusCode).
|
||||
Msg("Health got a good state")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/qsfera/server/pkg/clihelper"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// GetCommands provides all commands for this service
|
||||
func GetCommands(cfg *config.Config) []*cobra.Command {
|
||||
return []*cobra.Command{
|
||||
// start this service
|
||||
Server(cfg),
|
||||
|
||||
// interaction with this service
|
||||
|
||||
// infos about this service
|
||||
Health(cfg),
|
||||
Version(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// Execute is the entry point for the qsfera-thumbnails command.
|
||||
func Execute(cfg *config.Config) error {
|
||||
app := clihelper.DefaultApp(&cobra.Command{
|
||||
Use: "thumbnails",
|
||||
Short: "Example usage",
|
||||
})
|
||||
app.AddCommand(GetCommands(cfg)...)
|
||||
app.SetArgs(os.Args[1:])
|
||||
|
||||
return app.ExecuteContext(cfg.Context)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/signal"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/pkg/runner"
|
||||
ogrpc "github.com/qsfera/server/pkg/service/grpc"
|
||||
"github.com/qsfera/server/pkg/tracing"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config/parser"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/server/debug"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/server/grpc"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/server/http"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "server",
|
||||
Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnFatal(parser.ParseConfig(cfg))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
|
||||
|
||||
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.GrpcClient, err = ogrpc.NewClient(ogrpc.GetClientOptions(cfg.GRPCClientTLS)...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var cancel context.CancelFunc
|
||||
if cfg.Context == nil {
|
||||
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
|
||||
defer cancel()
|
||||
}
|
||||
ctx := cfg.Context
|
||||
|
||||
m := metrics.New()
|
||||
m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
gr := runner.NewGroup()
|
||||
|
||||
service := grpc.NewService(
|
||||
grpc.Logger(logger),
|
||||
grpc.Context(ctx),
|
||||
grpc.Config(cfg),
|
||||
grpc.Name(cfg.Service.Name),
|
||||
grpc.Namespace(cfg.GRPC.Namespace),
|
||||
grpc.Address(cfg.GRPC.Addr),
|
||||
grpc.Metrics(m),
|
||||
grpc.TraceProvider(traceProvider),
|
||||
grpc.MaxConcurrentRequests(cfg.GRPC.MaxConcurrentRequests),
|
||||
)
|
||||
gr.Add(runner.NewGoMicroGrpcServerRunner(cfg.Service.Name+".grpc", service))
|
||||
|
||||
server, err := debug.Server(
|
||||
debug.Logger(logger),
|
||||
debug.Config(cfg),
|
||||
debug.Context(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
gr.Add(runner.NewGolangHttpServerRunner(cfg.Service.Name+".debug", server))
|
||||
|
||||
httpServer, err := http.Server(
|
||||
http.Logger(logger),
|
||||
http.Context(ctx),
|
||||
http.Config(cfg),
|
||||
http.Metrics(m),
|
||||
http.Namespace(cfg.HTTP.Namespace),
|
||||
http.TraceProvider(traceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().
|
||||
Err(err).
|
||||
Str("transport", "http").
|
||||
Msg("Failed to initialize server")
|
||||
|
||||
return err
|
||||
}
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", httpServer))
|
||||
|
||||
grResults := gr.Run(ctx)
|
||||
|
||||
// return the first non-nil error found in the results
|
||||
for _, grResult := range grResults {
|
||||
if grResult.RunnerError != nil {
|
||||
return grResult.RunnerError
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/qsfera/server/pkg/registry"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/olekukonko/tablewriter/tw"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Version prints the service versions of all running instances.
|
||||
func Version(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print the version of this binary and the running service instances",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Println("Version: " + version.GetString())
|
||||
fmt.Printf("Compiled: %s\n", version.Compiled())
|
||||
fmt.Println("")
|
||||
|
||||
reg := registry.GetRegistry()
|
||||
services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name)
|
||||
if err != nil {
|
||||
fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err))
|
||||
return err
|
||||
}
|
||||
|
||||
if len(services) == 0 {
|
||||
fmt.Println("No running " + cfg.Service.Name + " service found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
|
||||
table.Header([]string{"Version", "Address", "Id"})
|
||||
for _, s := range services {
|
||||
for _, n := range s.Nodes {
|
||||
table.Append([]string{s.Version, n.Address, n.Id})
|
||||
}
|
||||
}
|
||||
table.Render()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Package config contains the configuration for the qsfera-thumbnails service
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"go-micro.dev/v4/client"
|
||||
)
|
||||
|
||||
// Config combines all available configuration parts.
|
||||
type Config struct {
|
||||
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;THUMBNAILS_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
GRPC GRPCConfig `yaml:"grpc"`
|
||||
HTTP HTTP `yaml:"http"`
|
||||
|
||||
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
GrpcClient client.Client `yaml:"-"`
|
||||
|
||||
Thumbnail Thumbnail `yaml:"thumbnail"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// FileSystemStorage defines the available filesystem storage configuration.
|
||||
type FileSystemStorage struct {
|
||||
RootDirectory string `yaml:"root_directory" env:"THUMBNAILS_FILESYSTEMSTORAGE_ROOT" desc:"The directory where the filesystem storage will store the thumbnails. If not defined, the root directory derives from $OC_BASE_DATA_PATH/thumbnails." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// Thumbnail defines the available thumbnail related configuration.
|
||||
type Thumbnail struct {
|
||||
Resolutions []string `yaml:"resolutions" env:"THUMBNAILS_RESOLUTIONS" desc:"The supported list of target resolutions in the format WidthxHeight like 32x32. You can define any resolution as required. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
FileSystemStorage FileSystemStorage `yaml:"filesystem_storage"`
|
||||
WebdavAllowInsecure bool `yaml:"webdav_allow_insecure" env:"OC_INSECURE;THUMBNAILS_WEBDAVSOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the webdav source." introductionVersion:"1.0.0"`
|
||||
CS3AllowInsecure bool `yaml:"cs3_allow_insecure" env:"OC_INSECURE;THUMBNAILS_CS3SOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the CS3 source." introductionVersion:"1.0.0"`
|
||||
RevaGateway string `yaml:"reva_gateway" env:"OC_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata" introductionVersion:"1.0.0"`
|
||||
FontMapFile string `yaml:"font_map_file" env:"THUMBNAILS_TXT_FONTMAP_FILE" desc:"The path to a font file for txt thumbnails." introductionVersion:"1.0.0"`
|
||||
TransferSecret string `yaml:"transfer_secret" env:"THUMBNAILS_TRANSFER_TOKEN" desc:"The secret to sign JWT to download the actual thumbnail file." introductionVersion:"1.0.0"`
|
||||
DataEndpoint string `yaml:"data_endpoint" env:"THUMBNAILS_DATA_ENDPOINT" desc:"The HTTP endpoint where the actual thumbnail file can be downloaded." introductionVersion:"1.0.0"`
|
||||
MaxInputWidth int `yaml:"max_input_width" env:"THUMBNAILS_MAX_INPUT_WIDTH" desc:"The maximum width of an input image which is being processed." introductionVersion:"1.0.0"`
|
||||
MaxInputHeight int `yaml:"max_input_height" env:"THUMBNAILS_MAX_INPUT_HEIGHT" desc:"The maximum height of an input image which is being processed." introductionVersion:"1.0.0"`
|
||||
MaxInputImageFileSize string `yaml:"max_input_image_file_size" env:"THUMBNAILS_MAX_INPUT_IMAGE_FILE_SIZE" desc:"The maximum file size of an input image which is being processed. Usable common abbreviations: [KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package config
|
||||
|
||||
// Debug defines the available debug configuration.
|
||||
type Debug struct {
|
||||
Addr string `yaml:"addr" env:"THUMBNAILS_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
|
||||
Token string `yaml:"token" env:"THUMBNAILS_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
|
||||
Pprof bool `yaml:"pprof" env:"THUMBNAILS_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
|
||||
Zpages bool `yaml:"zpages" env:"THUMBNAILS_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/defaults"
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"github.com/qsfera/server/pkg/structs"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
)
|
||||
|
||||
// FullDefaultConfig returns a fully initialized default configuration
|
||||
func FullDefaultConfig() *config.Config {
|
||||
cfg := DefaultConfig()
|
||||
EnsureDefaults(cfg)
|
||||
Sanitize(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultConfig returns a basic default configuration
|
||||
func DefaultConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Debug: config.Debug{
|
||||
Addr: "127.0.0.1:9189",
|
||||
Token: "",
|
||||
Pprof: false,
|
||||
Zpages: false,
|
||||
},
|
||||
GRPC: config.GRPCConfig{
|
||||
Addr: "127.0.0.1:9185",
|
||||
Namespace: "qsfera.api",
|
||||
MaxConcurrentRequests: 0,
|
||||
},
|
||||
HTTP: config.HTTP{
|
||||
Addr: "127.0.0.1:9186",
|
||||
Root: "/thumbnails",
|
||||
Namespace: "qsfera.web",
|
||||
CORS: config.CORS{
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"},
|
||||
AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With", "X-Request-Id", "Cache-Control"},
|
||||
AllowCredentials: true,
|
||||
},
|
||||
},
|
||||
Service: config.Service{
|
||||
Name: "thumbnails",
|
||||
},
|
||||
Thumbnail: config.Thumbnail{
|
||||
Resolutions: []string{"16x16", "32x32", "64x64", "128x128", "1080x1920", "1920x1080", "2160x3840", "3840x2160", "4320x7680", "7680x4320"},
|
||||
FileSystemStorage: config.FileSystemStorage{
|
||||
RootDirectory: path.Join(defaults.BaseDataPath(), "thumbnails"),
|
||||
},
|
||||
WebdavAllowInsecure: false,
|
||||
RevaGateway: shared.DefaultRevaConfig().Address,
|
||||
CS3AllowInsecure: false,
|
||||
DataEndpoint: "http://127.0.0.1:9186/thumbnails/data",
|
||||
MaxInputWidth: 7680,
|
||||
MaxInputHeight: 7680,
|
||||
MaxInputImageFileSize: "50MB",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDefaults adds default values to the configuration if they are not set yet
|
||||
func EnsureDefaults(cfg *config.Config) {
|
||||
if cfg.LogLevel == "" {
|
||||
cfg.LogLevel = "error"
|
||||
}
|
||||
|
||||
if cfg.GRPCClientTLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS)
|
||||
}
|
||||
if cfg.GRPC.TLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPC.TLS = structs.CopyOrZeroValue(cfg.Commons.GRPCServiceTLS)
|
||||
}
|
||||
|
||||
if cfg.Commons != nil {
|
||||
cfg.HTTP.TLS = cfg.Commons.HTTPServiceTLS
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize sanitized the configuration
|
||||
func Sanitize(cfg *config.Config) {
|
||||
// nothing to sanitize here atm
|
||||
if len(cfg.Thumbnail.Resolutions) == 1 && strings.Contains(cfg.Thumbnail.Resolutions[0], ",") {
|
||||
cfg.Thumbnail.Resolutions = strings.Split(cfg.Thumbnail.Resolutions[0], ",")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package config
|
||||
|
||||
import "github.com/qsfera/server/pkg/shared"
|
||||
|
||||
// GRPCConfig defines the available grpc configuration.
|
||||
type GRPCConfig struct {
|
||||
Addr string `yaml:"addr" env:"THUMBNAILS_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
MaxConcurrentRequests int `yaml:"max_concurrent_requests" env:"THUMBNAILS_MAX_CONCURRENT_REQUESTS" desc:"Number of maximum concurrent thumbnail requests. Default is 0 which is unlimited." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package config
|
||||
|
||||
import "github.com/qsfera/server/pkg/shared"
|
||||
|
||||
// CORS defines the available cors configuration.
|
||||
type CORS struct {
|
||||
AllowedOrigins []string `yaml:"allow_origins" env:"OC_CORS_ALLOW_ORIGINS;THUMBNAILS_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AllowedMethods []string `yaml:"allow_methods" env:"OC_CORS_ALLOW_METHODS;THUMBNAILS_CORS_ALLOW_METHODS" desc:"A list of allowed CORS methods. See following chapter for more details: *Access-Control-Request-Method* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Method. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AllowedHeaders []string `yaml:"allow_headers" env:"OC_CORS_ALLOW_HEADERS;THUMBNAILS_CORS_ALLOW_HEADERS" desc:"A list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AllowCredentials bool `yaml:"allow_credentials" env:"OC_CORS_ALLOW_CREDENTIALS;THUMBNAILS_CORS_ALLOW_CREDENTIALS" desc:"Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Addr string `yaml:"addr" env:"THUMBNAILS_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
TLS shared.HTTPServiceTLS `yaml:"tls"`
|
||||
Root string `yaml:"root" env:"THUMBNAILS_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
CORS CORS `yaml:"cors"`
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config/defaults"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/envdecode"
|
||||
)
|
||||
|
||||
// ParseConfig loads configuration from known paths.
|
||||
func ParseConfig(cfg *config.Config) error {
|
||||
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
// load all env variables relevant to the config in the current context.
|
||||
if err := envdecode.Decode(cfg); err != nil {
|
||||
// no environment variable set for this config is an expected "error"
|
||||
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sanitize config
|
||||
defaults.Sanitize(cfg)
|
||||
|
||||
return Validate(cfg)
|
||||
}
|
||||
|
||||
// Validate can validate the configuration
|
||||
func Validate(_ *config.Config) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package config
|
||||
|
||||
// Service defines the available service configuration.
|
||||
type Service struct {
|
||||
Name string `yaml:"-"`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package errors
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrImageTooLarge defines an error when an input image is too large
|
||||
ErrImageTooLarge = errors.New("thumbnails: image is too large")
|
||||
// ErrInvalidType represents the error when a type can't be encoded.
|
||||
ErrInvalidType = errors.New("thumbnails: can't encode this type")
|
||||
// ErrNoEncoderForType represents the error when an encoder couldn't be found for a type.
|
||||
ErrNoEncoderForType = errors.New("thumbnails: no encoder for this type found")
|
||||
// ErrNoGeneratorForType represents the error when a generator couldn't be found for a type.
|
||||
ErrNoGeneratorForType = errors.New("thumbnails: no generator for this type found")
|
||||
// ErrNoImageFromAudioFile defines an error when an image cannot be extracted from an audio file
|
||||
ErrNoImageFromAudioFile = errors.New("thumbnails: could not extract image from audio file")
|
||||
// ErrNoConverterForExtractedImageFromGgsFile defines an error when the extracted image from an ggs file could not be converted
|
||||
ErrNoConverterForExtractedImageFromGgsFile = errors.New("thumbnails: could not find converter for image extracted from ggs file")
|
||||
// ErrNoConverterForExtractedImageFromAudioFile defines an error when the extracted image from an audio file could not be converted
|
||||
ErrNoConverterForExtractedImageFromAudioFile = errors.New("thumbnails: could not find converter for image extracted from audio file")
|
||||
// ErrCS3AuthorizationMissing defines an error when the CS3 authorization is missing
|
||||
ErrCS3AuthorizationMissing = errors.New("thumbnails: cs3source - authorization missing")
|
||||
)
|
||||
@@ -0,0 +1,67 @@
|
||||
package metrics
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
var (
|
||||
// Namespace defines the namespace for the defines metrics.
|
||||
Namespace = "qsfera"
|
||||
|
||||
// Subsystem defines the subsystem for the defines metrics.
|
||||
Subsystem = "thumbnails"
|
||||
)
|
||||
|
||||
// Metrics defines the available metrics of this service.
|
||||
type Metrics struct {
|
||||
Counter *prometheus.CounterVec
|
||||
Latency *prometheus.SummaryVec
|
||||
Duration *prometheus.HistogramVec
|
||||
BuildInfo *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// New initializes the available metrics.
|
||||
func New() *Metrics {
|
||||
m := &Metrics{
|
||||
Counter: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "getthumbnail_total",
|
||||
Help: "How many GetThumbnail requests processed",
|
||||
}, []string{}),
|
||||
Latency: prometheus.NewSummaryVec(prometheus.SummaryOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "getthumbnail_latency_microseconds",
|
||||
Help: "GetThumbnail request latencies in microseconds",
|
||||
}, []string{}),
|
||||
Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "getthumbnail_duration_seconds",
|
||||
Help: "GetThumbnail method requests time in seconds",
|
||||
}, []string{}),
|
||||
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "build_info",
|
||||
Help: "Build information",
|
||||
}, []string{"version"}),
|
||||
}
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.Counter,
|
||||
)
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.Latency,
|
||||
)
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.Duration,
|
||||
)
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.BuildInfo,
|
||||
)
|
||||
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/sync"
|
||||
"golang.org/x/image/font"
|
||||
"golang.org/x/image/font/gofont/goregular"
|
||||
"golang.org/x/image/font/opentype"
|
||||
)
|
||||
|
||||
// FontMap maps a script with the target font to be used for that script
|
||||
// It also uses a DefaultFont in case there isn't a matching script in the map
|
||||
//
|
||||
// For cases like Japanese where multiple scripts are used, we rely on the text
|
||||
// analyzer to use the script which is unique to japanese (Hiragana or Katakana)
|
||||
// even if it has to overwrite the "official" detected script (Han). This means
|
||||
// that "Han" should be used just for chinese while "Hiragana" and "Katakana"
|
||||
// should be used for japanese
|
||||
type FontMap struct {
|
||||
FontMap map[string]string `json:"fontMap"`
|
||||
DefaultFont string `json:"defaultFont"`
|
||||
}
|
||||
|
||||
// FontMapData contains the location of the loaded file (in FLoc) and the FontMap loaded
|
||||
// from the file
|
||||
type FontMapData struct {
|
||||
FMap *FontMap
|
||||
FLoc string
|
||||
}
|
||||
|
||||
// LoadedFace contains the location of the font used, and the loaded face (font.Face)
|
||||
// ready to be used
|
||||
type LoadedFace struct {
|
||||
FontFile string
|
||||
Face font.Face
|
||||
}
|
||||
|
||||
// FontLoader represents a FontLoader. Use the "NewFontLoader" to get a instance
|
||||
type FontLoader struct {
|
||||
faceCache sync.Cache
|
||||
fontMapData *FontMapData
|
||||
faceOpts *opentype.FaceOptions
|
||||
}
|
||||
|
||||
// NewFontLoader creates a new FontLoader based on the fontMapFile. The FaceOptions will
|
||||
// be the same for all the font loaded by this instance.
|
||||
// Note that only the fonts described in the fontMapFile will be used.
|
||||
//
|
||||
// The fontMapFile has the following structure
|
||||
//
|
||||
// {
|
||||
// "fontMap": {
|
||||
// "Han": "packaged/myFont-CJK.otf",
|
||||
// "Arabic": "packaged/myFont-Arab.otf",
|
||||
// "Latin": "/fonts/regular/myFont.otf"
|
||||
// }
|
||||
// "defaultFont": "/fonts/regular/myFont.otf"
|
||||
// }
|
||||
//
|
||||
// The fontMapFile contains paths to where the fonts are located in the FS.
|
||||
// Absolute paths can be used as shown above. If a relative path is used,
|
||||
// it will be relative to the fontMapFile location. This should make the
|
||||
// packaging easier since all the fonts can be placed in the same directory
|
||||
// where the fontMapFile is, or in inner directories.
|
||||
func NewFontLoader(fontMapFile string, faceOpts *opentype.FaceOptions) (*FontLoader, error) {
|
||||
fontMap := &FontMap{}
|
||||
|
||||
if fontMapFile != "" {
|
||||
file, err := os.Open(fontMapFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
parser := json.NewDecoder(file)
|
||||
if err = parser.Decode(fontMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return &FontLoader{
|
||||
faceCache: sync.NewCache(5),
|
||||
fontMapData: &FontMapData{
|
||||
FMap: fontMap,
|
||||
FLoc: fontMapFile,
|
||||
},
|
||||
faceOpts: faceOpts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LoadFaceForScript loads and returns the font face to be used for that script according to the
|
||||
// FontMap set when the FontLoader was created. If the script doesn't have
|
||||
// an associated font, a default font will be used. Note that the default font
|
||||
// might not be able to handle properly the script
|
||||
func (fl *FontLoader) LoadFaceForScript(script string) (*LoadedFace, error) {
|
||||
var parsedFont *opentype.Font
|
||||
var parsingError error
|
||||
|
||||
fontFile := fl.fontMapData.FMap.DefaultFont
|
||||
if val, ok := fl.fontMapData.FMap.FontMap[script]; ok {
|
||||
fontFile = val
|
||||
}
|
||||
|
||||
if fontFile != "" && !filepath.IsAbs(fontFile) {
|
||||
fontFile = filepath.Join(filepath.Dir(fl.fontMapData.FLoc), fontFile)
|
||||
}
|
||||
|
||||
// if the face for the script isn't cached, load the font file and create a new face
|
||||
cachedFace := fl.faceCache.Load(fontFile)
|
||||
if cachedFace != nil {
|
||||
return cachedFace.V.(*LoadedFace), nil
|
||||
}
|
||||
|
||||
if fontFile == "" {
|
||||
parsedFont, parsingError = opentype.Parse(goregular.TTF)
|
||||
if parsingError != nil {
|
||||
return nil, parsingError
|
||||
}
|
||||
} else {
|
||||
// opentype.ParseReaderAt seems to require to keep the file opened
|
||||
// so read the font file into memory
|
||||
data, err := os.ReadFile(fontFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsedFont, parsingError = opentype.Parse(data)
|
||||
if parsingError != nil {
|
||||
return nil, parsingError
|
||||
}
|
||||
}
|
||||
|
||||
face, err := opentype.NewFace(parsedFont, fl.faceOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
loadedFace := &LoadedFace{
|
||||
FontFile: fontFile,
|
||||
Face: face,
|
||||
}
|
||||
fl.faceCache.Store(fontFile, loadedFace, time.Now().Add(10*time.Minute))
|
||||
return loadedFace, nil
|
||||
}
|
||||
|
||||
// GetFaceOptSize returns face opt size
|
||||
func (fl *FontLoader) GetFaceOptSize() float64 {
|
||||
return fl.faceOpts.Size
|
||||
}
|
||||
|
||||
// GetFaceOptDPI returns face opt DPI
|
||||
func (fl *FontLoader) GetFaceOptDPI() float64 {
|
||||
return fl.faceOpts.DPI
|
||||
}
|
||||
|
||||
// GetScriptList returns script list
|
||||
func (fl *FontLoader) GetScriptList() []string {
|
||||
fontMap := fl.fontMapData.FMap.FontMap
|
||||
|
||||
arePresent := map[string]bool{
|
||||
"Common": false,
|
||||
"Inherited": false,
|
||||
}
|
||||
listSize := len(fontMap)
|
||||
|
||||
for key := range arePresent {
|
||||
if _, inFontMap := fontMap[key]; inFontMap {
|
||||
arePresent[key] = true
|
||||
} else {
|
||||
listSize++
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, listSize)
|
||||
|
||||
i := 0
|
||||
for k := range fontMap {
|
||||
keys[i] = k
|
||||
i++
|
||||
}
|
||||
|
||||
for script, isPresent := range arePresent {
|
||||
if !isPresent {
|
||||
keys[i] = script
|
||||
i++
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"image/draw"
|
||||
"image/gif"
|
||||
"io"
|
||||
"math"
|
||||
"mime"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/image/font"
|
||||
"golang.org/x/image/font/opentype"
|
||||
"golang.org/x/image/math/fixed"
|
||||
|
||||
"github.com/dhowden/tag"
|
||||
|
||||
thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// FileConverter is the interface for the file converter
|
||||
type FileConverter interface {
|
||||
Convert(r io.Reader) (any, error)
|
||||
}
|
||||
|
||||
// GifDecoder is a converter for the gif file
|
||||
type GifDecoder struct{}
|
||||
|
||||
// Convert reads the gif file and returns the thumbnail image
|
||||
func (i GifDecoder) Convert(r io.Reader) (any, error) {
|
||||
img, err := gif.DecodeAll(r)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, `could not decode the image`)
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// GgsDecoder is a converter for the geogebra slides file
|
||||
type GgsDecoder struct{ thumbnailpath string }
|
||||
|
||||
// Convert reads the ggs file and returns the thumbnail image
|
||||
func (g GgsDecoder) Convert(r io.Reader) (any, error) {
|
||||
var buf bytes.Buffer
|
||||
_, err := io.Copy(&buf, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zipReader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, file := range zipReader.File {
|
||||
if file.Name == g.thumbnailpath {
|
||||
thumbnail, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
converter := ForType("image/png", nil)
|
||||
if converter == nil {
|
||||
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromGgsFile
|
||||
}
|
||||
img, err := converter.Convert(thumbnail)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, `could not decode the image`)
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
}
|
||||
return nil, errors.Errorf("%s not found", g.thumbnailpath)
|
||||
}
|
||||
|
||||
// AudioDecoder is a converter for the audio file
|
||||
type AudioDecoder struct{}
|
||||
|
||||
// Convert reads the audio file and extracts the thumbnail image from the id3 tag
|
||||
func (i AudioDecoder) Convert(r io.Reader) (any, error) {
|
||||
b, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := tag.ReadFrom(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
picture := m.Picture()
|
||||
if picture == nil {
|
||||
return nil, thumbnailerErrors.ErrNoImageFromAudioFile
|
||||
}
|
||||
|
||||
converter := ForType(picture.MIMEType, nil)
|
||||
if converter == nil {
|
||||
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
|
||||
}
|
||||
|
||||
return converter.Convert(bytes.NewReader(picture.Data))
|
||||
}
|
||||
|
||||
// TxtToImageConverter is a converter for the text file
|
||||
type TxtToImageConverter struct {
|
||||
fontLoader *FontLoader
|
||||
}
|
||||
|
||||
// Convert reads the text file and renders it into a thumbnail image
|
||||
func (t TxtToImageConverter) Convert(r io.Reader) (any, error) {
|
||||
img := image.NewRGBA(image.Rect(0, 0, 640, 480))
|
||||
|
||||
imgBounds := img.Bounds()
|
||||
draw.Draw(img, imgBounds, image.White, image.Point{}, draw.Src)
|
||||
|
||||
fontSizeAsInt := int(math.Ceil(t.fontLoader.GetFaceOptSize()))
|
||||
margin := 10
|
||||
minX := fixed.I(imgBounds.Min.X + margin)
|
||||
maxX := fixed.I(imgBounds.Max.X - margin)
|
||||
maxY := fixed.I(imgBounds.Max.Y - margin)
|
||||
initialPoint := fixed.P(imgBounds.Min.X+margin, imgBounds.Min.Y+margin+fontSizeAsInt)
|
||||
canvas := &font.Drawer{
|
||||
Dst: img,
|
||||
Src: image.Black,
|
||||
Dot: initialPoint,
|
||||
}
|
||||
|
||||
scriptList := t.fontLoader.GetScriptList()
|
||||
textAnalyzer := NewTextAnalyzer(scriptList)
|
||||
taOpts := AnalysisOpts{
|
||||
UseMergeMap: true,
|
||||
MergeMap: DefaultMergeMap,
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
Scan: // Label for the scanner loop, so we can break it easily
|
||||
for scanner.Scan() {
|
||||
txt := scanner.Text()
|
||||
height := fixed.I(fontSizeAsInt) // reset to default height
|
||||
|
||||
textResult := textAnalyzer.AnalyzeString(txt, taOpts)
|
||||
textResult.MergeCommon(DefaultMergeMap)
|
||||
|
||||
for _, sRange := range textResult.ScriptRanges {
|
||||
targetFontFace, err := t.fontLoader.LoadFaceForScript(sRange.TargetScript)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if the target script is "_unknown" it's expected that the loaded face
|
||||
// uses the default font
|
||||
faceHeight := targetFontFace.Face.Metrics().Height
|
||||
if faceHeight > height {
|
||||
height = faceHeight
|
||||
}
|
||||
|
||||
canvas.Face = targetFontFace.Face
|
||||
initialByte := sRange.Low
|
||||
for _, sRangeSpace := range sRange.Spaces {
|
||||
if canvas.Dot.Y > maxY {
|
||||
break Scan
|
||||
}
|
||||
|
||||
drawWord(canvas, textResult.Text[initialByte:sRangeSpace], minX, maxX, height, maxY)
|
||||
initialByte = sRangeSpace
|
||||
}
|
||||
|
||||
if initialByte <= sRange.High {
|
||||
// some bytes left to be written
|
||||
if canvas.Dot.Y > maxY {
|
||||
break Scan
|
||||
}
|
||||
|
||||
drawWord(canvas, textResult.Text[initialByte:sRange.High+1], minX, maxX, height, maxY)
|
||||
}
|
||||
}
|
||||
|
||||
canvas.Dot.X = minX
|
||||
canvas.Dot.Y += height.Mul(fixed.Int26_6(1<<6 + 1<<5)) // height * 1.5
|
||||
|
||||
if canvas.Dot.Y > maxY {
|
||||
break
|
||||
}
|
||||
}
|
||||
return img, scanner.Err()
|
||||
}
|
||||
|
||||
// GGPStruct is the layout of a ggp file (which is basically json)
|
||||
type GGPStruct struct {
|
||||
Sections []struct {
|
||||
Cards []struct {
|
||||
Element struct {
|
||||
Image struct {
|
||||
Base64Image string
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GgpDecoder is a converter for the geogebra pinboard file
|
||||
type GgpDecoder struct{}
|
||||
|
||||
// Convert reads the ggp file and returns the first thumbnail image
|
||||
func (j GgpDecoder) Convert(r io.Reader) (any, error) {
|
||||
ggp := &GGPStruct{}
|
||||
err := json.NewDecoder(r).Decode(ggp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
elem, err := extractBase64ImageFromGGP(ggp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b, err := base64.StdEncoding.DecodeString(elem)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
img, _, err := image.Decode(bytes.NewReader(b))
|
||||
return img, err
|
||||
}
|
||||
|
||||
func extractBase64ImageFromGGP(ggp *GGPStruct) (string, error) {
|
||||
if len(ggp.Sections) < 1 || len(ggp.Sections[0].Cards) < 1 {
|
||||
return "", errors.New("cant find thumbnail in ggp file")
|
||||
}
|
||||
|
||||
raw := strings.Split(ggp.Sections[0].Cards[0].Element.Image.Base64Image, "base64,")
|
||||
if len(raw) < 2 {
|
||||
return "", errors.New("cant decode ggp thumbnail")
|
||||
}
|
||||
|
||||
return raw[1], nil
|
||||
}
|
||||
|
||||
// Draw the word in the canvas. The mixX and maxX defines the drawable range
|
||||
// (X axis) where the word can be drawn (in case the word is too big and doesn't
|
||||
// fit in the canvas), and the incY defines the increment in the Y axis if we
|
||||
// need to draw the word in a new line
|
||||
//
|
||||
// Note that the word will likely start with a white space char
|
||||
func drawWord(canvas *font.Drawer, word string, minX, maxX, incY, maxY fixed.Int26_6) {
|
||||
// calculate the actual measurement of the string at a given X position
|
||||
measure := func(s string, dotX fixed.Int26_6) (min, max fixed.Int26_6) {
|
||||
bbox, _ := canvas.BoundString(s)
|
||||
return dotX + bbox.Min.X, dotX + bbox.Max.X
|
||||
}
|
||||
|
||||
// first try to draw the whole word
|
||||
absMin, absMax := measure(word, canvas.Dot.X)
|
||||
if absMin >= minX && absMax <= maxX {
|
||||
canvas.DrawString(word)
|
||||
return
|
||||
}
|
||||
|
||||
// try to draw the trimmed word in a new line
|
||||
trimmed := strings.TrimSpace(word)
|
||||
oldDot := canvas.Dot
|
||||
canvas.Dot.X = minX
|
||||
canvas.Dot.Y += incY
|
||||
|
||||
if canvas.Dot.Y <= maxY {
|
||||
tMin, tMax := measure(trimmed, canvas.Dot.X)
|
||||
if tMin >= minX && tMax <= maxX {
|
||||
canvas.DrawString(trimmed)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// if the trimmed word is still too big, draw it char by char
|
||||
canvas.Dot = oldDot
|
||||
for _, char := range trimmed {
|
||||
s := string(char)
|
||||
_, cMax := measure(s, canvas.Dot.X)
|
||||
|
||||
if cMax > maxX {
|
||||
canvas.Dot.X = minX
|
||||
canvas.Dot.Y += incY
|
||||
}
|
||||
|
||||
// stop drawing if we exceed maxY
|
||||
if canvas.Dot.Y > maxY {
|
||||
return
|
||||
}
|
||||
|
||||
// ensure that we don't start drawing before minX
|
||||
cMin, _ := measure(s, canvas.Dot.X)
|
||||
if cMin < minX {
|
||||
canvas.Dot.X += minX - cMin
|
||||
}
|
||||
|
||||
canvas.DrawString(s)
|
||||
}
|
||||
}
|
||||
|
||||
// ForType returns the converter for the specified mimeType
|
||||
func ForType(mimeType string, opts map[string]any) FileConverter {
|
||||
// We can ignore the error here because we parse it in IsMimeTypeSupported before and if it fails
|
||||
// return the service call. So we should only get here when the mimeType parses fine.
|
||||
mimeType, _, _ = mime.ParseMediaType(mimeType)
|
||||
switch mimeType {
|
||||
case "text/plain":
|
||||
fontFileMap := ""
|
||||
fontFaceOpts := &opentype.FaceOptions{
|
||||
Size: 12,
|
||||
DPI: 72,
|
||||
Hinting: font.HintingNone,
|
||||
}
|
||||
|
||||
if optedFontFileMap, ok := opts["fontFileMap"]; ok {
|
||||
if stringFontFileMap, ok := optedFontFileMap.(string); ok {
|
||||
fontFileMap = stringFontFileMap
|
||||
}
|
||||
}
|
||||
|
||||
if optedFontFaceOpts, ok := opts["fontFaceOpts"]; ok {
|
||||
if typedFontFaceOpts, ok := optedFontFaceOpts.(*opentype.FaceOptions); ok {
|
||||
fontFaceOpts = typedFontFaceOpts
|
||||
}
|
||||
}
|
||||
|
||||
fontLoader, err := NewFontLoader(fontFileMap, fontFaceOpts)
|
||||
if err != nil {
|
||||
// if it couldn't create the FontLoader with the specified fontFileMap,
|
||||
// try to use the default font
|
||||
fontLoader, _ = NewFontLoader("", fontFaceOpts)
|
||||
}
|
||||
return TxtToImageConverter{
|
||||
fontLoader: fontLoader,
|
||||
}
|
||||
case "application/vnd.geogebra.slides":
|
||||
return GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
case "application/vnd.geogebra.pinboard":
|
||||
return GgpDecoder{}
|
||||
case "image/gif":
|
||||
return GifDecoder{}
|
||||
case "audio/flac":
|
||||
fallthrough
|
||||
case "audio/mpeg":
|
||||
fallthrough
|
||||
case "audio/ogg":
|
||||
return AudioDecoder{}
|
||||
default:
|
||||
return ImageDecoder{}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ImageDecoder is a converter for the image file
|
||||
type ImageDecoder struct{}
|
||||
|
||||
// Convert reads the image file and returns the thumbnail image
|
||||
func (i ImageDecoder) Convert(r io.Reader) (any, error) {
|
||||
img, err := imaging.Decode(r, imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, `could not decode the image`)
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/image/font"
|
||||
"golang.org/x/image/font/opentype"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestImageDecoder(t *testing.T) {
|
||||
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "ImageDecoder Suite")
|
||||
}
|
||||
|
||||
var _ = Describe("ImageDecoder", func() {
|
||||
Describe("ImageDecoder", func() {
|
||||
var fileReader io.Reader
|
||||
BeforeEach(func() {
|
||||
fileContent, err := os.ReadFile("test_assets/noise.png")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fileReader = bytes.NewReader(fileContent)
|
||||
})
|
||||
|
||||
It("should decode an image", func() {
|
||||
decoder := ImageDecoder{}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should return an error if the image is invalid", func() {
|
||||
decoder := ImageDecoder{}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("not an image")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GifDecoder", func() {
|
||||
var fileReader io.Reader
|
||||
BeforeEach(func() {
|
||||
fileContent, err := os.ReadFile("test_assets/noise.gif")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fileReader = bytes.NewReader(fileContent)
|
||||
})
|
||||
|
||||
It("should decode a gif", func() {
|
||||
decoder := GifDecoder{}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should return an error if the gif is invalid", func() {
|
||||
decoder := GifDecoder{}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("not a gif")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GgsDecoder", func() {
|
||||
var fileReader io.Reader
|
||||
BeforeEach(func() {
|
||||
fileContent, err := os.ReadFile("test_assets/ggs_test.ggs")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fileReader = bytes.NewReader(fileContent)
|
||||
})
|
||||
|
||||
It("should decode a ggs", func() {
|
||||
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should return an error if the ggs is invalid", func() {
|
||||
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("not a ggs")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("should decode audio", func() {
|
||||
var fileReader io.Reader
|
||||
It("should decode an audio", func() {
|
||||
fileContent, err := os.ReadFile("test_assets/empty.mp3")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fileReader = bytes.NewReader(fileContent)
|
||||
decoder := AudioDecoder{}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
It("should decode an audio", func() {
|
||||
fileContent, err := os.ReadFile("test_assets/empty_no_image.mp3")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fileReader = bytes.NewReader(fileContent)
|
||||
decoder := AudioDecoder{}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
It("should return an error if the audio is invalid", func() {
|
||||
decoder := AudioDecoder{}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("not an audio")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("should decode text", func() {
|
||||
var decoder TxtToImageConverter
|
||||
BeforeEach(func() {
|
||||
fontFaceOpts := &opentype.FaceOptions{
|
||||
Size: 12,
|
||||
DPI: 72,
|
||||
Hinting: font.HintingNone,
|
||||
}
|
||||
|
||||
fontLoader, err := NewFontLoader("", fontFaceOpts)
|
||||
if err != nil {
|
||||
fontLoader, _ = NewFontLoader("", fontFaceOpts)
|
||||
}
|
||||
decoder = TxtToImageConverter{
|
||||
fontLoader: fontLoader,
|
||||
}
|
||||
})
|
||||
It("should decode a text", func() {
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("fails if the font is missing", func() {
|
||||
decoder.fontLoader.fontMapData = &FontMapData{
|
||||
FMap: &FontMap{
|
||||
DefaultFont: "/some/unknown/font.otf",
|
||||
},
|
||||
}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("test ForType", func() {
|
||||
It("should return an ImageDecoder for image types", func() {
|
||||
decoder := ForType("image/png", nil)
|
||||
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
|
||||
})
|
||||
|
||||
It("should return an GifDecoder for gif types", func() {
|
||||
decoder := ForType("image/gif", nil)
|
||||
Expect(decoder).To(BeAssignableToTypeOf(GifDecoder{}))
|
||||
})
|
||||
|
||||
It("should return an GgsDecoder for ggs types", func() {
|
||||
decoder := ForType("application/vnd.geogebra.ggs", nil)
|
||||
// This will not return the expected ggsDecoder, but an ImageDecoder since ggs contains an embedded png.
|
||||
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
|
||||
})
|
||||
|
||||
It("should return an AudioDecoder for audio types", func() {
|
||||
decoder := ForType("audio/mpeg", nil)
|
||||
Expect(decoder).To(BeAssignableToTypeOf(AudioDecoder{}))
|
||||
})
|
||||
|
||||
It("should return an TxtToImageConverter for text types", func() {
|
||||
decoder := ForType("text/plain", nil)
|
||||
Expect(decoder).To(BeAssignableToTypeOf(TxtToImageConverter{}))
|
||||
})
|
||||
|
||||
It("should return an ImageDecoder for unknown types", func() {
|
||||
decoder := ForType("unknown", nil)
|
||||
Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,20 @@
|
||||
//go:build enable_vips
|
||||
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
)
|
||||
|
||||
func init() {
|
||||
vips.LoggingSettings(nil, vips.LogLevelError)
|
||||
}
|
||||
|
||||
type ImageDecoder struct{}
|
||||
|
||||
func (v ImageDecoder) Convert(r io.Reader) (interface{}, error) {
|
||||
img, err := vips.NewImageFromReader(r)
|
||||
return img, err
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 181 KiB |
@@ -0,0 +1,309 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Default list of scripts to be analyzed within the string.
|
||||
//
|
||||
// Scripts that aren't present in the list will be considered as part
|
||||
// of the last "known" script. For example, if "Avestan" script (which isn't
|
||||
// present) is preceeded by "Arabic" script, then the "Avestan" script will
|
||||
// be considered as "Arabic"
|
||||
//
|
||||
// Punctuation symbols are usually considered part of the "Common" script
|
||||
var DefaultScripts = []string{
|
||||
"Arabic",
|
||||
"Common",
|
||||
"Devanagari",
|
||||
"Han",
|
||||
"Hangul",
|
||||
"Hiragana",
|
||||
"Inherited",
|
||||
"Katakana",
|
||||
"Latin",
|
||||
}
|
||||
|
||||
// Convenient map[string]map[string]string type used to merge multiple
|
||||
// scripts into one. This is mainly used for japanese language which uses
|
||||
// "Han", "Hiragana" and "Katakana" scripts.
|
||||
//
|
||||
// The map contains the expected previous script as first key, the expected
|
||||
// current script as second key, and the resulting script (if both keys
|
||||
// match) as value
|
||||
type MergeMap map[string]map[string]string
|
||||
|
||||
// The default mergeMap containing info for the japanese scripts
|
||||
var DefaultMergeMap = MergeMap{
|
||||
"Han": map[string]string{
|
||||
"Hiragana": "Hiragana",
|
||||
"Katakana": "Katakana",
|
||||
},
|
||||
"Hiragana": map[string]string{
|
||||
"Han": "Hiragana",
|
||||
"Katakana": "Hiragana",
|
||||
},
|
||||
"Katakana": map[string]string{
|
||||
"Han": "Katakana",
|
||||
"Hiragana": "Hiragana",
|
||||
},
|
||||
}
|
||||
|
||||
// Analysis options.
|
||||
type AnalysisOpts struct {
|
||||
UseMergeMap bool
|
||||
MergeMap MergeMap
|
||||
}
|
||||
|
||||
// A script range. The range should be attached to a string which could contain
|
||||
// multiple scripts. The "TargetScript" will go from bytes "Low" to "High"
|
||||
// (both inclusive), and contains a "RuneCount" number of runes or chars
|
||||
// (mostly for debugging purposes).
|
||||
// The Space contains the bytes (inside the range) that are considered as
|
||||
// white space.
|
||||
type ScriptRange struct {
|
||||
Low, High int
|
||||
Spaces []int
|
||||
TargetScript string
|
||||
RuneCount int
|
||||
}
|
||||
|
||||
// The result of a text analysis. It contains the analyzed text, a list of
|
||||
// script ranges (see the ScriptRange type) and a map containing how many
|
||||
// runes have been detected for a particular script.
|
||||
type TextAnalysis struct {
|
||||
ScriptRanges []ScriptRange
|
||||
RuneCount map[string]int
|
||||
Text string
|
||||
}
|
||||
|
||||
// The TextAnalyzer object contains private members. It should be created via
|
||||
// "NewTextAnalyzer" function.
|
||||
type TextAnalyzer struct {
|
||||
scripts map[string]*unicode.RangeTable
|
||||
scriptListCache []string
|
||||
}
|
||||
|
||||
// Create a new TextAnalyzer. A list of scripts must be provided.
|
||||
// You can use the "DefaultScripts" variable for a default list,
|
||||
// although it doesn't contain all the available scripts.
|
||||
// See the unicode.Scripts variable (in the unicode package) for a
|
||||
// full list. Note that using invalid scripts will cause an undefined
|
||||
// behavior
|
||||
func NewTextAnalyzer(scriptList []string) TextAnalyzer {
|
||||
scriptRanges := make(map[string]*unicode.RangeTable, len(scriptList))
|
||||
for _, script := range scriptList {
|
||||
scriptRanges[script] = unicode.Scripts[script]
|
||||
}
|
||||
return TextAnalyzer{
|
||||
scripts: scriptRanges,
|
||||
scriptListCache: scriptList,
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze the target string using the specified options.
|
||||
// A TextAnalysis will be returned with the result of the analysis.
|
||||
func (ta *TextAnalyzer) AnalyzeString(word string, opts AnalysisOpts) TextAnalysis {
|
||||
analysis := TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{},
|
||||
RuneCount: make(map[string]int),
|
||||
Text: word,
|
||||
}
|
||||
|
||||
if len(word) < 1 {
|
||||
return analysis
|
||||
}
|
||||
|
||||
firstRune, runeLen := utf8.DecodeRuneInString(word)
|
||||
|
||||
lastRange := &ScriptRange{
|
||||
Low: 0,
|
||||
Spaces: make([]int, 0),
|
||||
TargetScript: ta.chooseScriptFor(firstRune),
|
||||
}
|
||||
firstRuneIsWhiteSpace := unicode.Is(unicode.White_Space, firstRune)
|
||||
if firstRuneIsWhiteSpace {
|
||||
lastRange.Spaces = append(lastRange.Spaces, 0)
|
||||
}
|
||||
|
||||
runeCount := 1
|
||||
for wordIndex, char := range word[runeLen:] {
|
||||
wordIndex += runeLen // shifted from the original string
|
||||
script := ta.chooseScriptFor(char)
|
||||
|
||||
isWhiteSpace := unicode.Is(unicode.White_Space, char)
|
||||
if script != lastRange.TargetScript {
|
||||
if mapScript, isOk := ta.getMergeMapValue(opts, lastRange.TargetScript, script); isOk {
|
||||
lastRange.TargetScript = mapScript
|
||||
if isWhiteSpace {
|
||||
// TODO: Check if this is dead code.
|
||||
// whitespace should be part of the "Common" script, and the Common
|
||||
// script shouldn't be part of a mergeMap
|
||||
lastRange.Spaces = append(lastRange.Spaces, wordIndex)
|
||||
}
|
||||
runeCount++
|
||||
continue
|
||||
}
|
||||
|
||||
lastRange.High = wordIndex - 1
|
||||
lastRange.RuneCount = runeCount
|
||||
analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
|
||||
if _, exists := analysis.RuneCount[lastRange.TargetScript]; !exists {
|
||||
analysis.RuneCount[lastRange.TargetScript] = 0
|
||||
}
|
||||
analysis.RuneCount[lastRange.TargetScript] += runeCount
|
||||
lastRange = &ScriptRange{
|
||||
Low: wordIndex,
|
||||
Spaces: make([]int, 0),
|
||||
TargetScript: script,
|
||||
}
|
||||
runeCount = 0
|
||||
}
|
||||
runeCount++
|
||||
if isWhiteSpace {
|
||||
lastRange.Spaces = append(lastRange.Spaces, wordIndex)
|
||||
}
|
||||
}
|
||||
|
||||
// close the last range
|
||||
lastRange.High = len(word) - 1
|
||||
lastRange.RuneCount = runeCount
|
||||
analysis.RuneCount[lastRange.TargetScript] += runeCount
|
||||
analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
|
||||
|
||||
return analysis
|
||||
}
|
||||
|
||||
func (ta *TextAnalyzer) chooseScriptFor(char rune) string {
|
||||
script := "_unknown"
|
||||
for scriptIndex, scriptFound := range ta.scriptListCache {
|
||||
// if we can't match with a known script, do nothing and jump to the next char
|
||||
if unicode.Is(ta.scripts[scriptFound], char) {
|
||||
if scriptIndex > 3 {
|
||||
// we might expect more chars with the same script
|
||||
// so move the script first to match it faster next time
|
||||
ta.reorderScriptList(scriptFound)
|
||||
}
|
||||
return scriptFound
|
||||
}
|
||||
}
|
||||
return script
|
||||
}
|
||||
|
||||
// Reorder the scriptListCache in the TextAnalyzer in order to speed up
|
||||
// the next script searches. A "Latin" script is expected to be surrounded
|
||||
// by "Latin" chars, although "Common" script chars might be present too
|
||||
func (ta *TextAnalyzer) reorderScriptList(matchedScript string) {
|
||||
for index, script := range ta.scriptListCache {
|
||||
if script == matchedScript {
|
||||
if index != 0 {
|
||||
// move the script to the first position for a faster matching
|
||||
newList := append([]string{script}, ta.scriptListCache[:index]...)
|
||||
ta.scriptListCache = append(newList, ta.scriptListCache[index+1:]...)
|
||||
}
|
||||
// if index == 0 there is nothing to do: the element is already the first
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get the value from the merge map based on the previous and current scripts.
|
||||
// The information about using the merge map and the actual merge map will be
|
||||
// gotten from the AnalysisOpts passed as parameter
|
||||
func (ta *TextAnalyzer) getMergeMapValue(opts AnalysisOpts, previous, current string) (string, bool) {
|
||||
if opts.UseMergeMap {
|
||||
// This option mainly target japanese chars; multiple scripts can be used
|
||||
// in the same piece of text (Han, Hiragana and Katakana)
|
||||
// Instead of starting a new range, adjust the target script of the last range
|
||||
if expCurrent, currentOk := opts.MergeMap[previous]; currentOk {
|
||||
if expFinal, finalOk := expCurrent[current]; finalOk {
|
||||
return expFinal, finalOk
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Change the "Common" script to the one used in the previous script range.
|
||||
// The ranges will be readjusted and merged if they're adjacent.
|
||||
// This naive approach should be good enough for normal use cases
|
||||
//
|
||||
// The MergeMap is needed in case of the japanese language: the ranges
|
||||
// "Han"-"Common"-"Katakana" might be replaced to "Han"-"Hiragana"-"Katakana"
|
||||
// However, the ranges should be merged together into a big "Hiragana" range.
|
||||
// If the MergeMap isn't needed, use an empty one
|
||||
func (tr *TextAnalysis) MergeCommon(mergeMap MergeMap) {
|
||||
var finalRanges []ScriptRange
|
||||
|
||||
if len(tr.ScriptRanges) < 1 {
|
||||
// no ranges -> nothing to do
|
||||
return
|
||||
}
|
||||
|
||||
previousRange := &ScriptRange{}
|
||||
*previousRange = tr.ScriptRanges[0]
|
||||
for _, sRange := range tr.ScriptRanges[1:] {
|
||||
if previousRange.TargetScript == sRange.TargetScript {
|
||||
previousRange.High = sRange.High
|
||||
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
|
||||
previousRange.RuneCount += sRange.RuneCount
|
||||
} else if sRange.TargetScript == "Common" || sRange.TargetScript == "Inherited" {
|
||||
// new range will be absorbed into the previous one
|
||||
previousRange.High = sRange.High
|
||||
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
|
||||
previousRange.RuneCount += sRange.RuneCount
|
||||
tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
|
||||
tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
|
||||
} else if previousRange.TargetScript == "Common" || previousRange.TargetScript == "Inherited" {
|
||||
// might happen if the text starts with a Common script
|
||||
previousRange.High = sRange.High
|
||||
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
|
||||
tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
|
||||
tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
|
||||
previousRange.RuneCount += sRange.RuneCount
|
||||
previousRange.TargetScript = sRange.TargetScript
|
||||
} else {
|
||||
if mapScript, isOk := tr.getMergeMapValue(mergeMap, previousRange.TargetScript, sRange.TargetScript); isOk {
|
||||
if sRange.TargetScript == mapScript {
|
||||
// the previous range has changed the target script
|
||||
tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
|
||||
tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
|
||||
} else {
|
||||
// new range has been absorbed
|
||||
tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
|
||||
tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
|
||||
}
|
||||
previousRange.TargetScript = mapScript
|
||||
previousRange.High = sRange.High
|
||||
previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
|
||||
previousRange.RuneCount += sRange.RuneCount
|
||||
continue
|
||||
}
|
||||
finalRanges = append(finalRanges, *previousRange)
|
||||
*previousRange = sRange
|
||||
}
|
||||
}
|
||||
|
||||
finalRanges = append(finalRanges, *previousRange)
|
||||
tr.ScriptRanges = finalRanges
|
||||
delete(tr.RuneCount, "Common")
|
||||
delete(tr.RuneCount, "Inherited")
|
||||
for index, rCount := range tr.RuneCount {
|
||||
if rCount == 0 {
|
||||
delete(tr.RuneCount, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tr *TextAnalysis) getMergeMapValue(mMap MergeMap, previous, current string) (string, bool) {
|
||||
// This option mainly target japanese chars; multiple scripts can be used
|
||||
// in the same piece of text (Han, Hiragana and Katakana)
|
||||
// Instead of starting a new range, adjust the target script of the last range
|
||||
if expCurrent, currentOk := mMap[previous]; currentOk {
|
||||
if expFinal, finalOk := expCurrent[current]; finalOk {
|
||||
return expFinal, finalOk
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,614 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
inputs = [18]string{
|
||||
"basic latin",
|
||||
"trailing tab ",
|
||||
"Small text. \"$\", \"£\" and \"¥\" are currencies.",
|
||||
"latin with 🖖",
|
||||
"기본 한국어",
|
||||
"基本的な日本語",
|
||||
"ウーロン茶",
|
||||
"私はエンジニアです",
|
||||
"ティー私はエンジニアです",
|
||||
"私はエンジニアです ティー",
|
||||
"आधारभूत देवनागरी",
|
||||
"mixed 언어 传入 🚀!",
|
||||
"/k͜p/",
|
||||
// ä and a + ¨
|
||||
"ä ä",
|
||||
"базовый русский", // cyrillic script isn't part of our default
|
||||
"latin русский", // latin + cyrillic (cyrillic not supported)
|
||||
" space justified ",
|
||||
"",
|
||||
}
|
||||
)
|
||||
|
||||
func TestAnalyzeString(t *testing.T) {
|
||||
defaultOpts := AnalysisOpts{
|
||||
UseMergeMap: true,
|
||||
MergeMap: DefaultMergeMap,
|
||||
}
|
||||
|
||||
tables := []struct {
|
||||
input string
|
||||
opts AnalysisOpts
|
||||
eOut TextAnalysis
|
||||
}{
|
||||
{
|
||||
input: inputs[0],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 10, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 11},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 11,
|
||||
},
|
||||
Text: inputs[0],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[1],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 12, Spaces: []int{8, 12}, TargetScript: "Latin", RuneCount: 13},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 13,
|
||||
},
|
||||
Text: inputs[1],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[2],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 45, Spaces: []int{5, 11, 16, 21, 25, 30, 34}, TargetScript: "Latin", RuneCount: 44},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 44,
|
||||
},
|
||||
Text: inputs[2],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[3],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 14, Spaces: []int{5, 10}, TargetScript: "Latin", RuneCount: 12},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 12,
|
||||
},
|
||||
Text: inputs[3],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[4],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 15, Spaces: []int{6}, TargetScript: "Hangul", RuneCount: 6},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hangul": 6,
|
||||
},
|
||||
Text: inputs[4],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[5],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 7},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hiragana": 7,
|
||||
},
|
||||
Text: inputs[5],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[6],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 14, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Katakana": 5,
|
||||
},
|
||||
Text: inputs[6],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[7],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 9},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hiragana": 9,
|
||||
},
|
||||
Text: inputs[7],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[8],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 12},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hiragana": 12,
|
||||
},
|
||||
Text: inputs[8],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[9],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 36, Spaces: []int{27}, TargetScript: "Hiragana", RuneCount: 13},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hiragana": 13,
|
||||
},
|
||||
Text: inputs[9],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[10],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 45, Spaces: []int{21}, TargetScript: "Devanagari", RuneCount: 16},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Devanagari": 16,
|
||||
},
|
||||
Text: inputs[10],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[11],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
|
||||
{Low: 6, High: 12, Spaces: []int{12}, TargetScript: "Hangul", RuneCount: 3},
|
||||
{Low: 13, High: 24, Spaces: []int{19}, TargetScript: "Han", RuneCount: 5}, // 🚀 and ! are "Common" script and will be merged with "Han"
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 6,
|
||||
"Hangul": 3,
|
||||
"Han": 5,
|
||||
},
|
||||
Text: inputs[11],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[12],
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 5,
|
||||
},
|
||||
Text: inputs[12],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[13], // ä and a + ¨
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{2}, TargetScript: "Latin", RuneCount: 4},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 4,
|
||||
},
|
||||
Text: inputs[13],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[14], // cyrillic script isn't part of our default
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 28, Spaces: []int{14}, TargetScript: "_unknown", RuneCount: 15},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"_unknown": 15,
|
||||
},
|
||||
Text: inputs[14],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
|
||||
{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 6,
|
||||
"_unknown": 7,
|
||||
},
|
||||
Text: inputs[15],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[16], // latin + cyrillic (cyrillic script isn't part of our default)
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 16, Spaces: []int{0, 6, 16}, TargetScript: "Latin", RuneCount: 17},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 17,
|
||||
},
|
||||
Text: inputs[16],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[17], // latin + cyrillic (cyrillic script isn't part of our default)
|
||||
opts: defaultOpts,
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{},
|
||||
RuneCount: map[string]int{},
|
||||
Text: inputs[17],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
testname := fmt.Sprintf("Analyzing \"%s\" string", table.input)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
ta := NewTextAnalyzer(DefaultScripts)
|
||||
result := ta.AnalyzeString(table.input, table.opts)
|
||||
if table.opts.UseMergeMap {
|
||||
result.MergeCommon(table.opts.MergeMap)
|
||||
} else {
|
||||
result.MergeCommon(MergeMap{})
|
||||
}
|
||||
assert.Equal(t, table.eOut, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeStringRaw(t *testing.T) {
|
||||
tables := []struct {
|
||||
input string
|
||||
eOut TextAnalysis
|
||||
}{
|
||||
{
|
||||
input: inputs[0],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 6, High: 10, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 10,
|
||||
"Common": 1,
|
||||
},
|
||||
Text: inputs[0],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[1],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 7, Spaces: []int{}, TargetScript: "Latin", RuneCount: 8},
|
||||
{Low: 8, High: 8, Spaces: []int{8}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
|
||||
{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 11,
|
||||
"Common": 2,
|
||||
},
|
||||
Text: inputs[1],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[2],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
|
||||
{Low: 10, High: 21, Spaces: []int{11, 16, 21}, TargetScript: "Common", RuneCount: 11}, // £ takes 2 bytes
|
||||
{Low: 22, High: 24, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
|
||||
{Low: 25, High: 30, Spaces: []int{25, 30}, TargetScript: "Common", RuneCount: 5}, // ¥ takes 2 bytes
|
||||
{Low: 31, High: 33, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
|
||||
{Low: 34, High: 34, Spaces: []int{34}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 35, High: 44, Spaces: []int{}, TargetScript: "Latin", RuneCount: 10},
|
||||
{Low: 45, High: 45, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 25,
|
||||
"Common": 19,
|
||||
},
|
||||
Text: inputs[2],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[3],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
|
||||
{Low: 10, High: 14, Spaces: []int{10}, TargetScript: "Common", RuneCount: 2},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 9,
|
||||
"Common": 3,
|
||||
},
|
||||
Text: inputs[3],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[4],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
|
||||
{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 3},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hangul": 5,
|
||||
"Common": 1,
|
||||
},
|
||||
Text: inputs[4],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[5],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 8, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
|
||||
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
|
||||
{Low: 12, High: 20, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Hiragana": 1,
|
||||
"Han": 6,
|
||||
},
|
||||
Text: inputs[5],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[6],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 1},
|
||||
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1}, // ー U+30FC (KATAKANA-HIRAGANA PROLONGED SOUND MARK) seems to be counted as Common
|
||||
{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
|
||||
{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Katakana": 3,
|
||||
"Common": 1,
|
||||
"Han": 1,
|
||||
},
|
||||
Text: inputs[6],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[7],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
|
||||
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
|
||||
{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
|
||||
{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Han": 1,
|
||||
"Hiragana": 3,
|
||||
"Katakana": 5,
|
||||
},
|
||||
Text: inputs[7],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[8],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
|
||||
{Low: 6, High: 8, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
|
||||
{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
|
||||
{Low: 15, High: 29, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
|
||||
{Low: 30, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Han": 1,
|
||||
"Hiragana": 3,
|
||||
"Katakana": 7,
|
||||
"Common": 1,
|
||||
},
|
||||
Text: inputs[8],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[9],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
|
||||
{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
|
||||
{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
|
||||
{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
|
||||
{Low: 27, High: 27, Spaces: []int{27}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 28, High: 33, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
|
||||
{Low: 34, High: 36, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Han": 1,
|
||||
"Hiragana": 3,
|
||||
"Katakana": 7,
|
||||
"Common": 2,
|
||||
},
|
||||
Text: inputs[9],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[10],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 7},
|
||||
{Low: 21, High: 21, Spaces: []int{21}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 22, High: 45, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 8},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Devanagari": 15,
|
||||
"Common": 1,
|
||||
},
|
||||
Text: inputs[10],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[11],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
|
||||
{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 13, High: 18, Spaces: []int{}, TargetScript: "Han", RuneCount: 2},
|
||||
{Low: 19, High: 24, Spaces: []int{19}, TargetScript: "Common", RuneCount: 3},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 5,
|
||||
"Hangul": 2,
|
||||
"Han": 2,
|
||||
"Common": 5,
|
||||
},
|
||||
Text: inputs[11],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[12],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 0, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 1, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
|
||||
{Low: 2, High: 3, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
|
||||
{Low: 4, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
|
||||
{Low: 5, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 2,
|
||||
"Common": 2,
|
||||
"Inherited": 1,
|
||||
},
|
||||
Text: inputs[12],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[13], // ä and a + ¨
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
|
||||
{Low: 2, High: 2, Spaces: []int{2}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 3, High: 3, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
|
||||
{Low: 4, High: 5, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 2,
|
||||
"Common": 1,
|
||||
"Inherited": 1,
|
||||
},
|
||||
Text: inputs[13],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[14], // cyrillic script isn't part of our default
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 13, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
|
||||
{Low: 14, High: 14, Spaces: []int{14}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 15, High: 28, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"_unknown": 14,
|
||||
"Common": 1,
|
||||
},
|
||||
Text: inputs[14],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 5,
|
||||
"Common": 1,
|
||||
"_unknown": 7,
|
||||
},
|
||||
Text: inputs[15],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[16],
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{
|
||||
{Low: 0, High: 0, Spaces: []int{0}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 1, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
|
||||
{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
|
||||
{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Latin", RuneCount: 9},
|
||||
{Low: 16, High: 16, Spaces: []int{16}, TargetScript: "Common", RuneCount: 1},
|
||||
},
|
||||
RuneCount: map[string]int{
|
||||
"Latin": 14,
|
||||
"Common": 3,
|
||||
},
|
||||
Text: inputs[16],
|
||||
},
|
||||
},
|
||||
{
|
||||
input: inputs[17], // empty string
|
||||
eOut: TextAnalysis{
|
||||
ScriptRanges: []ScriptRange{},
|
||||
RuneCount: map[string]int{},
|
||||
Text: inputs[17],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
testname := fmt.Sprintf("Raw-Analyzing \"%s\" string", table.input)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
ta := NewTextAnalyzer(DefaultScripts)
|
||||
result := ta.AnalyzeString(table.input, AnalysisOpts{})
|
||||
|
||||
assert.Equal(t, table.eOut, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/checks"
|
||||
"github.com/qsfera/server/pkg/handlers"
|
||||
"github.com/qsfera/server/pkg/service/debug"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
)
|
||||
|
||||
// Server initializes the debug service and server.
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
checkHandler := handlers.NewCheckHandler(
|
||||
handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("web reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr)).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr)),
|
||||
)
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
debug.Name(options.Config.Service.Name),
|
||||
debug.Version(version.GetString()),
|
||||
debug.Address(options.Config.Debug.Addr),
|
||||
debug.Token(options.Config.Debug.Token),
|
||||
debug.Pprof(options.Config.Debug.Pprof),
|
||||
debug.Zpages(options.Config.Debug.Zpages),
|
||||
debug.Health(checkHandler),
|
||||
debug.Ready(checkHandler),
|
||||
debug.CorsAllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins),
|
||||
debug.CorsAllowedMethods(options.Config.HTTP.CORS.AllowedMethods),
|
||||
debug.CorsAllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders),
|
||||
debug.CorsAllowCredentials(options.Config.HTTP.CORS.AllowCredentials),
|
||||
), nil
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Namespace string
|
||||
TraceProvider trace.TracerProvider
|
||||
MaxConcurrentRequests int
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Name provides a name for the service.
|
||||
func Name(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Name = val
|
||||
}
|
||||
}
|
||||
|
||||
// Address provides an address for the service.
|
||||
func Address(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Address = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides a function to set the namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to set the TraceProvider option
|
||||
func TraceProvider(val trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = val
|
||||
}
|
||||
}
|
||||
|
||||
// MaxConcurrentRequests provides a function to set the MaxConcurrentRequests option.
|
||||
func MaxConcurrentRequests(val int) Option {
|
||||
return func(o *Options) {
|
||||
o.MaxConcurrentRequests = val
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/pkg/registry"
|
||||
"github.com/qsfera/server/pkg/service/grpc"
|
||||
"github.com/qsfera/server/pkg/service/grpc/handler/ratelimiter"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
svc "github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0/decorators"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
// NewService initializes the grpc service and server.
|
||||
func NewService(opts ...Option) grpc.Service {
|
||||
options := newOptions(opts...)
|
||||
|
||||
service, err := grpc.NewServiceWithClient(
|
||||
options.Config.GrpcClient,
|
||||
grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled),
|
||||
grpc.TLSCert(
|
||||
options.Config.GRPC.TLS.Cert,
|
||||
options.Config.GRPC.TLS.Key,
|
||||
),
|
||||
grpc.Logger(options.Logger),
|
||||
grpc.Namespace(options.Namespace),
|
||||
grpc.Name(options.Name),
|
||||
grpc.Version(version.GetString()),
|
||||
grpc.Address(options.Address),
|
||||
grpc.Context(options.Context),
|
||||
grpc.Version(version.GetString()),
|
||||
grpc.TraceProvider(options.TraceProvider),
|
||||
grpc.HandlerWrappers(ratelimiter.NewHandlerWrapper(options.MaxConcurrentRequests)),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating thumbnail service")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
tconf := options.Config.Thumbnail
|
||||
tm, err := pool.StringToTLSMode(options.Config.GRPCClientTLS.Mode)
|
||||
if err != nil {
|
||||
options.Logger.Error().Err(err).Msg("could not get gateway client tls mode")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
gatewaySelector, err := pool.GatewaySelector(tconf.RevaGateway,
|
||||
pool.WithTLSCACert(options.Config.GRPCClientTLS.CACert),
|
||||
pool.WithTLSMode(tm),
|
||||
pool.WithRegistry(registry.GetRegistry()),
|
||||
pool.WithTracerProvider(options.TraceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Error().Err(err).Msg("could not get gateway selector")
|
||||
return grpc.Service{}
|
||||
}
|
||||
b, err := bytesize.Parse(tconf.MaxInputImageFileSize)
|
||||
if err != nil {
|
||||
options.Logger.Error().Err(err).Msg("could not parse MaxInputImageFileSize")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
var thumbnail decorators.DecoratedService
|
||||
{
|
||||
thumbnail = svc.NewService(
|
||||
svc.Config(options.Config),
|
||||
svc.Logger(options.Logger),
|
||||
svc.ThumbnailSource(imgsource.NewWebDavSource(tconf, b)),
|
||||
svc.ThumbnailStorage(
|
||||
storage.NewFileSystemStorage(
|
||||
tconf.FileSystemStorage,
|
||||
options.Logger,
|
||||
),
|
||||
),
|
||||
svc.CS3Source(imgsource.NewCS3Source(tconf, gatewaySelector, b)),
|
||||
svc.GatewaySelector(gatewaySelector),
|
||||
)
|
||||
thumbnail = decorators.NewInstrument(thumbnail, options.Metrics)
|
||||
thumbnail = decorators.NewLogging(thumbnail, options.Logger)
|
||||
thumbnail = decorators.NewTracing(thumbnail, options.TraceProvider)
|
||||
}
|
||||
|
||||
_ = thumbnailssvc.RegisterThumbnailServiceHandler(
|
||||
service.Server(),
|
||||
thumbnail,
|
||||
)
|
||||
|
||||
return service
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Namespace string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Flags []pflag.Flag
|
||||
TraceProvider trace.TracerProvider
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides a function to set the Namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to configure the trace provider
|
||||
func TraceProvider(traceProvider trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
if traceProvider != nil {
|
||||
o.TraceProvider = traceProvider
|
||||
} else {
|
||||
o.TraceProvider = noop.NewTracerProvider()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flags provides a function to set the flags option.
|
||||
func Flags(flags ...pflag.Flag) Option {
|
||||
return func(o *Options) {
|
||||
o.Flags = append(o.Flags, flags...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/qsfera/server/pkg/cors"
|
||||
qsferamiddleware "github.com/qsfera/server/pkg/middleware"
|
||||
"github.com/qsfera/server/pkg/service/http"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
svc "github.com/qsfera/server/services/thumbnails/pkg/service/http/v0"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
"go-micro.dev/v4"
|
||||
)
|
||||
|
||||
// Server initializes the http service and server.
|
||||
func Server(opts ...Option) (http.Service, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
service, err := http.NewService(
|
||||
http.TLSConfig(options.Config.HTTP.TLS),
|
||||
http.Logger(options.Logger),
|
||||
http.Name(options.Config.Service.Name),
|
||||
http.Version(version.GetString()),
|
||||
http.Namespace(options.Config.HTTP.Namespace),
|
||||
http.Address(options.Config.HTTP.Addr),
|
||||
http.Context(options.Context),
|
||||
http.TraceProvider(options.TraceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Error().
|
||||
Err(err).
|
||||
Msg("Error initializing http service")
|
||||
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
|
||||
}
|
||||
|
||||
handle := svc.NewService(
|
||||
svc.Logger(options.Logger),
|
||||
svc.Config(options.Config),
|
||||
svc.Middleware(
|
||||
middleware.RealIP,
|
||||
middleware.RequestID,
|
||||
qsferamiddleware.Cors(
|
||||
cors.Logger(options.Logger),
|
||||
cors.AllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins),
|
||||
cors.AllowedMethods(options.Config.HTTP.CORS.AllowedMethods),
|
||||
cors.AllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders),
|
||||
cors.AllowCredentials(options.Config.HTTP.CORS.AllowCredentials),
|
||||
),
|
||||
qsferamiddleware.Version(
|
||||
options.Config.Service.Name,
|
||||
version.GetString(),
|
||||
),
|
||||
qsferamiddleware.Logger(options.Logger),
|
||||
),
|
||||
svc.ThumbnailStorage(
|
||||
storage.NewFileSystemStorage(
|
||||
options.Config.Thumbnail.FileSystemStorage,
|
||||
options.Logger,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
{
|
||||
handle = svc.NewInstrument(handle, options.Metrics)
|
||||
handle = svc.NewLogging(handle, options.Logger)
|
||||
}
|
||||
|
||||
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
|
||||
return http.Service{}, err
|
||||
}
|
||||
|
||||
return service, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package decorators
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
)
|
||||
|
||||
// DecoratedService is an interface acting as facade, holding all the interfaces that this
|
||||
// thumbnails microservice is expecting to implement.
|
||||
// For now, only the thumbnailssvc.ThumbnailServiceHandler is present,
|
||||
// but a future configsvc.ConfigServiceHandler is expected to be added here
|
||||
//
|
||||
// This interface will also act as the base interface to implement
|
||||
// a decorator pattern.
|
||||
type DecoratedService interface {
|
||||
thumbnailssvc.ThumbnailServiceHandler
|
||||
}
|
||||
|
||||
// Decorator is the base type to implement the decorators. It will provide a basic implementation
|
||||
// by delegating to the decoratedService
|
||||
//
|
||||
// Expected implementations will be like:
|
||||
// ```
|
||||
//
|
||||
// type MyDecorator struct {
|
||||
// Decorator
|
||||
// myCustomOpts *opts
|
||||
// additionalSrv *srv
|
||||
// }
|
||||
//
|
||||
// func NewMyDecorator(next DecoratedService, customOpts *customOpts) DecoratedService {
|
||||
// .....
|
||||
// return MyDecorator{
|
||||
// Decorator: Decorator{next: next},
|
||||
// myCustomOpts: opts,
|
||||
// additionalSrv: srv,
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// ```
|
||||
type Decorator struct {
|
||||
next DecoratedService
|
||||
}
|
||||
|
||||
// GetThumbnail is the base implementation for the thumbnailssvc.GetThumbnail.
|
||||
// It will just delegate to the underlying decoratedService
|
||||
//
|
||||
// Your custom decorator is expected to overwrite this function,
|
||||
// but it MUST call the underlying decoratedService at some point
|
||||
// ```
|
||||
//
|
||||
// func (d MyDecorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
// doSomething()
|
||||
// err := d.next.GetThumbnail(ctx, req, resp)
|
||||
// doAnotherThing()
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// ```
|
||||
func (deco Decorator) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, resp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
return deco.next.GetThumbnail(ctx, req, resp)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package decorators
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// NewInstrument returns a service that instruments metrics.
|
||||
func NewInstrument(next DecoratedService, metrics *metrics.Metrics) DecoratedService {
|
||||
return instrument{
|
||||
Decorator: Decorator{next: next},
|
||||
metrics: metrics,
|
||||
}
|
||||
}
|
||||
|
||||
type instrument struct {
|
||||
Decorator
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// GetThumbnail implements the ThumbnailServiceHandler interface.
|
||||
func (i instrument) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
|
||||
us := v * 1000_000
|
||||
i.metrics.Latency.WithLabelValues().Observe(us)
|
||||
i.metrics.Duration.WithLabelValues().Observe(v)
|
||||
}))
|
||||
defer timer.ObserveDuration()
|
||||
|
||||
err := i.next.GetThumbnail(ctx, req, rsp)
|
||||
|
||||
if err != nil {
|
||||
i.metrics.Counter.WithLabelValues().Inc()
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package decorators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
merrors "go-micro.dev/v4/errors"
|
||||
)
|
||||
|
||||
// NewLogging returns a service that logs messages.
|
||||
func NewLogging(next DecoratedService, logger log.Logger) DecoratedService {
|
||||
return logging{
|
||||
Decorator: Decorator{next: next},
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
type logging struct {
|
||||
Decorator
|
||||
logger log.Logger
|
||||
}
|
||||
|
||||
// GetThumbnail implements the ThumbnailServiceHandler interface.
|
||||
func (l logging) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
start := time.Now()
|
||||
err := l.next.GetThumbnail(ctx, req, rsp)
|
||||
|
||||
logger := l.logger.With().
|
||||
Str("method", "Thumbnails.GetThumbnail").
|
||||
Dur("duration", time.Since(start)).
|
||||
Logger()
|
||||
|
||||
if err != nil {
|
||||
fromError := merrors.FromError(err)
|
||||
switch fromError.GetCode() {
|
||||
case http.StatusNotFound:
|
||||
logger.Debug().
|
||||
Str("error_detail", fromError.GetDetail()).
|
||||
Msg("no thumbnail found")
|
||||
default:
|
||||
logger.Warn().
|
||||
Err(err).
|
||||
Msg("Failed to execute")
|
||||
}
|
||||
} else {
|
||||
logger.Debug().
|
||||
Msg("")
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package decorators
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
)
|
||||
|
||||
// NewTracing returns a service that instruments traces.
|
||||
func NewTracing(next DecoratedService, tp trace.TracerProvider) DecoratedService {
|
||||
return tracing{
|
||||
Decorator: Decorator{next: next},
|
||||
tp: tp,
|
||||
}
|
||||
}
|
||||
|
||||
type tracing struct {
|
||||
Decorator
|
||||
tp trace.TracerProvider
|
||||
}
|
||||
|
||||
// GetThumbnail implements the ThumbnailServiceHandler interface.
|
||||
func (t tracing) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
var span trace.Span
|
||||
|
||||
if t.tp != nil {
|
||||
tracer := t.tp.Tracer("thumbnails")
|
||||
spanOpts := []trace.SpanStartOption{
|
||||
trace.WithSpanKind(trace.SpanKindServer),
|
||||
}
|
||||
ctx, span = tracer.Start(ctx, "Thumbnails.GetThumbnail", spanOpts...)
|
||||
defer span.End()
|
||||
|
||||
span.SetAttributes(
|
||||
attribute.KeyValue{Key: "filepath", Value: attribute.StringValue(req.GetFilepath())},
|
||||
attribute.KeyValue{Key: "thumbnail_type", Value: attribute.StringValue(req.GetThumbnailType().String())},
|
||||
attribute.KeyValue{Key: "width", Value: attribute.IntValue(int(req.GetWidth()))},
|
||||
attribute.KeyValue{Key: "height", Value: attribute.IntValue(int(req.GetHeight()))},
|
||||
)
|
||||
}
|
||||
|
||||
return t.next.GetThumbnail(ctx, req, rsp)
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
Config *config.Config
|
||||
Middleware []func(http.Handler) http.Handler
|
||||
ThumbnailStorage storage.Storage
|
||||
ImageSource imgsource.Source
|
||||
CS3Source imgsource.Source
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// ThumbnailStorage provides a function to set the thumbnail storage option.
|
||||
func ThumbnailStorage(val storage.Storage) Option {
|
||||
return func(o *Options) {
|
||||
o.ThumbnailStorage = val
|
||||
}
|
||||
}
|
||||
|
||||
// ThumbnailSource provides a function to set the image source option.
|
||||
func ThumbnailSource(val imgsource.Source) Option {
|
||||
return func(o *Options) {
|
||||
o.ImageSource = val
|
||||
}
|
||||
}
|
||||
|
||||
// CS3Source provides a function to set the CS3Source option
|
||||
func CS3Source(val imgsource.Source) Option {
|
||||
return func(o *Options) {
|
||||
o.CS3Source = val
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector adds a grpc client selector for the gateway service
|
||||
func GatewaySelector(val pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = val
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
merrors "go-micro.dev/v4/errors"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
thumbnailssvc "github.com/qsfera/server/protogen/gen/qsfera/services/thumbnails/v0"
|
||||
terrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/service/grpc/v0/decorators"
|
||||
tjwt "github.com/qsfera/server/services/thumbnails/pkg/service/jwt"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/imgsource"
|
||||
)
|
||||
|
||||
// NewService returns a service implementation for Service.
|
||||
func NewService(opts ...Option) decorators.DecoratedService {
|
||||
options := newOptions(opts...)
|
||||
logger := options.Logger
|
||||
resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions)
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("resolutions not configured correctly")
|
||||
}
|
||||
svc := Thumbnail{
|
||||
serviceID: options.Config.GRPC.Namespace + "." + options.Config.Service.Name,
|
||||
manager: thumbnail.NewSimpleManager(
|
||||
resolutions,
|
||||
options.ThumbnailStorage,
|
||||
logger,
|
||||
options.Config.Thumbnail.MaxInputWidth,
|
||||
options.Config.Thumbnail.MaxInputHeight,
|
||||
),
|
||||
webdavSource: options.ImageSource,
|
||||
cs3Source: options.CS3Source,
|
||||
logger: logger,
|
||||
selector: options.GatewaySelector,
|
||||
preprocessorOpts: PreprocessorOpts{
|
||||
TxtFontFileMap: options.Config.Thumbnail.FontMapFile,
|
||||
},
|
||||
dataEndpoint: options.Config.Thumbnail.DataEndpoint,
|
||||
transferSecret: options.Config.Thumbnail.TransferSecret,
|
||||
}
|
||||
|
||||
return svc
|
||||
}
|
||||
|
||||
// Thumbnail implements the GRPC handler.
|
||||
type Thumbnail struct {
|
||||
serviceID string
|
||||
dataEndpoint string
|
||||
transferSecret string
|
||||
manager thumbnail.Manager
|
||||
webdavSource imgsource.Source
|
||||
cs3Source imgsource.Source
|
||||
logger log.Logger
|
||||
selector pool.Selectable[gateway.GatewayAPIClient]
|
||||
preprocessorOpts PreprocessorOpts
|
||||
}
|
||||
|
||||
// PreprocessorOpts holds the options for the preprocessor
|
||||
type PreprocessorOpts struct {
|
||||
TxtFontFileMap string
|
||||
}
|
||||
|
||||
// GetThumbnail retrieves a thumbnail for an image
|
||||
func (g Thumbnail) GetThumbnail(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest, rsp *thumbnailssvc.GetThumbnailResponse) error {
|
||||
var err error
|
||||
var key string
|
||||
switch {
|
||||
case req.GetWebdavSource() != nil:
|
||||
key, err = g.handleWebdavSource(ctx, req)
|
||||
case req.GetCs3Source() != nil:
|
||||
key, err = g.handleCS3Source(ctx, req)
|
||||
default:
|
||||
g.logger.Error().Msg("no image source provided")
|
||||
return merrors.BadRequest(g.serviceID, "image source is missing")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
claims := tjwt.ThumbnailClaims{
|
||||
Key: key,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(1 * time.Minute)),
|
||||
},
|
||||
}
|
||||
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
transferToken, err := token.SignedString([]byte(g.transferSecret))
|
||||
if err != nil {
|
||||
g.logger.Error().
|
||||
Err(err).
|
||||
Msg("GetThumbnail: failed to sign token")
|
||||
return merrors.InternalServerError(g.serviceID, "couldn't finish request")
|
||||
}
|
||||
rsp.DataEndpoint = g.dataEndpoint
|
||||
rsp.TransferToken = transferToken
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g Thumbnail) checkThumbnail(req *thumbnailssvc.GetThumbnailRequest, sRes *provider.StatResponse) (string, thumbnail.Request, error) {
|
||||
tr := thumbnail.Request{}
|
||||
if !sRes.GetInfo().GetPermissionSet().GetInitiateFileDownload() {
|
||||
return "", tr, merrors.Forbidden(g.serviceID, "no download permission")
|
||||
}
|
||||
|
||||
tType := thumbnail.GetExtForMime(sRes.GetInfo().GetMimeType())
|
||||
if tType == "" {
|
||||
tType = req.GetThumbnailType().String()
|
||||
}
|
||||
tr, err := thumbnail.PrepareRequest(int(req.GetWidth()), int(req.GetHeight()), tType, sRes.GetInfo().GetChecksum().GetSum(), req.GetProcessor())
|
||||
if err != nil {
|
||||
return "", tr, merrors.BadRequest(g.serviceID, "%s", err.Error())
|
||||
}
|
||||
|
||||
if key, exists := g.manager.CheckThumbnail(tr); exists {
|
||||
return key, tr, nil
|
||||
}
|
||||
return "", tr, nil
|
||||
}
|
||||
|
||||
func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
|
||||
src := req.GetCs3Source()
|
||||
sRes, err := g.stat(src.GetPath(), src.GetAuthorization())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
key, tr, err := g.checkThumbnail(req, sRes)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", err
|
||||
case key != "":
|
||||
// we have matching thumbnail already, use that
|
||||
return key, nil
|
||||
}
|
||||
|
||||
ctx = imgsource.ContextSetAuthorization(ctx, src.GetAuthorization())
|
||||
r, err := g.cs3Source.Get(ctx, src.GetPath())
|
||||
switch {
|
||||
case errors.Is(err, terrors.ErrImageTooLarge):
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
case err != nil:
|
||||
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
|
||||
}
|
||||
|
||||
defer r.Close()
|
||||
ppOpts := map[string]any{
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
}
|
||||
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
|
||||
img, err := pp.Convert(r)
|
||||
if err != nil {
|
||||
g.logger.Error().Err(err).Msg("failed to convert image")
|
||||
}
|
||||
|
||||
if img == nil || err != nil {
|
||||
return "", merrors.NotFound(g.serviceID, "could not get image")
|
||||
}
|
||||
|
||||
key, err = g.manager.Generate(tr, img)
|
||||
if errors.Is(err, terrors.ErrImageTooLarge) {
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
}
|
||||
return key, err
|
||||
}
|
||||
|
||||
func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.GetThumbnailRequest) (string, error) {
|
||||
src := req.GetWebdavSource()
|
||||
imgURL, err := url.Parse(src.GetUrl())
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "source url is invalid")
|
||||
}
|
||||
|
||||
var auth, statPath string
|
||||
if src.GetIsPublicLink() {
|
||||
q := imgURL.Query()
|
||||
var rsp *gateway.AuthenticateResponse
|
||||
client, err := g.selector.Next()
|
||||
if err != nil {
|
||||
return "", merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
|
||||
}
|
||||
if q.Get("signature") != "" && q.Get("expiration") != "" {
|
||||
// Handle pre-signed public links
|
||||
sig := q.Get("signature")
|
||||
exp := q.Get("expiration")
|
||||
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
|
||||
Type: "publicshares",
|
||||
ClientId: src.GetPublicLinkToken(),
|
||||
ClientSecret: strings.Join([]string{"signature", sig, exp}, "|"),
|
||||
})
|
||||
} else {
|
||||
rsp, err = client.Authenticate(ctx, &gateway.AuthenticateRequest{
|
||||
Type: "publicshares",
|
||||
ClientId: src.GetPublicLinkToken(),
|
||||
// We pass an empty password because we expect non pre-signed public links
|
||||
// to not be password protected
|
||||
ClientSecret: "password|",
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", merrors.InternalServerError(g.serviceID, "could not authenticate: %s", err.Error())
|
||||
}
|
||||
auth = rsp.GetToken()
|
||||
statPath = path.Join("/public", src.GetPublicLinkToken(), req.GetFilepath())
|
||||
} else {
|
||||
auth = src.GetRevaAuthorization()
|
||||
statPath = req.GetFilepath()
|
||||
}
|
||||
sRes, err := g.stat(statPath, auth)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
key, tr, err := g.checkThumbnail(req, sRes)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", err
|
||||
case key != "":
|
||||
// we have matching thumbnail already, use that
|
||||
return key, nil
|
||||
}
|
||||
|
||||
if src.GetWebdavAuthorization() != "" {
|
||||
ctx = imgsource.ContextSetAuthorization(ctx, src.GetWebdavAuthorization())
|
||||
}
|
||||
|
||||
// add signature and expiration to webdav url
|
||||
signature, expiration := imgURL.Query().Get("signature"), imgURL.Query().Get("expiration")
|
||||
params := url.Values{}
|
||||
params.Add("signature", signature)
|
||||
params.Add("expiration", expiration)
|
||||
imgURL.RawQuery = params.Encode()
|
||||
|
||||
r, err := g.webdavSource.Get(ctx, imgURL.String())
|
||||
switch {
|
||||
case errors.Is(err, terrors.ErrImageTooLarge):
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
case err != nil:
|
||||
return "", merrors.InternalServerError(g.serviceID, "could not get image from source: %s", err.Error())
|
||||
}
|
||||
defer r.Close()
|
||||
ppOpts := map[string]any{
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
}
|
||||
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
|
||||
img, err := pp.Convert(r)
|
||||
if img == nil || err != nil {
|
||||
return "", merrors.NotFound(g.serviceID, "could not get image")
|
||||
}
|
||||
|
||||
key, err = g.manager.Generate(tr, img)
|
||||
if errors.Is(err, terrors.ErrImageTooLarge) {
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
}
|
||||
return key, err
|
||||
}
|
||||
|
||||
func (g Thumbnail) stat(path, auth string) (*provider.StatResponse, error) {
|
||||
ctx := metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
|
||||
|
||||
ref, err := storagespace.ParseReference(path)
|
||||
if err != nil {
|
||||
// If the path is not a spaces reference try to handle it like a plain
|
||||
// path reference.
|
||||
ref = provider.Reference{
|
||||
Path: path,
|
||||
}
|
||||
}
|
||||
|
||||
client, err := g.selector.Next()
|
||||
if err != nil {
|
||||
return nil, merrors.InternalServerError(g.serviceID, "could not select next gateway client: %s", err.Error())
|
||||
}
|
||||
req := &provider.StatRequest{Ref: &ref}
|
||||
rsp, err := client.Stat(ctx, req)
|
||||
if err != nil {
|
||||
g.logger.Error().Err(err).Str("path", path).Msg("could not stat file")
|
||||
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", err.Error())
|
||||
}
|
||||
|
||||
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
switch rsp.GetStatus().GetCode() {
|
||||
case rpc.Code_CODE_NOT_FOUND:
|
||||
return nil, merrors.NotFound(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
|
||||
default:
|
||||
g.logger.Error().Str("status_message", rsp.GetStatus().GetMessage()).Str("path", path).Msg("could not stat file")
|
||||
return nil, merrors.InternalServerError(g.serviceID, "could not stat file: %s", rsp.GetStatus().GetMessage())
|
||||
}
|
||||
}
|
||||
if rsp.GetInfo().GetType() != provider.ResourceType_RESOURCE_TYPE_FILE {
|
||||
return nil, merrors.BadRequest(g.serviceID, "Unsupported file type")
|
||||
}
|
||||
if utils.ReadPlainFromOpaque(rsp.GetInfo().GetOpaque(), "status") == "processing" {
|
||||
return nil, &merrors.Error{
|
||||
Id: g.serviceID,
|
||||
Code: http.StatusTooEarly,
|
||||
Detail: "File Processing",
|
||||
Status: http.StatusText(http.StatusTooEarly),
|
||||
}
|
||||
}
|
||||
if rsp.GetInfo().GetChecksum().GetSum() == "" {
|
||||
g.logger.Error().Msg("resource info is missing checksum")
|
||||
return nil, merrors.NotFound(g.serviceID, "resource info is missing a checksum")
|
||||
}
|
||||
if !thumbnail.IsMimeTypeSupported(rsp.GetInfo().GetMimeType()) {
|
||||
return nil, merrors.NotFound(g.serviceID, "Unsupported file type")
|
||||
}
|
||||
return rsp, nil
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/metrics"
|
||||
)
|
||||
|
||||
// NewInstrument returns a service that instruments metrics.
|
||||
func NewInstrument(next Service, metrics *metrics.Metrics) Service {
|
||||
return instrument{
|
||||
next: next,
|
||||
metrics: metrics,
|
||||
}
|
||||
}
|
||||
|
||||
type instrument struct {
|
||||
next Service
|
||||
metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// ServeHTTP implements the Service interface.
|
||||
func (i instrument) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
i.next.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// GetThumbnail implements the Service interface.
|
||||
func (i instrument) GetThumbnail(w http.ResponseWriter, r *http.Request) {
|
||||
i.next.GetThumbnail(w, r)
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
)
|
||||
|
||||
// NewLogging returns a service that logs messages.
|
||||
func NewLogging(next Service, logger log.Logger) Service {
|
||||
return logging{
|
||||
next: next,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
type logging struct {
|
||||
next Service
|
||||
logger log.Logger
|
||||
}
|
||||
|
||||
// ServeHTTP implements the Service interface.
|
||||
func (l logging) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
l.next.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// GetThumbnail implements the Service interface.
|
||||
func (l logging) GetThumbnail(w http.ResponseWriter, r *http.Request) {
|
||||
l.next.GetThumbnail(w, r)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
Config *config.Config
|
||||
Middleware []func(http.Handler) http.Handler
|
||||
ThumbnailStorage storage.Storage
|
||||
TraceProvider trace.TracerProvider
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// Middleware provides a function to set the middleware option.
|
||||
func Middleware(val ...func(http.Handler) http.Handler) Option {
|
||||
return func(o *Options) {
|
||||
o.Middleware = val
|
||||
}
|
||||
}
|
||||
|
||||
// ThumbnailStorage provides a function to set the ThumbnailStorage option.
|
||||
func ThumbnailStorage(storage storage.Storage) Option {
|
||||
return func(o *Options) {
|
||||
o.ThumbnailStorage = storage
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/riandyrn/otelchi"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/pkg/tracing"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
tjwt "github.com/qsfera/server/services/thumbnails/pkg/service/jwt"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
|
||||
const (
|
||||
keyContextKey contextKey = "key"
|
||||
)
|
||||
|
||||
// Service defines the service handlers.
|
||||
type Service interface {
|
||||
ServeHTTP(w http.ResponseWriter, r *http.Request)
|
||||
GetThumbnail(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// NewService returns a service implementation for Service.
|
||||
func NewService(opts ...Option) Service {
|
||||
options := newOptions(opts...)
|
||||
|
||||
m := chi.NewMux()
|
||||
m.Use(options.Middleware...)
|
||||
|
||||
m.Use(
|
||||
otelchi.Middleware(
|
||||
"thumbnails",
|
||||
otelchi.WithChiRoutes(m),
|
||||
otelchi.WithTracerProvider(options.TraceProvider),
|
||||
otelchi.WithPropagators(tracing.GetPropagator()),
|
||||
),
|
||||
)
|
||||
|
||||
logger := options.Logger
|
||||
resolutions, err := thumbnail.ParseResolutions(options.Config.Thumbnail.Resolutions)
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err).Msg("resolutions not configured correctly")
|
||||
}
|
||||
svc := Thumbnails{
|
||||
config: options.Config,
|
||||
mux: m,
|
||||
logger: options.Logger,
|
||||
manager: thumbnail.NewSimpleManager(
|
||||
resolutions,
|
||||
options.ThumbnailStorage,
|
||||
logger,
|
||||
options.Config.Thumbnail.MaxInputWidth,
|
||||
options.Config.Thumbnail.MaxInputHeight,
|
||||
),
|
||||
}
|
||||
|
||||
m.Route(options.Config.HTTP.Root, func(r chi.Router) {
|
||||
r.Use(svc.TransferTokenValidator)
|
||||
r.Get("/data", svc.GetThumbnail)
|
||||
})
|
||||
|
||||
_ = chi.Walk(m, func(method string, route string, _ http.Handler, middlewares ...func(http.Handler) http.Handler) error {
|
||||
options.Logger.Debug().Str("method", method).Str("route", route).Int("middlewares", len(middlewares)).Msg("serving endpoint")
|
||||
return nil
|
||||
})
|
||||
|
||||
return svc
|
||||
}
|
||||
|
||||
// Thumbnails implements the business logic for Service.
|
||||
type Thumbnails struct {
|
||||
config *config.Config
|
||||
logger log.Logger
|
||||
mux *chi.Mux
|
||||
manager thumbnail.Manager
|
||||
}
|
||||
|
||||
// ServeHTTP implements the Service interface.
|
||||
func (s Thumbnails) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// GetThumbnail implements the Service interface.
|
||||
func (s Thumbnails) GetThumbnail(w http.ResponseWriter, r *http.Request) {
|
||||
logger := s.logger.SubloggerWithRequestID(r.Context())
|
||||
key := r.Context().Value(keyContextKey).(string)
|
||||
|
||||
thumbnailBytes, err := s.manager.GetThumbnail(key)
|
||||
if err != nil {
|
||||
logger.Debug().
|
||||
Err(err).
|
||||
Str("key", key).
|
||||
Msg("could not get the thumbnail")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(thumbnailBytes)))
|
||||
if _, err = w.Write(thumbnailBytes); err != nil {
|
||||
logger.Error().
|
||||
Err(err).
|
||||
Str("key", key).
|
||||
Msg("could not write the thumbnail response")
|
||||
}
|
||||
}
|
||||
|
||||
// TransferTokenValidator validates a transfer token
|
||||
func (s Thumbnails) TransferTokenValidator(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tokenString := r.Header.Get("Transfer-Token")
|
||||
if tokenString == "" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
logger := s.logger.SubloggerWithRequestID(r.Context())
|
||||
token, err := jwt.ParseWithClaims(tokenString, &tjwt.ThumbnailClaims{}, func(token *jwt.Token) (any, error) {
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
|
||||
}
|
||||
return []byte(s.config.Thumbnail.TransferSecret), nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Debug().
|
||||
Err(err).
|
||||
Str("transfer-token", tokenString).
|
||||
Msg("failed to parse transfer token")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
if claims, ok := token.Claims.(*tjwt.ThumbnailClaims); ok && token.Valid {
|
||||
ctx := context.WithValue(r.Context(), keyContextKey, claims.Key)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
return
|
||||
}
|
||||
logger.Debug().Msg("invalid transfer token")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package jwt
|
||||
|
||||
import "github.com/golang-jwt/jwt/v5"
|
||||
|
||||
// ThumbnailClaims defines the claims for thumb-nailing
|
||||
type ThumbnailClaims struct {
|
||||
jwt.RegisteredClaims
|
||||
Key string `json:"key"`
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image/gif"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
typePng = "png"
|
||||
typeJpg = "jpg"
|
||||
typeJpeg = "jpeg"
|
||||
typeGif = "gif"
|
||||
typeGgs = "ggs"
|
||||
typeGgp = "ggp"
|
||||
typeWebp = "webp"
|
||||
)
|
||||
|
||||
// Encoder encodes the thumbnail to a specific format.
|
||||
type Encoder interface {
|
||||
// Encode encodes the image to a format.
|
||||
Encode(w io.Writer, img any) error
|
||||
// Types returns the formats suffixes.
|
||||
Types() []string
|
||||
// MimeType returns the mimetype used by the encoder.
|
||||
MimeType() string
|
||||
}
|
||||
|
||||
// GifEncoder encodes to gif
|
||||
type GifEncoder struct{}
|
||||
|
||||
// Encode encodes the image to a gif format
|
||||
func (e GifEncoder) Encode(w io.Writer, img any) error {
|
||||
g, ok := img.(*gif.GIF)
|
||||
if !ok {
|
||||
return errors.ErrInvalidType
|
||||
}
|
||||
return gif.EncodeAll(w, g)
|
||||
}
|
||||
|
||||
// Types returns the supported types of the GifEncoder
|
||||
func (e GifEncoder) Types() []string {
|
||||
return []string{typeGif}
|
||||
}
|
||||
|
||||
// MimeType returns the mimetype used by the encoder.
|
||||
func (e GifEncoder) MimeType() string {
|
||||
return "image/gif"
|
||||
}
|
||||
|
||||
// EncoderForType returns the encoder for a given file type
|
||||
// or nil if the type is not supported.
|
||||
func EncoderForType(fileType string) (Encoder, error) {
|
||||
switch strings.ToLower(fileType) {
|
||||
case typePng, typeGgs, typeGgp:
|
||||
return PngEncoder{}, nil
|
||||
case typeJpg, typeJpeg, typeWebp:
|
||||
return JpegEncoder{}, nil
|
||||
case typeGif:
|
||||
return GifEncoder{}, nil
|
||||
default:
|
||||
return nil, errors.ErrNoEncoderForType
|
||||
}
|
||||
}
|
||||
|
||||
// GetExtForMime return the supported extension by mime
|
||||
func GetExtForMime(fileType string) string {
|
||||
ext := strings.TrimPrefix(strings.TrimSpace(strings.ToLower(fileType)), "image/")
|
||||
switch ext {
|
||||
case typeJpg, typeJpeg, typePng, typeGif, typeWebp:
|
||||
return ext
|
||||
case "application/vnd.geogebra.slides":
|
||||
return typeGgs
|
||||
case "application/vnd.geogebra.pinboard":
|
||||
return typeGgp
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// PngEncoder encodes to png
|
||||
type PngEncoder struct{}
|
||||
|
||||
// Encode encodes to png format
|
||||
func (e PngEncoder) Encode(w io.Writer, img any) error {
|
||||
m, ok := img.(image.Image)
|
||||
if !ok {
|
||||
return errors.ErrInvalidType
|
||||
}
|
||||
return png.Encode(w, m)
|
||||
}
|
||||
|
||||
// Types returns the png suffix
|
||||
func (e PngEncoder) Types() []string {
|
||||
return []string{typePng}
|
||||
}
|
||||
|
||||
// MimeType returns the mimetype for png files.
|
||||
func (e PngEncoder) MimeType() string {
|
||||
return "image/png"
|
||||
}
|
||||
|
||||
// JpegEncoder encodes to jpg
|
||||
type JpegEncoder struct{}
|
||||
|
||||
// Encode encodes to jpg
|
||||
func (e JpegEncoder) Encode(w io.Writer, img any) error {
|
||||
m, ok := img.(image.Image)
|
||||
if !ok {
|
||||
return errors.ErrInvalidType
|
||||
}
|
||||
return jpeg.Encode(w, m, nil)
|
||||
}
|
||||
|
||||
// Types returns the jpg suffixes.
|
||||
func (e JpegEncoder) Types() []string {
|
||||
return []string{typeJpeg, typeJpg}
|
||||
}
|
||||
|
||||
// MimeType returns the mimetype for jpg files.
|
||||
func (e JpegEncoder) MimeType() string {
|
||||
return "image/jpeg"
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package thumbnail
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEncoderForType(t *testing.T) {
|
||||
table := map[string]Encoder{
|
||||
"jpg": JpegEncoder{},
|
||||
"JPG": JpegEncoder{},
|
||||
"jpeg": JpegEncoder{},
|
||||
"JPEG": JpegEncoder{},
|
||||
"png": PngEncoder{},
|
||||
"PNG": PngEncoder{},
|
||||
"invalid": nil,
|
||||
}
|
||||
|
||||
for k, v := range table {
|
||||
e, _ := EncoderForType(k)
|
||||
if e != v {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// PngEncoder encodes to png
|
||||
type PngEncoder struct{}
|
||||
|
||||
// Encode encodes to png format
|
||||
func (e PngEncoder) Encode(w io.Writer, img interface{}) error {
|
||||
m, ok := img.(*vips.ImageRef)
|
||||
if !ok {
|
||||
return errors.ErrInvalidType
|
||||
}
|
||||
|
||||
buf, _, err := m.ExportPng(vips.NewPngExportParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// Types returns the png suffix
|
||||
func (e PngEncoder) Types() []string {
|
||||
return []string{typePng}
|
||||
}
|
||||
|
||||
// MimeType returns the mimetype for png files.
|
||||
func (e PngEncoder) MimeType() string {
|
||||
return "image/png"
|
||||
}
|
||||
|
||||
// JpegEncoder encodes to jpg
|
||||
type JpegEncoder struct{}
|
||||
|
||||
// Encode encodes to jpg
|
||||
func (e JpegEncoder) Encode(w io.Writer, img interface{}) error {
|
||||
m, ok := img.(*vips.ImageRef)
|
||||
if !ok {
|
||||
return errors.ErrInvalidType
|
||||
}
|
||||
|
||||
buf, _, err := m.ExportJpeg(vips.NewJpegExportParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// Types returns the jpg suffixes.
|
||||
func (e JpegEncoder) Types() []string {
|
||||
return []string{typeJpeg, typeJpg}
|
||||
}
|
||||
|
||||
// MimeType returns the mimetype for jpg files.
|
||||
func (e JpegEncoder) MimeType() string {
|
||||
return "image/jpeg"
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/gif"
|
||||
"strings"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// Generator generates a web friendly file version.
|
||||
type Generator interface {
|
||||
Generate(size image.Rectangle, img any) (any, error)
|
||||
Dimensions(img any) (image.Rectangle, error)
|
||||
ProcessorID() string
|
||||
}
|
||||
|
||||
// GifGenerator is used to create a web friendly version of the provided gif image.
|
||||
type GifGenerator struct {
|
||||
processor Processor
|
||||
}
|
||||
|
||||
func NewGifGenerator(filetype, process string) (GifGenerator, error) {
|
||||
processor, err := ProcessorFor(process, filetype)
|
||||
if err != nil {
|
||||
return GifGenerator{}, err
|
||||
}
|
||||
return GifGenerator{processor: processor}, nil
|
||||
|
||||
}
|
||||
|
||||
// ProcessorID returns the processor identification.
|
||||
func (g GifGenerator) ProcessorID() string {
|
||||
return g.processor.ID()
|
||||
}
|
||||
|
||||
// Generate generates a alternative gif version.
|
||||
func (g GifGenerator) Generate(size image.Rectangle, img any) (any, error) {
|
||||
// Code inspired by https://github.com/willnorris/gifresize/blob/db93a7e1dcb1c279f7eeb99cc6d90b9e2e23e871/gifresize.go
|
||||
|
||||
m, ok := img.(*gif.GIF)
|
||||
if !ok {
|
||||
return nil, errors.ErrInvalidType
|
||||
}
|
||||
// Create a new RGBA image to hold the incremental frames.
|
||||
srcX, srcY := m.Config.Width, m.Config.Height
|
||||
b := image.Rect(0, 0, srcX, srcY)
|
||||
tmp := image.NewRGBA(b)
|
||||
|
||||
for i, frame := range m.Image {
|
||||
bounds := frame.Bounds()
|
||||
prev := tmp
|
||||
draw.Draw(tmp, bounds, frame, bounds.Min, draw.Over)
|
||||
processed := g.processor.Process(tmp, size.Dx(), size.Dy(), imaging.Lanczos)
|
||||
m.Image[i] = g.imageToPaletted(processed, frame.Palette)
|
||||
|
||||
switch m.Disposal[i] {
|
||||
case gif.DisposalBackground:
|
||||
tmp = image.NewRGBA(b)
|
||||
case gif.DisposalPrevious:
|
||||
tmp = prev
|
||||
}
|
||||
}
|
||||
m.Config.Width = size.Dx()
|
||||
m.Config.Height = size.Dy()
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (g GifGenerator) Dimensions(img any) (image.Rectangle, error) {
|
||||
m, ok := img.(*gif.GIF)
|
||||
if !ok {
|
||||
return image.Rectangle{}, errors.ErrInvalidType
|
||||
}
|
||||
return m.Image[0].Bounds(), nil
|
||||
}
|
||||
|
||||
func (g GifGenerator) imageToPaletted(img image.Image, p color.Palette) *image.Paletted {
|
||||
b := img.Bounds()
|
||||
pm := image.NewPaletted(b, p)
|
||||
draw.FloydSteinberg.Draw(pm, b, img, image.Point{})
|
||||
return pm
|
||||
}
|
||||
|
||||
// GeneratorFor returns the generator for a given file type
|
||||
// or nil if the type is not supported.
|
||||
func GeneratorFor(fileType, processorID string) (Generator, error) {
|
||||
switch strings.ToLower(fileType) {
|
||||
case typePng, typeJpg, typeJpeg, typeGgs, typeGgp, typeWebp:
|
||||
return NewSimpleGenerator(fileType, processorID)
|
||||
case typeGif:
|
||||
return NewGifGenerator(fileType, processorID)
|
||||
default:
|
||||
return nil, errors.ErrNoGeneratorForType
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// SimpleGenerator is the default image generator and is used for all image types expect gif.
|
||||
type SimpleGenerator struct {
|
||||
processor Processor
|
||||
}
|
||||
|
||||
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
|
||||
processor, err := ProcessorFor(process, filetype)
|
||||
if err != nil {
|
||||
return SimpleGenerator{}, err
|
||||
}
|
||||
return SimpleGenerator{processor: processor}, nil
|
||||
}
|
||||
|
||||
// ProcessorID returns the processor identification.
|
||||
func (g SimpleGenerator) ProcessorID() string {
|
||||
return g.processor.ID()
|
||||
}
|
||||
|
||||
// Generate generates a alternative image version.
|
||||
func (g SimpleGenerator) Generate(size image.Rectangle, img any) (any, error) {
|
||||
m, ok := img.(image.Image)
|
||||
if !ok {
|
||||
return nil, errors.ErrInvalidType
|
||||
}
|
||||
|
||||
return g.processor.Process(m, size.Dx(), size.Dy(), imaging.Lanczos), nil
|
||||
}
|
||||
|
||||
func (g SimpleGenerator) Dimensions(img any) (image.Rectangle, error) {
|
||||
m, ok := img.(image.Image)
|
||||
if !ok {
|
||||
return image.Rectangle{}, errors.ErrInvalidType
|
||||
}
|
||||
return m.Bounds(), nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
//go:build enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"strings"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"golang.org/x/image/bmp"
|
||||
)
|
||||
|
||||
// SimpleGenerator is the default image generator and is used for all image types expect gif.
|
||||
type SimpleGenerator struct {
|
||||
crop vips.Interesting
|
||||
size vips.Size
|
||||
process string
|
||||
}
|
||||
|
||||
func NewSimpleGenerator(filetype, process string) (SimpleGenerator, error) {
|
||||
switch strings.ToLower(process) {
|
||||
case "thumbnail":
|
||||
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
|
||||
case "fit":
|
||||
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeBoth}, nil
|
||||
case "resize":
|
||||
return SimpleGenerator{crop: vips.InterestingNone, process: process, size: vips.SizeForce}, nil
|
||||
default:
|
||||
return SimpleGenerator{crop: vips.InterestingAttention, process: process, size: vips.SizeBoth}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessorID returns the processor identification.
|
||||
func (g SimpleGenerator) ProcessorID() string {
|
||||
return g.process
|
||||
}
|
||||
|
||||
// Generate generates a alternative image version.
|
||||
func (g SimpleGenerator) Generate(size image.Rectangle, img interface{}) (interface{}, error) {
|
||||
var m *vips.ImageRef
|
||||
var err error
|
||||
switch img.(type) {
|
||||
case *image.RGBA:
|
||||
// This comes from the txt preprocessor
|
||||
var buf bytes.Buffer
|
||||
if err = bmp.Encode(&buf, img.(*image.RGBA)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err = vips.NewImageFromReader(&buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
case *vips.ImageRef:
|
||||
m = img.(*vips.ImageRef)
|
||||
default:
|
||||
return nil, errors.ErrInvalidType
|
||||
}
|
||||
|
||||
if err := m.ThumbnailWithSize(size.Dx(), 0, g.crop, g.size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := m.RemoveMetadata(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (g SimpleGenerator) Dimensions(img interface{}) (image.Rectangle, error) {
|
||||
switch img.(type) {
|
||||
case *image.RGBA:
|
||||
m := img.(*image.RGBA)
|
||||
return m.Bounds(), nil
|
||||
case *vips.ImageRef:
|
||||
m := img.(*vips.ImageRef)
|
||||
return image.Rect(0, 0, m.Width(), m.Height()), nil
|
||||
default:
|
||||
return image.Rectangle{}, errors.ErrInvalidType
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package imgsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
const (
|
||||
// TokenTransportHeader holds the header key for the reva transfer token
|
||||
// "github.com/opencloud-eu/reva/v2/internal/http/services/datagateway" is internal so we redeclare it here
|
||||
TokenTransportHeader = "X-Reva-Transfer"
|
||||
)
|
||||
|
||||
// CS3 implements a CS3 image source
|
||||
type CS3 struct {
|
||||
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
insecure bool
|
||||
maxImageFileSize uint64
|
||||
}
|
||||
|
||||
// NewCS3Source configures a new CS3 image source
|
||||
func NewCS3Source(cfg config.Thumbnail, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], b bytesize.ByteSize) CS3 {
|
||||
return CS3{
|
||||
gatewaySelector: gatewaySelector,
|
||||
insecure: cfg.CS3AllowInsecure,
|
||||
maxImageFileSize: b.Bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
// Get downloads the file from a cs3 service
|
||||
// The caller MUST make sure to close the returned ReadCloser
|
||||
func (s CS3) Get(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||
auth, ok := ContextGetAuthorization(ctx)
|
||||
if !ok {
|
||||
return nil, errors.ErrCS3AuthorizationMissing
|
||||
}
|
||||
ref, err := storagespace.ParseReference(path)
|
||||
if err != nil {
|
||||
// If the path is not a spaces reference try to handle it like a plain
|
||||
// path reference.
|
||||
ref = provider.Reference{
|
||||
Path: path,
|
||||
}
|
||||
}
|
||||
|
||||
ctx = metadata.AppendToOutgoingContext(context.Background(), revactx.TokenHeader, auth)
|
||||
err = s.checkImageFileSize(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gwc, err := s.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rsp, err := gwc.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &ref})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if rsp.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
return nil, fmt.Errorf("could not load image: %s", rsp.GetStatus().GetMessage())
|
||||
}
|
||||
var ep, tk string
|
||||
for _, p := range rsp.GetProtocols() {
|
||||
if p.GetProtocol() == "spaces" {
|
||||
ep, tk = p.GetDownloadEndpoint(), p.GetToken()
|
||||
break
|
||||
}
|
||||
}
|
||||
if (ep == "" || tk == "") && len(rsp.GetProtocols()) > 0 {
|
||||
ep, tk = rsp.GetProtocols()[0].GetDownloadEndpoint(), rsp.GetProtocols()[0].GetToken()
|
||||
}
|
||||
|
||||
httpReq, err := rhttp.NewRequest(ctx, "GET", ep, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpReq.Header.Set(revactx.TokenHeader, auth)
|
||||
httpReq.Header.Set(TokenTransportHeader, tk)
|
||||
|
||||
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: s.insecure, //nolint:gosec
|
||||
}
|
||||
client := &http.Client{}
|
||||
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", path, resp.StatusCode)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (s CS3) checkImageFileSize(ctx context.Context, ref provider.Reference) error {
|
||||
gwc, err := s.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stat, err := gwc.Stat(ctx, &provider.StatRequest{Ref: &ref})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stat.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
return fmt.Errorf("could not stat image: %s", stat.GetStatus().GetMessage())
|
||||
}
|
||||
if stat.GetInfo().GetSize() > s.maxImageFileSize {
|
||||
return errors.ErrImageTooLarge
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package imgsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
type key int
|
||||
|
||||
const (
|
||||
auth key = iota
|
||||
)
|
||||
|
||||
// Source defines the interface for image sources
|
||||
type Source interface {
|
||||
Get(ctx context.Context, path string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// ContextSetAuthorization puts the authorization in the context.
|
||||
func ContextSetAuthorization(parent context.Context, authorization string) context.Context {
|
||||
return context.WithValue(parent, auth, authorization)
|
||||
}
|
||||
|
||||
// ContextGetAuthorization gets the authorization from the context.
|
||||
func ContextGetAuthorization(ctx context.Context) (string, bool) {
|
||||
val := ctx.Value(auth)
|
||||
if val == nil {
|
||||
return "", false
|
||||
}
|
||||
return val.(string), true
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package imgsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
_ "image/gif" // Import the gif package so that image.Decode can understand gifs
|
||||
_ "image/jpeg" // Import the jpeg package so that image.Decode can understand jpegs
|
||||
_ "image/png" // Import the png package so that image.Decode can understand pngs
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// NewWebDavSource creates a new webdav instance.
|
||||
func NewWebDavSource(cfg config.Thumbnail, b bytesize.ByteSize) WebDav {
|
||||
return WebDav{
|
||||
insecure: cfg.WebdavAllowInsecure,
|
||||
maxImageFileSize: b.Bytes(),
|
||||
}
|
||||
}
|
||||
|
||||
// WebDav implements the Source interface for webdav services
|
||||
type WebDav struct {
|
||||
insecure bool
|
||||
maxImageFileSize uint64
|
||||
}
|
||||
|
||||
// Get downloads the file from a webdav service
|
||||
// The caller MUST make sure to close the returned ReadCloser
|
||||
func (s WebDav) Get(ctx context.Context, url string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
|
||||
}
|
||||
|
||||
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: s.insecure, //nolint:gosec
|
||||
}
|
||||
|
||||
if auth, ok := ContextGetAuthorization(ctx); ok {
|
||||
req.Header.Add("Authorization", auth)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, `could not get the image "%s"`, url)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("could not get the image \"%s\". Request returned with statuscode %d ", url, resp.StatusCode)
|
||||
}
|
||||
|
||||
contentLength := resp.Header.Get("Content-Length")
|
||||
if contentLength == "" {
|
||||
// no size information - let's assume it is too big
|
||||
return nil, thumbnailerErrors.ErrImageTooLarge
|
||||
}
|
||||
c, err := strconv.ParseUint(contentLength, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, `could not parse content length of webdav response "%s"`, url)
|
||||
}
|
||||
if c > s.maxImageFileSize {
|
||||
return nil, thumbnailerErrors.ErrImageTooLarge
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
var (
|
||||
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
|
||||
SupportedMimeTypes = map[string]struct{}{
|
||||
"image/png": {},
|
||||
"image/jpg": {},
|
||||
"image/jpeg": {},
|
||||
"image/gif": {},
|
||||
"image/bmp": {},
|
||||
"image/x-ms-bmp": {},
|
||||
"image/tiff": {},
|
||||
"text/plain": {},
|
||||
"audio/flac": {},
|
||||
"audio/mpeg": {},
|
||||
"audio/ogg": {},
|
||||
"application/vnd.geogebra.slides": {},
|
||||
"application/vnd.geogebra.pinboard": {},
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
//go:build enable_vips
|
||||
|
||||
package thumbnail
|
||||
|
||||
import "github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
func init() {
|
||||
// temporary remove TIFF and JP2K from go-vips' list of supported
|
||||
// imagetypes
|
||||
delete(vips.ImageTypes, vips.ImageTypeTIFF)
|
||||
delete(vips.ImageTypes, vips.ImageTypeJP2K)
|
||||
}
|
||||
|
||||
var (
|
||||
// SupportedMimeTypes contains an all mimetypes which are supported by the thumbnailer.
|
||||
SupportedMimeTypes = map[string]struct{}{
|
||||
"image/png": {},
|
||||
"image/jpg": {},
|
||||
"image/jpeg": {},
|
||||
"image/gif": {},
|
||||
"image/bmp": {},
|
||||
"image/x-ms-bmp": {},
|
||||
"text/plain": {},
|
||||
"audio/flac": {},
|
||||
"audio/mpeg": {},
|
||||
"audio/ogg": {},
|
||||
"application/vnd.geogebra.slides": {},
|
||||
"application/vnd.geogebra.pinboard": {},
|
||||
"image/webp": {},
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
"strings"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
)
|
||||
|
||||
// Processor processes the thumbnail by applying different transformations to it.
|
||||
type Processor interface {
|
||||
ID() string
|
||||
Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
|
||||
}
|
||||
|
||||
// DefinableProcessor is the simplest processor, it holds a replaceable image converter function.
|
||||
type DefinableProcessor struct {
|
||||
Slug string
|
||||
Converter func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image
|
||||
}
|
||||
|
||||
// ID returns the processor identification.
|
||||
func (p DefinableProcessor) ID() string { return p.Slug }
|
||||
|
||||
// Process transforms the given image.
|
||||
func (p DefinableProcessor) Process(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
|
||||
return p.Converter(img, width, height, filter)
|
||||
}
|
||||
|
||||
// ProcessorFor returns a matching Processor
|
||||
func ProcessorFor(id, fileType string) (DefinableProcessor, error) {
|
||||
switch strings.ToLower(id) {
|
||||
case "fit":
|
||||
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Fit}, nil
|
||||
case "resize":
|
||||
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Resize}, nil
|
||||
case "fill":
|
||||
return DefinableProcessor{Slug: strings.ToLower(id), Converter: func(img image.Image, width, height int, filter imaging.ResampleFilter) image.Image {
|
||||
return imaging.Fill(img, width, height, imaging.Center, filter)
|
||||
}}, nil
|
||||
case "thumbnail":
|
||||
return DefinableProcessor{Slug: strings.ToLower(id), Converter: imaging.Thumbnail}, nil
|
||||
default:
|
||||
switch strings.ToLower(fileType) {
|
||||
case typeGif:
|
||||
return DefinableProcessor{Converter: imaging.Resize}, nil
|
||||
default:
|
||||
return DefinableProcessor{Converter: imaging.Thumbnail}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package thumbnail_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail"
|
||||
)
|
||||
|
||||
func TestProcessorFor(t *testing.T) {
|
||||
tests := []struct {
|
||||
id string
|
||||
fileType string
|
||||
wantP thumbnail.Processor
|
||||
wantE error
|
||||
}{
|
||||
{
|
||||
id: "fit",
|
||||
fileType: "",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "fit", Converter: imaging.Fit},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "fit",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "FIT",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "fit"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "resize",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "RESIZE",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "resize"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "fill",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "FILL",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "fill"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "thumbnail",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "THUMBNAIL",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{Slug: "thumbnail"},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "",
|
||||
fileType: "jpg",
|
||||
wantP: thumbnail.DefinableProcessor{},
|
||||
wantE: nil,
|
||||
},
|
||||
{
|
||||
id: "",
|
||||
fileType: "gif",
|
||||
wantP: thumbnail.DefinableProcessor{},
|
||||
wantE: nil,
|
||||
},
|
||||
}
|
||||
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
p, e := thumbnail.ProcessorFor(tt.id, tt.fileType)
|
||||
assert.Equal(p.ID(), tt.wantP.ID())
|
||||
assert.Equal(e, tt.wantE)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
_resolutionSeparator = "x"
|
||||
)
|
||||
|
||||
// ParseResolution returns an image.Rectangle representing the resolution given as a string
|
||||
func ParseResolution(s string) (image.Rectangle, error) {
|
||||
parts := strings.Split(s, _resolutionSeparator)
|
||||
if len(parts) != 2 {
|
||||
return image.Rectangle{}, fmt.Errorf("failed to parse resolution: %s. Expected format <width>x<height>", s)
|
||||
}
|
||||
width, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return image.Rectangle{}, fmt.Errorf("width: %s has an invalid value. Expected an integer", parts[0])
|
||||
}
|
||||
height, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
return image.Rectangle{}, fmt.Errorf("height: %s has an invalid value. Expected an integer", parts[1])
|
||||
}
|
||||
return image.Rect(0, 0, width, height), nil
|
||||
}
|
||||
|
||||
// Resolutions is a list of image.Rectangle representing resolutions.
|
||||
type Resolutions []image.Rectangle
|
||||
|
||||
// ParseResolutions creates an instance of Resolutions from resolution strings.
|
||||
func ParseResolutions(strs []string) (Resolutions, error) {
|
||||
rs := make(Resolutions, 0, len(strs))
|
||||
for _, s := range strs {
|
||||
r, err := ParseResolution(s)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not parse resolutions")
|
||||
}
|
||||
rs = append(rs, r)
|
||||
}
|
||||
return rs, nil
|
||||
}
|
||||
|
||||
// ClosestMatch returns the resolution which is closest to the provided resolution.
|
||||
// If there is no exact match the resolution will be the next higher one.
|
||||
// If the given resolution is bigger than all available resolutions the biggest available one is used.
|
||||
func (rs Resolutions) ClosestMatch(requested image.Rectangle, sourceSize image.Rectangle) image.Rectangle {
|
||||
isLandscape := sourceSize.Dx() > sourceSize.Dy()
|
||||
sourceLen := dimensionLength(sourceSize, isLandscape)
|
||||
requestedLen := dimensionLength(requested, isLandscape)
|
||||
isSourceSmaller := sourceLen < requestedLen
|
||||
|
||||
// We don't want to scale images up.
|
||||
if isSourceSmaller {
|
||||
return sourceSize
|
||||
}
|
||||
|
||||
if len(rs) == 0 {
|
||||
return requested
|
||||
}
|
||||
|
||||
var match image.Rectangle
|
||||
// Since we want to search for the smallest difference we start with the highest possible number
|
||||
minDiff := math.MaxInt32
|
||||
|
||||
for _, current := range rs {
|
||||
cLen := dimensionLength(current, isLandscape)
|
||||
diff := requestedLen - cLen
|
||||
if diff > 0 {
|
||||
// current is smaller
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert diff to positive value
|
||||
// Multiplying by -1 is safe since we aren't getting positive numbers here
|
||||
// because of the check above
|
||||
absDiff := diff * -1
|
||||
if absDiff < minDiff {
|
||||
minDiff = absDiff
|
||||
match = current
|
||||
}
|
||||
}
|
||||
|
||||
if (match == image.Rectangle{}) {
|
||||
match = rs[len(rs)-1]
|
||||
}
|
||||
return match
|
||||
}
|
||||
|
||||
func dimensionLength(rect image.Rectangle, isLandscape bool) int {
|
||||
if isLandscape {
|
||||
return rect.Dx()
|
||||
}
|
||||
return rect.Dy()
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInitWithEmptyArray(t *testing.T) {
|
||||
rs, err := ParseResolutions([]string{})
|
||||
if err != nil {
|
||||
t.Errorf("Init with an empty array should not fail. Error: %s.\n", err.Error())
|
||||
}
|
||||
if len(rs) != 0 {
|
||||
t.Error("Init with an empty array should return an empty Resolutions instance.\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithNil(t *testing.T) {
|
||||
rs, err := ParseResolutions(nil)
|
||||
if err != nil {
|
||||
t.Errorf("Init with nil parameter should not fail. Error: %s.\n", err.Error())
|
||||
}
|
||||
if len(rs) != 0 {
|
||||
t.Error("Init with nil parameter should return an empty Resolutions instance.\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithInvalidValuesInArray(t *testing.T) {
|
||||
_, err := ParseResolutions([]string{"invalid"})
|
||||
if err == nil {
|
||||
t.Error("Init with invalid parameter should fail.\n")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInit(t *testing.T) {
|
||||
rs, err := ParseResolutions([]string{"16x16"})
|
||||
if err != nil {
|
||||
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
|
||||
}
|
||||
if len(rs) != 1 {
|
||||
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), 1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitWithMultipleResolutions(t *testing.T) {
|
||||
rStrs := []string{"16x16", "32x32", "64x64", "128x128"}
|
||||
rs, err := ParseResolutions(rStrs)
|
||||
if err != nil {
|
||||
t.Errorf("Init with valid parameter should not fail. Error: %s.\n", err.Error())
|
||||
}
|
||||
if len(rs) != len(rStrs) {
|
||||
t.Errorf("resolutions has size %d, expected size %d.\n", len(rs), len(rStrs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosestMatchWithEmptyResolutions(t *testing.T) {
|
||||
rs, _ := ParseResolutions(nil)
|
||||
want := image.Rect(0, 0, 24, 24)
|
||||
imgSize := image.Rect(0, 0, 24, 24)
|
||||
|
||||
r := rs.ClosestMatch(want, imgSize)
|
||||
if r.Dx() != want.Dx() || r.Dy() != want.Dy() {
|
||||
t.Errorf("ClosestMatch from empty resolutions should return the given resolution")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosestMatch(t *testing.T) {
|
||||
rs, _ := ParseResolutions([]string{"16x16", "24x24", "32x32", "64x64", "128x128"})
|
||||
|
||||
testData := [][]image.Rectangle{
|
||||
{image.Rect(0, 0, 17, 17), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
|
||||
{image.Rect(0, 0, 12, 17), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 24, 24)},
|
||||
{image.Rect(0, 0, 24, 24), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
|
||||
{image.Rect(0, 0, 20, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 24, 24)},
|
||||
{image.Rect(0, 0, 20, 80), image.Rect(0, 0, 1080, 1920), image.Rect(0, 0, 128, 128)},
|
||||
{image.Rect(0, 0, 80, 20), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
|
||||
{image.Rect(0, 0, 48, 48), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 64, 64)},
|
||||
{image.Rect(0, 0, 1024, 1024), image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 128, 128)},
|
||||
{image.Rect(0, 0, 1920, 1080), image.Rect(0, 0, 256, 36), image.Rect(0, 0, 256, 36)},
|
||||
}
|
||||
|
||||
for _, row := range testData {
|
||||
given := row[0]
|
||||
imgSize := row[1]
|
||||
expected := row[2]
|
||||
|
||||
match := rs.ClosestMatch(given, imgSize)
|
||||
|
||||
if match != expected {
|
||||
t.Errorf("Expected resolution %dx%d got %dx%d", expected.Dx(), expected.Dy(), match.Dx(), match.Dy())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithEmptyString(t *testing.T) {
|
||||
if _, err := ParseResolution(""); err == nil {
|
||||
t.Error("Parse with empty string should return an error.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithInvalidWidth(t *testing.T) {
|
||||
_, err := ParseResolution("invalidx42")
|
||||
if err == nil {
|
||||
t.Error("Parse with invalid width should return an error.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithInvalidHeight(t *testing.T) {
|
||||
_, err := ParseResolution("42xinvalid")
|
||||
if err == nil {
|
||||
t.Error("Parse with invalid height should return an error.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseResolution(t *testing.T) {
|
||||
rStr := "42x23"
|
||||
r, _ := ParseResolution(rStr)
|
||||
if r.Dx() != 42 || r.Dy() != 23 {
|
||||
t.Errorf("Expected resolution %s got %s", rStr, r.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/config"
|
||||
)
|
||||
|
||||
const (
|
||||
filesDir = "files"
|
||||
)
|
||||
|
||||
// NewFileSystemStorage creates a new instance of FileSystem
|
||||
func NewFileSystemStorage(cfg config.FileSystemStorage, logger log.Logger) FileSystem {
|
||||
return FileSystem{
|
||||
root: cfg.RootDirectory,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// FileSystem represents a storage for the thumbnails using the local file system.
|
||||
type FileSystem struct {
|
||||
root string
|
||||
logger log.Logger
|
||||
}
|
||||
|
||||
// Stat returns if a file for the given key exists on the filesystem
|
||||
func (s FileSystem) Stat(key string) bool {
|
||||
img := filepath.Join(s.root, filesDir, key)
|
||||
if _, err := os.Stat(img); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Get returns the file content for the given key
|
||||
func (s FileSystem) Get(key string) ([]byte, error) {
|
||||
img := filepath.Join(s.root, filesDir, key)
|
||||
content, err := os.ReadFile(img)
|
||||
if err != nil {
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
s.logger.Debug().Str("err", err.Error()).Str("key", key).Msg("could not load thumbnail from store")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// Put stores image data in the file system for the given key
|
||||
func (s FileSystem) Put(key string, img []byte) error {
|
||||
imgPath := filepath.Join(s.root, filesDir, key)
|
||||
dir := filepath.Dir(imgPath)
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return errors.Wrapf(err, "error while creating directory %s", dir)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(imgPath); os.IsNotExist(err) {
|
||||
f, err := os.CreateTemp(dir, "tmpthumb")
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not create temporary file for \"%s\"", key)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// if there was a problem writing, remove the temporary file
|
||||
if _, writeErr := f.Write(img); writeErr != nil {
|
||||
if remErr := os.Remove(f.Name()); remErr != nil {
|
||||
return errors.Wrapf(remErr, "could not cleanup temporary file for \"%s\"", key)
|
||||
}
|
||||
return errors.Wrapf(writeErr, "could not write to temporary file for \"%s\"", key)
|
||||
}
|
||||
|
||||
// if there wasn't a problem, ensure the data is written into disk
|
||||
if synErr := f.Sync(); synErr != nil {
|
||||
return errors.Wrapf(synErr, "could not sync temporary file data into the disk for \"%s\"", key)
|
||||
}
|
||||
|
||||
// rename the temporary file to the final file
|
||||
if renErr := os.Rename(f.Name(), imgPath); renErr != nil {
|
||||
// if we couldn't rename, remove the temporary file
|
||||
if remErr := os.Remove(f.Name()); remErr != nil {
|
||||
return errors.Wrapf(remErr, "rename failed and could not cleanup temporary file for \"%s\"", key)
|
||||
}
|
||||
return errors.Wrapf(renErr, "could not rename temporary file to \"%s\"", key)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildKey generate the unique key for a thumbnail.
|
||||
// The key is structure as follows:
|
||||
//
|
||||
// <first two letters of checksum>/<next two letters of checksum>/<rest of checksum>/<width>x<height>.<filetype>
|
||||
//
|
||||
// e.g. 97/9f/4c8db98f7b82e768ef478d3c8612/500x300.png
|
||||
//
|
||||
// The key also represents the path to the thumbnail in the filesystem under the configured root directory.
|
||||
func (s FileSystem) BuildKey(r Request) string {
|
||||
checksum := r.Checksum
|
||||
filetype := r.Types[0]
|
||||
|
||||
parts := []string{strconv.Itoa(r.Resolution.Dx()), "x", strconv.Itoa(r.Resolution.Dy())}
|
||||
|
||||
if r.Characteristic != "" {
|
||||
parts = append(parts, "-", r.Characteristic)
|
||||
}
|
||||
|
||||
parts = append(parts, ".", filetype)
|
||||
|
||||
return filepath.Join(checksum[:2], checksum[2:4], checksum[4:], strings.Join(parts, ""))
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package storage_test
|
||||
|
||||
import (
|
||||
"image"
|
||||
"testing"
|
||||
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
)
|
||||
|
||||
func TestFileSystem_BuildKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
r storage.Request
|
||||
want string
|
||||
}{
|
||||
{
|
||||
r: storage.Request{
|
||||
Checksum: "120EA8A25E5D487BF68B5F7096440019",
|
||||
Types: []string{"png", "jpg"},
|
||||
Resolution: image.Rectangle{
|
||||
Min: image.Point{
|
||||
X: 1,
|
||||
Y: 2,
|
||||
},
|
||||
Max: image.Point{
|
||||
X: 3,
|
||||
Y: 4,
|
||||
},
|
||||
},
|
||||
Characteristic: "",
|
||||
},
|
||||
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2.png",
|
||||
},
|
||||
{
|
||||
r: storage.Request{
|
||||
Checksum: "120EA8A25E5D487BF68B5F7096440019",
|
||||
Types: []string{"png", "jpg"},
|
||||
Resolution: image.Rectangle{
|
||||
Min: image.Point{
|
||||
X: 1,
|
||||
Y: 2,
|
||||
},
|
||||
Max: image.Point{
|
||||
X: 3,
|
||||
Y: 4,
|
||||
},
|
||||
},
|
||||
Characteristic: "fill",
|
||||
},
|
||||
want: "12/0E/A8A25E5D487BF68B5F7096440019/2x2-fill.png",
|
||||
},
|
||||
}
|
||||
|
||||
s := storage.FileSystem{}
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
assert.Equal(s.BuildKey(tt.r), tt.want)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"image"
|
||||
)
|
||||
|
||||
// Request combines different attributes needed for storage operations.
|
||||
type Request struct {
|
||||
// The checksum of the source file
|
||||
// Will be used to determine if a thumbnail exists
|
||||
Checksum string
|
||||
// Types provided by the encoder.
|
||||
// Contains the mimetypes of the thumbnail.
|
||||
// In case of jpg/jpeg it will contain both.
|
||||
Types []string
|
||||
// The resolution of the thumbnail
|
||||
Resolution image.Rectangle
|
||||
// Characteristic defines the different image characteristics,
|
||||
// for example, if it's scaled up to fit in the bounding box or not,
|
||||
// is it a chroma version of the image, and so on...
|
||||
// the main propose for this is to be able to differentiate between images which have
|
||||
// the same resolution but different characteristics.
|
||||
Characteristic string
|
||||
}
|
||||
|
||||
// Storage defines the interface for a thumbnail store.
|
||||
type Storage interface {
|
||||
Stat(key string) bool
|
||||
Get(key string) ([]byte, error)
|
||||
Put(key string, img []byte) error
|
||||
BuildKey(r Request) string
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"mime"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
)
|
||||
|
||||
// Request bundles information needed to generate a thumbnail for a file
|
||||
type Request struct {
|
||||
Resolution image.Rectangle
|
||||
Encoder Encoder
|
||||
Generator Generator
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// Manager is responsible for generating thumbnails
|
||||
type Manager interface {
|
||||
// Generate creates a thumbnail and stores it.
|
||||
// The function returns a key with which the actual file can be retrieved.
|
||||
Generate(r Request, img any) (string, error)
|
||||
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
|
||||
// The function will return a status if the file exists and the key to the file.
|
||||
CheckThumbnail(r Request) (string, bool)
|
||||
// GetThumbnail will load the thumbnail from the storage and return its content.
|
||||
GetThumbnail(key string) ([]byte, error)
|
||||
}
|
||||
|
||||
// NewSimpleManager creates a new instance of SimpleManager
|
||||
func NewSimpleManager(resolutions Resolutions, storage storage.Storage, logger log.Logger, maxInputWidth, maxInputHeight int) SimpleManager {
|
||||
return SimpleManager{
|
||||
storage: storage,
|
||||
logger: logger,
|
||||
resolutions: resolutions,
|
||||
maxDimension: image.Point{X: maxInputWidth, Y: maxInputHeight},
|
||||
}
|
||||
}
|
||||
|
||||
// SimpleManager is a simple implementation of Manager
|
||||
type SimpleManager struct {
|
||||
storage storage.Storage
|
||||
logger log.Logger
|
||||
resolutions Resolutions
|
||||
maxDimension image.Point
|
||||
}
|
||||
|
||||
// Generate creates a thumbnail and stores it
|
||||
func (s SimpleManager) Generate(r Request, img any) (string, error) {
|
||||
var match image.Rectangle
|
||||
|
||||
inputDimensions, err := r.Generator.Dimensions(img)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
match = s.resolutions.ClosestMatch(r.Resolution, inputDimensions)
|
||||
|
||||
// validate max input image dimensions - 6016x4000
|
||||
if inputDimensions.Size().X > s.maxDimension.X || inputDimensions.Size().Y > s.maxDimension.Y {
|
||||
return "", errors.ErrImageTooLarge
|
||||
}
|
||||
|
||||
thumbnail, err := r.Generator.Generate(match, img)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if err := r.Encoder.Encode(buf, thumbnail); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
k := s.storage.BuildKey(mapToStorageRequest(r))
|
||||
if err := s.storage.Put(k, buf.Bytes()); err != nil {
|
||||
s.logger.Error().Err(err).Msg("could not store thumbnail")
|
||||
return "", err
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// CheckThumbnail checks if a thumbnail with the requested attributes exists.
|
||||
func (s SimpleManager) CheckThumbnail(r Request) (string, bool) {
|
||||
k := s.storage.BuildKey(mapToStorageRequest(r))
|
||||
return k, s.storage.Stat(k)
|
||||
}
|
||||
|
||||
// GetThumbnail will load the thumbnail from the storage and return its content.
|
||||
func (s SimpleManager) GetThumbnail(key string) ([]byte, error) {
|
||||
return s.storage.Get(key)
|
||||
}
|
||||
|
||||
func mapToStorageRequest(r Request) storage.Request {
|
||||
return storage.Request{
|
||||
Checksum: r.Checksum,
|
||||
Resolution: r.Resolution,
|
||||
Types: r.Encoder.Types(),
|
||||
Characteristic: r.Generator.ProcessorID(),
|
||||
}
|
||||
}
|
||||
|
||||
// IsMimeTypeSupported validate if the mime type is supported
|
||||
func IsMimeTypeSupported(m string) bool {
|
||||
mimeType, _, err := mime.ParseMediaType(m)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, supported := SupportedMimeTypes[mimeType]
|
||||
return supported
|
||||
}
|
||||
|
||||
// PrepareRequest prepare the request based on image parameters
|
||||
func PrepareRequest(width, height int, tType, checksum, pID string) (Request, error) {
|
||||
generator, err := GeneratorFor(tType, pID)
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
encoder, err := EncoderForType(tType)
|
||||
if err != nil {
|
||||
return Request{}, err
|
||||
}
|
||||
|
||||
return Request{
|
||||
Resolution: image.Rect(0, 0, width, height),
|
||||
Generator: generator,
|
||||
Encoder: encoder,
|
||||
Checksum: checksum,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package thumbnail
|
||||
|
||||
import (
|
||||
"image"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/errors"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/preprocessor"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/thumbnails/pkg/thumbnail/storage"
|
||||
)
|
||||
|
||||
type NoOpManager struct {
|
||||
storage.Storage
|
||||
}
|
||||
|
||||
func (m NoOpManager) BuildKey(_ storage.Request) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m NoOpManager) Set(_, _ string, _ []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func BenchmarkGet(b *testing.B) {
|
||||
|
||||
sut := NewSimpleManager(
|
||||
Resolutions{},
|
||||
NoOpManager{},
|
||||
log.NewLogger(),
|
||||
6016,
|
||||
4000,
|
||||
)
|
||||
|
||||
res, _ := ParseResolution("32x32")
|
||||
req := Request{
|
||||
Resolution: res,
|
||||
Checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
}
|
||||
cwd, _ := os.Getwd()
|
||||
p := filepath.Join(cwd, "../../testdata/test.png")
|
||||
f, _ := os.Open(p)
|
||||
defer f.Close()
|
||||
img, ext, _ := image.Decode(f)
|
||||
req.Encoder, _ = EncoderForType(ext)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = sut.Generate(req, img)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareRequest(t *testing.T) {
|
||||
type args struct {
|
||||
width int
|
||||
height int
|
||||
tType string
|
||||
checksum string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "Test successful prepare the request for jpg",
|
||||
args: args{
|
||||
width: 32,
|
||||
height: 32,
|
||||
tType: "jpg",
|
||||
checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
want: Request{
|
||||
Resolution: image.Rect(0, 0, 32, 32),
|
||||
Encoder: JpegEncoder{},
|
||||
Generator: SimpleGenerator{},
|
||||
Checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Test successful prepare the request for png",
|
||||
args: args{
|
||||
width: 32,
|
||||
height: 32,
|
||||
tType: "png",
|
||||
checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
want: Request{
|
||||
Resolution: image.Rect(0, 0, 32, 32),
|
||||
Encoder: PngEncoder{},
|
||||
Generator: SimpleGenerator{},
|
||||
Checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Test successful prepare the request for gif",
|
||||
args: args{
|
||||
width: 32,
|
||||
height: 32,
|
||||
tType: "gif",
|
||||
checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
want: Request{
|
||||
Resolution: image.Rect(0, 0, 32, 32),
|
||||
Encoder: GifEncoder{},
|
||||
Generator: GifGenerator{},
|
||||
Checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Test error when prepare the request for bmp",
|
||||
args: args{
|
||||
width: 32,
|
||||
height: 32,
|
||||
tType: "bmp",
|
||||
checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := PrepareRequest(tt.args.width, tt.args.height, tt.args.tType, tt.args.checksum, "")
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("PrepareRequest() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
// funcs are not reflactable, ignore
|
||||
if diff := cmp.Diff(tt.want, got, cmpopts.IgnoreFields(Request{}, "Generator")); diff != "" {
|
||||
t.Errorf("PrepareRequest(): %v", diff)
|
||||
}
|
||||
if reflect.TypeOf(got.Generator) != reflect.TypeOf(tt.want.Generator) {
|
||||
t.Errorf("PrepareRequest() = %v, want %v", reflect.TypeOf(got.Generator), reflect.TypeOf(tt.want.Generator))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewGenerationTooBigImage(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fileName string
|
||||
mimeType string
|
||||
}{
|
||||
{name: "png", mimeType: "image/png", fileName: "../../testdata/test.png"},
|
||||
{name: "jpg", mimeType: "image/jpeg", fileName: "../../testdata/test.jpg"},
|
||||
{name: "ggs", mimeType: "application/vnd.geogebra.slides", fileName: "../../testdata/test.ggs"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sut := NewSimpleManager(
|
||||
Resolutions{},
|
||||
NoOpManager{},
|
||||
log.NewLogger(),
|
||||
1024,
|
||||
768,
|
||||
)
|
||||
|
||||
res, _ := ParseResolution("32x32")
|
||||
req := Request{
|
||||
Resolution: res,
|
||||
Checksum: "1872ade88f3013edeb33decd74a4f947",
|
||||
}
|
||||
cwd, _ := os.Getwd()
|
||||
p := filepath.Join(cwd, tt.fileName)
|
||||
f, _ := os.Open(p)
|
||||
defer f.Close()
|
||||
|
||||
preproc := preprocessor.ForType(tt.mimeType, nil)
|
||||
convert, err := preproc.Convert(f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ext := path.Ext(tt.fileName)
|
||||
req.Encoder, _ = EncoderForType(ext)
|
||||
req.Generator, err = GeneratorFor(ext, "fit")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
generate, err := sut.Generate(req, convert)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
assert.ErrorIs(t, err, errors.ErrImageTooLarge)
|
||||
assert.Equal(t, "", generate)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user