Initial QSfera import

Server/services/thumbnails/pkg/preprocessor/fontloader.go

@@ -0,0 +1,192 @@
+package preprocessor
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/qsfera/server/pkg/sync"
+	"golang.org/x/image/font"
+	"golang.org/x/image/font/gofont/goregular"
+	"golang.org/x/image/font/opentype"
+)
+
+// FontMap maps a script with the target font to be used for that script
+// It also uses a DefaultFont in case there isn't a matching script in the map
+//
+// For cases like Japanese where multiple scripts are used, we rely on the text
+// analyzer to use the script which is unique to japanese (Hiragana or Katakana)
+// even if it has to overwrite the "official" detected script (Han). This means
+// that "Han" should be used just for chinese while "Hiragana" and "Katakana"
+// should be used for japanese
+type FontMap struct {
+	FontMap     map[string]string `json:"fontMap"`
+	DefaultFont string            `json:"defaultFont"`
+}
+
+// FontMapData contains the location of the loaded file (in FLoc) and the FontMap loaded
+// from the file
+type FontMapData struct {
+	FMap *FontMap
+	FLoc string
+}
+
+// LoadedFace contains the location of the font used, and the loaded face (font.Face)
+// ready to be used
+type LoadedFace struct {
+	FontFile string
+	Face     font.Face
+}
+
+// FontLoader represents a FontLoader. Use the "NewFontLoader" to get a instance
+type FontLoader struct {
+	faceCache   sync.Cache
+	fontMapData *FontMapData
+	faceOpts    *opentype.FaceOptions
+}
+
+// NewFontLoader creates a new FontLoader based on the fontMapFile. The FaceOptions will
+// be the same for all the font loaded by this instance.
+// Note that only the fonts described in the fontMapFile will be used.
+//
+// The fontMapFile has the following structure
+//
+//	{
+//		"fontMap": {
+//			"Han": "packaged/myFont-CJK.otf",
+//			"Arabic": "packaged/myFont-Arab.otf",
+//			"Latin": "/fonts/regular/myFont.otf"
+//		}
+//		"defaultFont": "/fonts/regular/myFont.otf"
+//	}
+//
+// The fontMapFile contains paths to where the fonts are located in the FS.
+// Absolute paths can be used as shown above. If a relative path is used,
+// it will be relative to the fontMapFile location. This should make the
+// packaging easier since all the fonts can be placed in the same directory
+// where the fontMapFile is, or in inner directories.
+func NewFontLoader(fontMapFile string, faceOpts *opentype.FaceOptions) (*FontLoader, error) {
+	fontMap := &FontMap{}
+
+	if fontMapFile != "" {
+		file, err := os.Open(fontMapFile)
+		if err != nil {
+			return nil, err
+		}
+		defer file.Close()
+
+		parser := json.NewDecoder(file)
+		if err = parser.Decode(fontMap); err != nil {
+			return nil, err
+		}
+	}
+
+	return &FontLoader{
+		faceCache: sync.NewCache(5),
+		fontMapData: &FontMapData{
+			FMap: fontMap,
+			FLoc: fontMapFile,
+		},
+		faceOpts: faceOpts,
+	}, nil
+}
+
+// LoadFaceForScript loads and returns the font face to be used for that script according to the
+// FontMap set when the FontLoader was created. If the script doesn't have
+// an associated font, a default font will be used. Note that the default font
+// might not be able to handle properly the script
+func (fl *FontLoader) LoadFaceForScript(script string) (*LoadedFace, error) {
+	var parsedFont *opentype.Font
+	var parsingError error
+
+	fontFile := fl.fontMapData.FMap.DefaultFont
+	if val, ok := fl.fontMapData.FMap.FontMap[script]; ok {
+		fontFile = val
+	}
+
+	if fontFile != "" && !filepath.IsAbs(fontFile) {
+		fontFile = filepath.Join(filepath.Dir(fl.fontMapData.FLoc), fontFile)
+	}
+
+	// if the face for the script isn't cached, load the font file and create a new face
+	cachedFace := fl.faceCache.Load(fontFile)
+	if cachedFace != nil {
+		return cachedFace.V.(*LoadedFace), nil
+	}
+
+	if fontFile == "" {
+		parsedFont, parsingError = opentype.Parse(goregular.TTF)
+		if parsingError != nil {
+			return nil, parsingError
+		}
+	} else {
+		// opentype.ParseReaderAt seems to require to keep the file opened
+		// so read the font file into memory
+		data, err := os.ReadFile(fontFile)
+		if err != nil {
+			return nil, err
+		}
+		parsedFont, parsingError = opentype.Parse(data)
+		if parsingError != nil {
+			return nil, parsingError
+		}
+	}
+
+	face, err := opentype.NewFace(parsedFont, fl.faceOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	loadedFace := &LoadedFace{
+		FontFile: fontFile,
+		Face:     face,
+	}
+	fl.faceCache.Store(fontFile, loadedFace, time.Now().Add(10*time.Minute))
+	return loadedFace, nil
+}
+
+// GetFaceOptSize returns face opt size
+func (fl *FontLoader) GetFaceOptSize() float64 {
+	return fl.faceOpts.Size
+}
+
+// GetFaceOptDPI returns face opt DPI
+func (fl *FontLoader) GetFaceOptDPI() float64 {
+	return fl.faceOpts.DPI
+}
+
+// GetScriptList returns script list
+func (fl *FontLoader) GetScriptList() []string {
+	fontMap := fl.fontMapData.FMap.FontMap
+
+	arePresent := map[string]bool{
+		"Common":    false,
+		"Inherited": false,
+	}
+	listSize := len(fontMap)
+
+	for key := range arePresent {
+		if _, inFontMap := fontMap[key]; inFontMap {
+			arePresent[key] = true
+		} else {
+			listSize++
+		}
+	}
+
+	keys := make([]string, listSize)
+
+	i := 0
+	for k := range fontMap {
+		keys[i] = k
+		i++
+	}
+
+	for script, isPresent := range arePresent {
+		if !isPresent {
+			keys[i] = script
+			i++
+		}
+	}
+	return keys
+}

Server/services/thumbnails/pkg/preprocessor/preprocessor.go

@@ -0,0 +1,350 @@
+package preprocessor
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"image"
+	"image/draw"
+	"image/gif"
+	"io"
+	"math"
+	"mime"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/image/font"
+	"golang.org/x/image/font/opentype"
+	"golang.org/x/image/math/fixed"
+
+	"github.com/dhowden/tag"
+
+	thumbnailerErrors "github.com/qsfera/server/services/thumbnails/pkg/errors"
+)
+
+// FileConverter is the interface for the file converter
+type FileConverter interface {
+	Convert(r io.Reader) (any, error)
+}
+
+// GifDecoder is a converter for the gif file
+type GifDecoder struct{}
+
+// Convert reads the gif file and returns the thumbnail image
+func (i GifDecoder) Convert(r io.Reader) (any, error) {
+	img, err := gif.DecodeAll(r)
+	if err != nil {
+		return nil, errors.Wrap(err, `could not decode the image`)
+	}
+	return img, nil
+}
+
+// GgsDecoder is a converter for the geogebra slides file
+type GgsDecoder struct{ thumbnailpath string }
+
+// Convert reads the ggs file and returns the thumbnail image
+func (g GgsDecoder) Convert(r io.Reader) (any, error) {
+	var buf bytes.Buffer
+	_, err := io.Copy(&buf, r)
+	if err != nil {
+		return nil, err
+	}
+	zipReader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+	if err != nil {
+		return nil, err
+	}
+	for _, file := range zipReader.File {
+		if file.Name == g.thumbnailpath {
+			thumbnail, err := file.Open()
+			if err != nil {
+				return nil, err
+			}
+			converter := ForType("image/png", nil)
+			if converter == nil {
+				return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromGgsFile
+			}
+			img, err := converter.Convert(thumbnail)
+			if err != nil {
+				return nil, errors.Wrap(err, `could not decode the image`)
+			}
+			return img, nil
+		}
+	}
+	return nil, errors.Errorf("%s not found", g.thumbnailpath)
+}
+
+// AudioDecoder is a converter for the audio file
+type AudioDecoder struct{}
+
+// Convert reads the audio file and extracts the thumbnail image from the id3 tag
+func (i AudioDecoder) Convert(r io.Reader) (any, error) {
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	m, err := tag.ReadFrom(bytes.NewReader(b))
+	if err != nil {
+		return nil, err
+	}
+
+	picture := m.Picture()
+	if picture == nil {
+		return nil, thumbnailerErrors.ErrNoImageFromAudioFile
+	}
+
+	converter := ForType(picture.MIMEType, nil)
+	if converter == nil {
+		return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
+	}
+
+	return converter.Convert(bytes.NewReader(picture.Data))
+}
+
+// TxtToImageConverter is a converter for the text file
+type TxtToImageConverter struct {
+	fontLoader *FontLoader
+}
+
+// Convert reads the text file and renders it into a thumbnail image
+func (t TxtToImageConverter) Convert(r io.Reader) (any, error) {
+	img := image.NewRGBA(image.Rect(0, 0, 640, 480))
+
+	imgBounds := img.Bounds()
+	draw.Draw(img, imgBounds, image.White, image.Point{}, draw.Src)
+
+	fontSizeAsInt := int(math.Ceil(t.fontLoader.GetFaceOptSize()))
+	margin := 10
+	minX := fixed.I(imgBounds.Min.X + margin)
+	maxX := fixed.I(imgBounds.Max.X - margin)
+	maxY := fixed.I(imgBounds.Max.Y - margin)
+	initialPoint := fixed.P(imgBounds.Min.X+margin, imgBounds.Min.Y+margin+fontSizeAsInt)
+	canvas := &font.Drawer{
+		Dst: img,
+		Src: image.Black,
+		Dot: initialPoint,
+	}
+
+	scriptList := t.fontLoader.GetScriptList()
+	textAnalyzer := NewTextAnalyzer(scriptList)
+	taOpts := AnalysisOpts{
+		UseMergeMap: true,
+		MergeMap:    DefaultMergeMap,
+	}
+
+	scanner := bufio.NewScanner(r)
+Scan: // Label for the scanner loop, so we can break it easily
+	for scanner.Scan() {
+		txt := scanner.Text()
+		height := fixed.I(fontSizeAsInt) // reset to default height
+
+		textResult := textAnalyzer.AnalyzeString(txt, taOpts)
+		textResult.MergeCommon(DefaultMergeMap)
+
+		for _, sRange := range textResult.ScriptRanges {
+			targetFontFace, err := t.fontLoader.LoadFaceForScript(sRange.TargetScript)
+			if err != nil {
+				return nil, err
+			}
+
+			// if the target script is "_unknown" it's expected that the loaded face
+			// uses the default font
+			faceHeight := targetFontFace.Face.Metrics().Height
+			if faceHeight > height {
+				height = faceHeight
+			}
+
+			canvas.Face = targetFontFace.Face
+			initialByte := sRange.Low
+			for _, sRangeSpace := range sRange.Spaces {
+				if canvas.Dot.Y > maxY {
+					break Scan
+				}
+
+				drawWord(canvas, textResult.Text[initialByte:sRangeSpace], minX, maxX, height, maxY)
+				initialByte = sRangeSpace
+			}
+
+			if initialByte <= sRange.High {
+				// some bytes left to be written
+				if canvas.Dot.Y > maxY {
+					break Scan
+				}
+
+				drawWord(canvas, textResult.Text[initialByte:sRange.High+1], minX, maxX, height, maxY)
+			}
+		}
+
+		canvas.Dot.X = minX
+		canvas.Dot.Y += height.Mul(fixed.Int26_6(1<<6 + 1<<5)) // height * 1.5
+
+		if canvas.Dot.Y > maxY {
+			break
+		}
+	}
+	return img, scanner.Err()
+}
+
+// GGPStruct is the layout of a ggp file (which is basically json)
+type GGPStruct struct {
+	Sections []struct {
+		Cards []struct {
+			Element struct {
+				Image struct {
+					Base64Image string
+				}
+			}
+		}
+	}
+}
+
+// GgpDecoder is a converter for the geogebra pinboard file
+type GgpDecoder struct{}
+
+// Convert reads the ggp file and returns the first thumbnail image
+func (j GgpDecoder) Convert(r io.Reader) (any, error) {
+	ggp := &GGPStruct{}
+	err := json.NewDecoder(r).Decode(ggp)
+	if err != nil {
+		return nil, err
+	}
+
+	elem, err := extractBase64ImageFromGGP(ggp)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := base64.StdEncoding.DecodeString(elem)
+	if err != nil {
+		return nil, err
+	}
+
+	img, _, err := image.Decode(bytes.NewReader(b))
+	return img, err
+}
+
+func extractBase64ImageFromGGP(ggp *GGPStruct) (string, error) {
+	if len(ggp.Sections) < 1 || len(ggp.Sections[0].Cards) < 1 {
+		return "", errors.New("cant find thumbnail in ggp file")
+	}
+
+	raw := strings.Split(ggp.Sections[0].Cards[0].Element.Image.Base64Image, "base64,")
+	if len(raw) < 2 {
+		return "", errors.New("cant decode ggp thumbnail")
+	}
+
+	return raw[1], nil
+}
+
+// Draw the word in the canvas. The mixX and maxX defines the drawable range
+// (X axis) where the word can be drawn (in case the word is too big and doesn't
+// fit in the canvas), and the incY defines the increment in the Y axis if we
+// need to draw the word in a new line
+//
+// Note that the word will likely start with a white space char
+func drawWord(canvas *font.Drawer, word string, minX, maxX, incY, maxY fixed.Int26_6) {
+	// calculate the actual measurement of the string at a given X position
+	measure := func(s string, dotX fixed.Int26_6) (min, max fixed.Int26_6) {
+		bbox, _ := canvas.BoundString(s)
+		return dotX + bbox.Min.X, dotX + bbox.Max.X
+	}
+
+	// first try to draw the whole word
+	absMin, absMax := measure(word, canvas.Dot.X)
+	if absMin >= minX && absMax <= maxX {
+		canvas.DrawString(word)
+		return
+	}
+
+	// try to draw the trimmed word in a new line
+	trimmed := strings.TrimSpace(word)
+	oldDot := canvas.Dot
+	canvas.Dot.X = minX
+	canvas.Dot.Y += incY
+
+	if canvas.Dot.Y <= maxY {
+		tMin, tMax := measure(trimmed, canvas.Dot.X)
+		if tMin >= minX && tMax <= maxX {
+			canvas.DrawString(trimmed)
+			return
+		}
+	}
+
+	// if the trimmed word is still too big, draw it char by char
+	canvas.Dot = oldDot
+	for _, char := range trimmed {
+		s := string(char)
+		_, cMax := measure(s, canvas.Dot.X)
+
+		if cMax > maxX {
+			canvas.Dot.X = minX
+			canvas.Dot.Y += incY
+		}
+
+		// stop drawing if we exceed maxY
+		if canvas.Dot.Y > maxY {
+			return
+		}
+
+		// ensure that we don't start drawing before minX
+		cMin, _ := measure(s, canvas.Dot.X)
+		if cMin < minX {
+			canvas.Dot.X += minX - cMin
+		}
+
+		canvas.DrawString(s)
+	}
+}
+
+// ForType returns the converter for the specified mimeType
+func ForType(mimeType string, opts map[string]any) FileConverter {
+	// We can ignore the error here because we parse it in IsMimeTypeSupported before and if it fails
+	// return the service call. So we should only get here when the mimeType parses fine.
+	mimeType, _, _ = mime.ParseMediaType(mimeType)
+	switch mimeType {
+	case "text/plain":
+		fontFileMap := ""
+		fontFaceOpts := &opentype.FaceOptions{
+			Size:    12,
+			DPI:     72,
+			Hinting: font.HintingNone,
+		}
+
+		if optedFontFileMap, ok := opts["fontFileMap"]; ok {
+			if stringFontFileMap, ok := optedFontFileMap.(string); ok {
+				fontFileMap = stringFontFileMap
+			}
+		}
+
+		if optedFontFaceOpts, ok := opts["fontFaceOpts"]; ok {
+			if typedFontFaceOpts, ok := optedFontFaceOpts.(*opentype.FaceOptions); ok {
+				fontFaceOpts = typedFontFaceOpts
+			}
+		}
+
+		fontLoader, err := NewFontLoader(fontFileMap, fontFaceOpts)
+		if err != nil {
+			// if it couldn't create the FontLoader with the specified fontFileMap,
+			// try to use the default font
+			fontLoader, _ = NewFontLoader("", fontFaceOpts)
+		}
+		return TxtToImageConverter{
+			fontLoader: fontLoader,
+		}
+	case "application/vnd.geogebra.slides":
+		return GgsDecoder{"_slide0/geogebra_thumbnail.png"}
+	case "application/vnd.geogebra.pinboard":
+		return GgpDecoder{}
+	case "image/gif":
+		return GifDecoder{}
+	case "audio/flac":
+		fallthrough
+	case "audio/mpeg":
+		fallthrough
+	case "audio/ogg":
+		return AudioDecoder{}
+	default:
+		return ImageDecoder{}
+	}
+}

Server/services/thumbnails/pkg/preprocessor/preprocessor_imaging.go

@@ -0,0 +1,22 @@
+//go:build !enable_vips
+
+package preprocessor
+
+import (
+	"io"
+
+	"github.com/kovidgoyal/imaging"
+	"github.com/pkg/errors"
+)
+
+// ImageDecoder is a converter for the image file
+type ImageDecoder struct{}
+
+// Convert reads the image file and returns the thumbnail image
+func (i ImageDecoder) Convert(r io.Reader) (any, error) {
+	img, err := imaging.Decode(r, imaging.AutoOrientation(true))
+	if err != nil {
+		return nil, errors.Wrap(err, `could not decode the image`)
+	}
+	return img, nil
+}

Server/services/thumbnails/pkg/preprocessor/preprocessor_test.go

@@ -0,0 +1,197 @@
+package preprocessor
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"golang.org/x/image/font"
+	"golang.org/x/image/font/opentype"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestImageDecoder(t *testing.T) {
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ImageDecoder Suite")
+}
+
+var _ = Describe("ImageDecoder", func() {
+	Describe("ImageDecoder", func() {
+		var fileReader io.Reader
+		BeforeEach(func() {
+			fileContent, err := os.ReadFile("test_assets/noise.png")
+			if err != nil {
+				panic(err)
+			}
+			fileReader = bytes.NewReader(fileContent)
+		})
+
+		It("should decode an image", func() {
+			decoder := ImageDecoder{}
+			img, err := decoder.Convert(fileReader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(img).ToNot(BeNil())
+		})
+
+		It("should return an error if the image is invalid", func() {
+			decoder := ImageDecoder{}
+			img, err := decoder.Convert(bytes.NewReader([]byte("not an image")))
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+	})
+
+	Describe("GifDecoder", func() {
+		var fileReader io.Reader
+		BeforeEach(func() {
+			fileContent, err := os.ReadFile("test_assets/noise.gif")
+			if err != nil {
+				panic(err)
+			}
+			fileReader = bytes.NewReader(fileContent)
+		})
+
+		It("should decode a gif", func() {
+			decoder := GifDecoder{}
+			img, err := decoder.Convert(fileReader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(img).ToNot(BeNil())
+		})
+
+		It("should return an error if the gif is invalid", func() {
+			decoder := GifDecoder{}
+			img, err := decoder.Convert(bytes.NewReader([]byte("not a gif")))
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+	})
+
+	Describe("GgsDecoder", func() {
+		var fileReader io.Reader
+		BeforeEach(func() {
+			fileContent, err := os.ReadFile("test_assets/ggs_test.ggs")
+			if err != nil {
+				panic(err)
+			}
+			fileReader = bytes.NewReader(fileContent)
+		})
+
+		It("should decode a ggs", func() {
+			decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
+			img, err := decoder.Convert(fileReader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(img).ToNot(BeNil())
+		})
+
+		It("should return an error if the ggs is invalid", func() {
+			decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
+			img, err := decoder.Convert(bytes.NewReader([]byte("not a ggs")))
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+	})
+
+	Describe("should decode audio", func() {
+		var fileReader io.Reader
+		It("should decode an audio", func() {
+			fileContent, err := os.ReadFile("test_assets/empty.mp3")
+			if err != nil {
+				panic(err)
+			}
+			fileReader = bytes.NewReader(fileContent)
+			decoder := AudioDecoder{}
+			img, err := decoder.Convert(fileReader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(img).ToNot(BeNil())
+		})
+		It("should decode an audio", func() {
+			fileContent, err := os.ReadFile("test_assets/empty_no_image.mp3")
+			if err != nil {
+				panic(err)
+			}
+			fileReader = bytes.NewReader(fileContent)
+			decoder := AudioDecoder{}
+			img, err := decoder.Convert(fileReader)
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+		It("should return an error if the audio is invalid", func() {
+			decoder := AudioDecoder{}
+			img, err := decoder.Convert(bytes.NewReader([]byte("not an audio")))
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+	})
+
+	Describe("should decode text", func() {
+		var decoder TxtToImageConverter
+		BeforeEach(func() {
+			fontFaceOpts := &opentype.FaceOptions{
+				Size:    12,
+				DPI:     72,
+				Hinting: font.HintingNone,
+			}
+
+			fontLoader, err := NewFontLoader("", fontFaceOpts)
+			if err != nil {
+				fontLoader, _ = NewFontLoader("", fontFaceOpts)
+			}
+			decoder = TxtToImageConverter{
+				fontLoader: fontLoader,
+			}
+		})
+		It("should decode a text", func() {
+			img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(img).ToNot(BeNil())
+		})
+
+		It("fails if the font is missing", func() {
+			decoder.fontLoader.fontMapData = &FontMapData{
+				FMap: &FontMap{
+					DefaultFont: "/some/unknown/font.otf",
+				},
+			}
+			img, err := decoder.Convert(bytes.NewReader([]byte("This is a test text")))
+			Expect(err).To(HaveOccurred())
+			Expect(img).To(BeNil())
+		})
+	})
+
+	Describe("test ForType", func() {
+		It("should return an ImageDecoder for image types", func() {
+			decoder := ForType("image/png", nil)
+			Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
+		})
+
+		It("should return an GifDecoder for gif types", func() {
+			decoder := ForType("image/gif", nil)
+			Expect(decoder).To(BeAssignableToTypeOf(GifDecoder{}))
+		})
+
+		It("should return an GgsDecoder for ggs types", func() {
+			decoder := ForType("application/vnd.geogebra.ggs", nil)
+			// This will not return the expected ggsDecoder, but an ImageDecoder since ggs contains an embedded png.
+			Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
+		})
+
+		It("should return an AudioDecoder for audio types", func() {
+			decoder := ForType("audio/mpeg", nil)
+			Expect(decoder).To(BeAssignableToTypeOf(AudioDecoder{}))
+		})
+
+		It("should return an TxtToImageConverter for text types", func() {
+			decoder := ForType("text/plain", nil)
+			Expect(decoder).To(BeAssignableToTypeOf(TxtToImageConverter{}))
+		})
+
+		It("should return an ImageDecoder for unknown types", func() {
+			decoder := ForType("unknown", nil)
+			Expect(decoder).To(BeAssignableToTypeOf(ImageDecoder{}))
+		})
+	})
+})

Server/services/thumbnails/pkg/preprocessor/preprocessor_vips.go

@@ -0,0 +1,20 @@
+//go:build enable_vips
+
+package preprocessor
+
+import (
+	"io"
+
+	"github.com/davidbyttow/govips/v2/vips"
+)
+
+func init() {
+	vips.LoggingSettings(nil, vips.LogLevelError)
+}
+
+type ImageDecoder struct{}
+
+func (v ImageDecoder) Convert(r io.Reader) (interface{}, error) {
+	img, err := vips.NewImageFromReader(r)
+	return img, err
+}

Server/services/thumbnails/pkg/preprocessor/textanalyzer.go

@@ -0,0 +1,309 @@
+package preprocessor
+
+import (
+	"unicode"
+	"unicode/utf8"
+)
+
+// Default list of scripts to be analyzed within the string.
+//
+// Scripts that aren't present in the list will be considered as part
+// of the last "known" script. For example, if "Avestan" script (which isn't
+// present) is preceeded by "Arabic" script, then the "Avestan" script will
+// be considered as "Arabic"
+//
+// Punctuation symbols are usually considered part of the "Common" script
+var DefaultScripts = []string{
+	"Arabic",
+	"Common",
+	"Devanagari",
+	"Han",
+	"Hangul",
+	"Hiragana",
+	"Inherited",
+	"Katakana",
+	"Latin",
+}
+
+// Convenient map[string]map[string]string type used to merge multiple
+// scripts into one. This is mainly used for japanese language which uses
+// "Han", "Hiragana" and "Katakana" scripts.
+//
+// The map contains the expected previous script as first key, the expected
+// current script as second key, and the resulting script (if both keys
+// match) as value
+type MergeMap map[string]map[string]string
+
+// The default mergeMap containing info for the japanese scripts
+var DefaultMergeMap = MergeMap{
+	"Han": map[string]string{
+		"Hiragana": "Hiragana",
+		"Katakana": "Katakana",
+	},
+	"Hiragana": map[string]string{
+		"Han":      "Hiragana",
+		"Katakana": "Hiragana",
+	},
+	"Katakana": map[string]string{
+		"Han":      "Katakana",
+		"Hiragana": "Hiragana",
+	},
+}
+
+// Analysis options.
+type AnalysisOpts struct {
+	UseMergeMap bool
+	MergeMap    MergeMap
+}
+
+// A script range. The range should be attached to a string which could contain
+// multiple scripts. The "TargetScript" will go from bytes "Low" to "High"
+// (both inclusive), and contains a "RuneCount" number of runes or chars
+// (mostly for debugging purposes).
+// The Space contains the bytes (inside the range) that are considered as
+// white space.
+type ScriptRange struct {
+	Low, High    int
+	Spaces       []int
+	TargetScript string
+	RuneCount    int
+}
+
+// The result of a text analysis. It contains the analyzed text, a list of
+// script ranges (see the ScriptRange type) and a map containing how many
+// runes have been detected for a particular script.
+type TextAnalysis struct {
+	ScriptRanges []ScriptRange
+	RuneCount    map[string]int
+	Text         string
+}
+
+// The TextAnalyzer object contains private members. It should be created via
+// "NewTextAnalyzer" function.
+type TextAnalyzer struct {
+	scripts         map[string]*unicode.RangeTable
+	scriptListCache []string
+}
+
+// Create a new TextAnalyzer. A list of scripts must be provided.
+// You can use the "DefaultScripts" variable for a default list,
+// although it doesn't contain all the available scripts.
+// See the unicode.Scripts variable (in the unicode package) for a
+// full list. Note that using invalid scripts will cause an undefined
+// behavior
+func NewTextAnalyzer(scriptList []string) TextAnalyzer {
+	scriptRanges := make(map[string]*unicode.RangeTable, len(scriptList))
+	for _, script := range scriptList {
+		scriptRanges[script] = unicode.Scripts[script]
+	}
+	return TextAnalyzer{
+		scripts:         scriptRanges,
+		scriptListCache: scriptList,
+	}
+}
+
+// Analyze the target string using the specified options.
+// A TextAnalysis will be returned with the result of the analysis.
+func (ta *TextAnalyzer) AnalyzeString(word string, opts AnalysisOpts) TextAnalysis {
+	analysis := TextAnalysis{
+		ScriptRanges: []ScriptRange{},
+		RuneCount:    make(map[string]int),
+		Text:         word,
+	}
+
+	if len(word) < 1 {
+		return analysis
+	}
+
+	firstRune, runeLen := utf8.DecodeRuneInString(word)
+
+	lastRange := &ScriptRange{
+		Low:          0,
+		Spaces:       make([]int, 0),
+		TargetScript: ta.chooseScriptFor(firstRune),
+	}
+	firstRuneIsWhiteSpace := unicode.Is(unicode.White_Space, firstRune)
+	if firstRuneIsWhiteSpace {
+		lastRange.Spaces = append(lastRange.Spaces, 0)
+	}
+
+	runeCount := 1
+	for wordIndex, char := range word[runeLen:] {
+		wordIndex += runeLen // shifted from the original string
+		script := ta.chooseScriptFor(char)
+
+		isWhiteSpace := unicode.Is(unicode.White_Space, char)
+		if script != lastRange.TargetScript {
+			if mapScript, isOk := ta.getMergeMapValue(opts, lastRange.TargetScript, script); isOk {
+				lastRange.TargetScript = mapScript
+				if isWhiteSpace {
+					// TODO: Check if this is dead code.
+					// whitespace should be part of the "Common" script, and the Common
+					// script shouldn't be part of a mergeMap
+					lastRange.Spaces = append(lastRange.Spaces, wordIndex)
+				}
+				runeCount++
+				continue
+			}
+
+			lastRange.High = wordIndex - 1
+			lastRange.RuneCount = runeCount
+			analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
+			if _, exists := analysis.RuneCount[lastRange.TargetScript]; !exists {
+				analysis.RuneCount[lastRange.TargetScript] = 0
+			}
+			analysis.RuneCount[lastRange.TargetScript] += runeCount
+			lastRange = &ScriptRange{
+				Low:          wordIndex,
+				Spaces:       make([]int, 0),
+				TargetScript: script,
+			}
+			runeCount = 0
+		}
+		runeCount++
+		if isWhiteSpace {
+			lastRange.Spaces = append(lastRange.Spaces, wordIndex)
+		}
+	}
+
+	// close the last range
+	lastRange.High = len(word) - 1
+	lastRange.RuneCount = runeCount
+	analysis.RuneCount[lastRange.TargetScript] += runeCount
+	analysis.ScriptRanges = append(analysis.ScriptRanges, *lastRange)
+
+	return analysis
+}
+
+func (ta *TextAnalyzer) chooseScriptFor(char rune) string {
+	script := "_unknown"
+	for scriptIndex, scriptFound := range ta.scriptListCache {
+		// if we can't match with a known script, do nothing and jump to the next char
+		if unicode.Is(ta.scripts[scriptFound], char) {
+			if scriptIndex > 3 {
+				// we might expect more chars with the same script
+				// so move the script first to match it faster next time
+				ta.reorderScriptList(scriptFound)
+			}
+			return scriptFound
+		}
+	}
+	return script
+}
+
+// Reorder the scriptListCache in the TextAnalyzer in order to speed up
+// the next script searches. A "Latin" script is expected to be surrounded
+// by "Latin" chars, although "Common" script chars might be present too
+func (ta *TextAnalyzer) reorderScriptList(matchedScript string) {
+	for index, script := range ta.scriptListCache {
+		if script == matchedScript {
+			if index != 0 {
+				// move the script to the first position for a faster matching
+				newList := append([]string{script}, ta.scriptListCache[:index]...)
+				ta.scriptListCache = append(newList, ta.scriptListCache[index+1:]...)
+			}
+			// if index == 0 there is nothing to do: the element is already the first
+			break
+		}
+	}
+}
+
+// Get the value from the merge map based on the previous and current scripts.
+// The information about using the merge map and the actual merge map will be
+// gotten from the AnalysisOpts passed as parameter
+func (ta *TextAnalyzer) getMergeMapValue(opts AnalysisOpts, previous, current string) (string, bool) {
+	if opts.UseMergeMap {
+		// This option mainly target japanese chars; multiple scripts can be used
+		// in the same piece of text (Han, Hiragana and Katakana)
+		// Instead of starting a new range, adjust the target script of the last range
+		if expCurrent, currentOk := opts.MergeMap[previous]; currentOk {
+			if expFinal, finalOk := expCurrent[current]; finalOk {
+				return expFinal, finalOk
+			}
+		}
+	}
+	return "", false
+}
+
+// Change the "Common" script to the one used in the previous script range.
+// The ranges will be readjusted and merged if they're adjacent.
+// This naive approach should be good enough for normal use cases
+//
+// The MergeMap is needed in case of the japanese language: the ranges
+// "Han"-"Common"-"Katakana" might be replaced to "Han"-"Hiragana"-"Katakana"
+// However, the ranges should be merged together into a big "Hiragana" range.
+// If the MergeMap isn't needed, use an empty one
+func (tr *TextAnalysis) MergeCommon(mergeMap MergeMap) {
+	var finalRanges []ScriptRange
+
+	if len(tr.ScriptRanges) < 1 {
+		// no ranges -> nothing to do
+		return
+	}
+
+	previousRange := &ScriptRange{}
+	*previousRange = tr.ScriptRanges[0]
+	for _, sRange := range tr.ScriptRanges[1:] {
+		if previousRange.TargetScript == sRange.TargetScript {
+			previousRange.High = sRange.High
+			previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
+			previousRange.RuneCount += sRange.RuneCount
+		} else if sRange.TargetScript == "Common" || sRange.TargetScript == "Inherited" {
+			// new range will be absorbed into the previous one
+			previousRange.High = sRange.High
+			previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
+			previousRange.RuneCount += sRange.RuneCount
+			tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
+			tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
+		} else if previousRange.TargetScript == "Common" || previousRange.TargetScript == "Inherited" {
+			// might happen if the text starts with a Common script
+			previousRange.High = sRange.High
+			previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
+			tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
+			tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
+			previousRange.RuneCount += sRange.RuneCount
+			previousRange.TargetScript = sRange.TargetScript
+		} else {
+			if mapScript, isOk := tr.getMergeMapValue(mergeMap, previousRange.TargetScript, sRange.TargetScript); isOk {
+				if sRange.TargetScript == mapScript {
+					// the previous range has changed the target script
+					tr.RuneCount[previousRange.TargetScript] -= previousRange.RuneCount
+					tr.RuneCount[sRange.TargetScript] += previousRange.RuneCount
+				} else {
+					// new range has been absorbed
+					tr.RuneCount[sRange.TargetScript] -= sRange.RuneCount
+					tr.RuneCount[previousRange.TargetScript] += sRange.RuneCount
+				}
+				previousRange.TargetScript = mapScript
+				previousRange.High = sRange.High
+				previousRange.Spaces = append(previousRange.Spaces, sRange.Spaces...)
+				previousRange.RuneCount += sRange.RuneCount
+				continue
+			}
+			finalRanges = append(finalRanges, *previousRange)
+			*previousRange = sRange
+		}
+	}
+
+	finalRanges = append(finalRanges, *previousRange)
+	tr.ScriptRanges = finalRanges
+	delete(tr.RuneCount, "Common")
+	delete(tr.RuneCount, "Inherited")
+	for index, rCount := range tr.RuneCount {
+		if rCount == 0 {
+			delete(tr.RuneCount, index)
+		}
+	}
+}
+
+func (tr *TextAnalysis) getMergeMapValue(mMap MergeMap, previous, current string) (string, bool) {
+	// This option mainly target japanese chars; multiple scripts can be used
+	// in the same piece of text (Han, Hiragana and Katakana)
+	// Instead of starting a new range, adjust the target script of the last range
+	if expCurrent, currentOk := mMap[previous]; currentOk {
+		if expFinal, finalOk := expCurrent[current]; finalOk {
+			return expFinal, finalOk
+		}
+	}
+	return "", false
+}

Server/services/thumbnails/pkg/preprocessor/textanalyzer_test.go

@@ -0,0 +1,614 @@
+package preprocessor
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	inputs = [18]string{
+		"basic latin",
+		"trailing tab	",
+		"Small text. \"$\", \"£\" and \"¥\" are currencies.",
+		"latin with 🖖",
+		"기본 한국어",
+		"基本的な日本語",
+		"ウーロン茶",
+		"私はエンジニアです",
+		"ティー私はエンジニアです",
+		"私はエンジニアです ティー",
+		"आधारभूत देवनागरी",
+		"mixed 언어 传入 🚀!",
+		"/k͜p/",
+		// ä and a + ¨
+		"ä ä",
+		"базовый русский", // cyrillic script isn't part of our default
+		"latin русский",   // latin + cyrillic (cyrillic not supported)
+		" space justified ",
+		"",
+	}
+)
+
+func TestAnalyzeString(t *testing.T) {
+	defaultOpts := AnalysisOpts{
+		UseMergeMap: true,
+		MergeMap:    DefaultMergeMap,
+	}
+
+	tables := []struct {
+		input string
+		opts  AnalysisOpts
+		eOut  TextAnalysis
+	}{
+		{
+			input: inputs[0],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 10, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 11},
+				},
+				RuneCount: map[string]int{
+					"Latin": 11,
+				},
+				Text: inputs[0],
+			},
+		},
+		{
+			input: inputs[1],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 12, Spaces: []int{8, 12}, TargetScript: "Latin", RuneCount: 13},
+				},
+				RuneCount: map[string]int{
+					"Latin": 13,
+				},
+				Text: inputs[1],
+			},
+		},
+		{
+			input: inputs[2],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 45, Spaces: []int{5, 11, 16, 21, 25, 30, 34}, TargetScript: "Latin", RuneCount: 44},
+				},
+				RuneCount: map[string]int{
+					"Latin": 44,
+				},
+				Text: inputs[2],
+			},
+		},
+		{
+			input: inputs[3],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 14, Spaces: []int{5, 10}, TargetScript: "Latin", RuneCount: 12},
+				},
+				RuneCount: map[string]int{
+					"Latin": 12,
+				},
+				Text: inputs[3],
+			},
+		},
+		{
+			input: inputs[4],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 15, Spaces: []int{6}, TargetScript: "Hangul", RuneCount: 6},
+				},
+				RuneCount: map[string]int{
+					"Hangul": 6,
+				},
+				Text: inputs[4],
+			},
+		},
+		{
+			input: inputs[5],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 7},
+				},
+				RuneCount: map[string]int{
+					"Hiragana": 7,
+				},
+				Text: inputs[5],
+			},
+		},
+		{
+			input: inputs[6],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 14, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
+				},
+				RuneCount: map[string]int{
+					"Katakana": 5,
+				},
+				Text: inputs[6],
+			},
+		},
+		{
+			input: inputs[7],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 9},
+				},
+				RuneCount: map[string]int{
+					"Hiragana": 9,
+				},
+				Text: inputs[7],
+			},
+		},
+		{
+			input: inputs[8],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 12},
+				},
+				RuneCount: map[string]int{
+					"Hiragana": 12,
+				},
+				Text: inputs[8],
+			},
+		},
+		{
+			input: inputs[9],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 36, Spaces: []int{27}, TargetScript: "Hiragana", RuneCount: 13},
+				},
+				RuneCount: map[string]int{
+					"Hiragana": 13,
+				},
+				Text: inputs[9],
+			},
+		},
+		{
+			input: inputs[10],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 45, Spaces: []int{21}, TargetScript: "Devanagari", RuneCount: 16},
+				},
+				RuneCount: map[string]int{
+					"Devanagari": 16,
+				},
+				Text: inputs[10],
+			},
+		},
+		{
+			input: inputs[11],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
+					{Low: 6, High: 12, Spaces: []int{12}, TargetScript: "Hangul", RuneCount: 3},
+					{Low: 13, High: 24, Spaces: []int{19}, TargetScript: "Han", RuneCount: 5}, // 🚀 and ! are "Common" script and will be merged with "Han"
+				},
+				RuneCount: map[string]int{
+					"Latin":  6,
+					"Hangul": 3,
+					"Han":    5,
+				},
+				Text: inputs[11],
+			},
+		},
+		{
+			input: inputs[12],
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+				},
+				RuneCount: map[string]int{
+					"Latin": 5,
+				},
+				Text: inputs[12],
+			},
+		},
+		{
+			input: inputs[13], // ä and a + ¨
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{2}, TargetScript: "Latin", RuneCount: 4},
+				},
+				RuneCount: map[string]int{
+					"Latin": 4,
+				},
+				Text: inputs[13],
+			},
+		},
+		{
+			input: inputs[14], // cyrillic script isn't part of our default
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 28, Spaces: []int{14}, TargetScript: "_unknown", RuneCount: 15},
+				},
+				RuneCount: map[string]int{
+					"_unknown": 15,
+				},
+				Text: inputs[14],
+			},
+		},
+		{
+			input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{5}, TargetScript: "Latin", RuneCount: 6},
+					{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
+				},
+				RuneCount: map[string]int{
+					"Latin":    6,
+					"_unknown": 7,
+				},
+				Text: inputs[15],
+			},
+		},
+		{
+			input: inputs[16], // latin + cyrillic (cyrillic script isn't part of our default)
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 16, Spaces: []int{0, 6, 16}, TargetScript: "Latin", RuneCount: 17},
+				},
+				RuneCount: map[string]int{
+					"Latin": 17,
+				},
+				Text: inputs[16],
+			},
+		},
+		{
+			input: inputs[17], // latin + cyrillic (cyrillic script isn't part of our default)
+			opts:  defaultOpts,
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{},
+				RuneCount:    map[string]int{},
+				Text:         inputs[17],
+			},
+		},
+	}
+
+	for _, table := range tables {
+		testname := fmt.Sprintf("Analyzing \"%s\" string", table.input)
+		t.Run(testname, func(t *testing.T) {
+			ta := NewTextAnalyzer(DefaultScripts)
+			result := ta.AnalyzeString(table.input, table.opts)
+			if table.opts.UseMergeMap {
+				result.MergeCommon(table.opts.MergeMap)
+			} else {
+				result.MergeCommon(MergeMap{})
+			}
+			assert.Equal(t, table.eOut, result)
+		})
+	}
+}
+
+func TestAnalyzeStringRaw(t *testing.T) {
+	tables := []struct {
+		input string
+		eOut  TextAnalysis
+	}{
+		{
+			input: inputs[0],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
+					{Low: 6, High: 10, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+				},
+				RuneCount: map[string]int{
+					"Latin":  10,
+					"Common": 1,
+				},
+				Text: inputs[0],
+			},
+		},
+		{
+			input: inputs[1],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 7, Spaces: []int{}, TargetScript: "Latin", RuneCount: 8},
+					{Low: 8, High: 8, Spaces: []int{8}, TargetScript: "Common", RuneCount: 1},
+					{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
+					{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Latin":  11,
+					"Common": 2,
+				},
+				Text: inputs[1],
+			},
+		},
+		{
+			input: inputs[2],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
+					{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
+					{Low: 10, High: 21, Spaces: []int{11, 16, 21}, TargetScript: "Common", RuneCount: 11}, // £ takes 2 bytes
+					{Low: 22, High: 24, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
+					{Low: 25, High: 30, Spaces: []int{25, 30}, TargetScript: "Common", RuneCount: 5}, // ¥ takes 2 bytes
+					{Low: 31, High: 33, Spaces: []int{}, TargetScript: "Latin", RuneCount: 3},
+					{Low: 34, High: 34, Spaces: []int{34}, TargetScript: "Common", RuneCount: 1},
+					{Low: 35, High: 44, Spaces: []int{}, TargetScript: "Latin", RuneCount: 10},
+					{Low: 45, High: 45, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Latin":  25,
+					"Common": 19,
+				},
+				Text: inputs[2],
+			},
+		},
+		{
+			input: inputs[3],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
+					{Low: 6, High: 9, Spaces: []int{}, TargetScript: "Latin", RuneCount: 4},
+					{Low: 10, High: 14, Spaces: []int{10}, TargetScript: "Common", RuneCount: 2},
+				},
+				RuneCount: map[string]int{
+					"Latin":  9,
+					"Common": 3,
+				},
+				Text: inputs[3],
+			},
+		},
+		{
+			input: inputs[4],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
+					{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
+					{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 3},
+				},
+				RuneCount: map[string]int{
+					"Hangul": 5,
+					"Common": 1,
+				},
+				Text: inputs[4],
+			},
+		},
+		{
+			input: inputs[5],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 8, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
+					{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
+					{Low: 12, High: 20, Spaces: []int{}, TargetScript: "Han", RuneCount: 3},
+				},
+				RuneCount: map[string]int{
+					"Hiragana": 1,
+					"Han":      6,
+				},
+				Text: inputs[5],
+			},
+		},
+		{
+			input: inputs[6],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 1},
+					{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1}, // ー U+30FC (KATAKANA-HIRAGANA PROLONGED SOUND MARK) seems to be counted as Common
+					{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
+					{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Katakana": 3,
+					"Common":   1,
+					"Han":      1,
+				},
+				Text: inputs[6],
+			},
+		},
+		{
+			input: inputs[7],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
+					{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
+					{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
+					{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
+				},
+				RuneCount: map[string]int{
+					"Han":      1,
+					"Hiragana": 3,
+					"Katakana": 5,
+				},
+				Text: inputs[7],
+			},
+		},
+		{
+			input: inputs[8],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 5, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
+					{Low: 6, High: 8, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
+					{Low: 9, High: 11, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
+					{Low: 12, High: 14, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
+					{Low: 15, High: 29, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
+					{Low: 30, High: 35, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
+				},
+				RuneCount: map[string]int{
+					"Han":      1,
+					"Hiragana": 3,
+					"Katakana": 7,
+					"Common":   1,
+				},
+				Text: inputs[8],
+			},
+		},
+		{
+			input: inputs[9],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 2, Spaces: []int{}, TargetScript: "Han", RuneCount: 1},
+					{Low: 3, High: 5, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 1},
+					{Low: 6, High: 20, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 5},
+					{Low: 21, High: 26, Spaces: []int{}, TargetScript: "Hiragana", RuneCount: 2},
+					{Low: 27, High: 27, Spaces: []int{27}, TargetScript: "Common", RuneCount: 1},
+					{Low: 28, High: 33, Spaces: []int{}, TargetScript: "Katakana", RuneCount: 2},
+					{Low: 34, High: 36, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Han":      1,
+					"Hiragana": 3,
+					"Katakana": 7,
+					"Common":   2,
+				},
+				Text: inputs[9],
+			},
+		},
+		{
+			input: inputs[10],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 20, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 7},
+					{Low: 21, High: 21, Spaces: []int{21}, TargetScript: "Common", RuneCount: 1},
+					{Low: 22, High: 45, Spaces: []int{}, TargetScript: "Devanagari", RuneCount: 8},
+				},
+				RuneCount: map[string]int{
+					"Devanagari": 15,
+					"Common":     1,
+				},
+				Text: inputs[10],
+			},
+		},
+		{
+			input: inputs[11],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
+					{Low: 6, High: 11, Spaces: []int{}, TargetScript: "Hangul", RuneCount: 2},
+					{Low: 12, High: 12, Spaces: []int{12}, TargetScript: "Common", RuneCount: 1},
+					{Low: 13, High: 18, Spaces: []int{}, TargetScript: "Han", RuneCount: 2},
+					{Low: 19, High: 24, Spaces: []int{19}, TargetScript: "Common", RuneCount: 3},
+				},
+				RuneCount: map[string]int{
+					"Latin":  5,
+					"Hangul": 2,
+					"Han":    2,
+					"Common": 5,
+				},
+				Text: inputs[11],
+			},
+		},
+		{
+			input: inputs[12],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 0, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
+					{Low: 1, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
+					{Low: 2, High: 3, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
+					{Low: 4, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
+					{Low: 5, High: 5, Spaces: []int{}, TargetScript: "Common", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Latin":     2,
+					"Common":    2,
+					"Inherited": 1,
+				},
+				Text: inputs[12],
+			},
+		},
+		{
+			input: inputs[13], // ä and a + ¨
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 1, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
+					{Low: 2, High: 2, Spaces: []int{2}, TargetScript: "Common", RuneCount: 1},
+					{Low: 3, High: 3, Spaces: []int{}, TargetScript: "Latin", RuneCount: 1},
+					{Low: 4, High: 5, Spaces: []int{}, TargetScript: "Inherited", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Latin":     2,
+					"Common":    1,
+					"Inherited": 1,
+				},
+				Text: inputs[13],
+			},
+		},
+		{
+			input: inputs[14], // cyrillic script isn't part of our default
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 13, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
+					{Low: 14, High: 14, Spaces: []int{14}, TargetScript: "Common", RuneCount: 1},
+					{Low: 15, High: 28, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
+				},
+				RuneCount: map[string]int{
+					"_unknown": 14,
+					"Common":   1,
+				},
+				Text: inputs[14],
+			},
+		},
+		{
+			input: inputs[15], // latin + cyrillic (cyrillic script isn't part of our default)
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 4, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 5, High: 5, Spaces: []int{5}, TargetScript: "Common", RuneCount: 1},
+					{Low: 6, High: 19, Spaces: []int{}, TargetScript: "_unknown", RuneCount: 7},
+				},
+				RuneCount: map[string]int{
+					"Latin":    5,
+					"Common":   1,
+					"_unknown": 7,
+				},
+				Text: inputs[15],
+			},
+		},
+		{
+			input: inputs[16],
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{
+					{Low: 0, High: 0, Spaces: []int{0}, TargetScript: "Common", RuneCount: 1},
+					{Low: 1, High: 5, Spaces: []int{}, TargetScript: "Latin", RuneCount: 5},
+					{Low: 6, High: 6, Spaces: []int{6}, TargetScript: "Common", RuneCount: 1},
+					{Low: 7, High: 15, Spaces: []int{}, TargetScript: "Latin", RuneCount: 9},
+					{Low: 16, High: 16, Spaces: []int{16}, TargetScript: "Common", RuneCount: 1},
+				},
+				RuneCount: map[string]int{
+					"Latin":  14,
+					"Common": 3,
+				},
+				Text: inputs[16],
+			},
+		},
+		{
+			input: inputs[17], // empty string
+			eOut: TextAnalysis{
+				ScriptRanges: []ScriptRange{},
+				RuneCount:    map[string]int{},
+				Text:         inputs[17],
+			},
+		},
+	}
+
+	for _, table := range tables {
+		testname := fmt.Sprintf("Raw-Analyzing \"%s\" string", table.input)
+		t.Run(testname, func(t *testing.T) {
+			ta := NewTextAnalyzer(DefaultScripts)
+			result := ta.AnalyzeString(table.input, AnalysisOpts{})
+
+			assert.Equal(t, table.eOut, result)
+		})
+	}
+}