Initial QSfera import

Server/services/collaboration/pkg/connector/utf7/utf7.go

@@ -0,0 +1,308 @@
+package utf7
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"strings"
+	"unicode"
+	"unicode/utf16"
+)
+
+const (
+	rangeASCII = "ascii"
+	rangeUTF7  = "utf7"
+)
+
+// Range represents a range with a lower and upper bounds. The range has a
+// name for easier identification
+type Range struct {
+	Name string
+	Low  int
+	High int
+}
+
+// Range table for ASCII chars belonging to the "direct character" group
+// for UTF-7
+var utf7AsciiRT = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x27, 0x29, 1}, // '()
+		{0x2c, 0x2f, 1}, // ,-./
+		{0x30, 0x39, 1}, // 0-9
+		{0x3a, 0x3f, 5}, // :?
+		{0x41, 0x5a, 1}, // A-Z
+		{0x61, 0x7a, 1}, // a-z
+	},
+}
+
+// EncodeString will encode the provided UTF-8 string into UTF-7 format
+//
+// The encoding process will have the following peculiarities
+// * Any char outside the "direct characters" will be encoded. This means that
+// only "a-z", "A-Z", "0-9" and "'(),-.:?" chars will remain intact while the
+// rest will be encoded. "Optional direct chars" (such as the space) will
+// be encoded.
+// * The "+" char will be encoded as any other character, so the result will
+// be "+ACs-", not "+-"
+// * Sequences of chars will be encoded as a single group. For example,
+// "こんにちは" will be encoded as "+MFMwkzBrMGEwbw-"
+// * All encoded sequences will be enclosed between "+" and "-"
+func EncodeString(s string) string {
+	runes := []rune(s)
+
+	ranges := analyzeRunes(runes)
+
+	var sb strings.Builder
+	// doubling the number of bytes of the string is usually enough
+	sb.Grow(len(s) * 2)
+
+	for _, v := range ranges {
+		if v.Name == rangeASCII {
+			for _, v := range runes[v.Low:v.High] {
+				sb.WriteRune(v)
+			}
+		} else {
+			utf7Bytes := convertToUtf7(runes[v.Low:v.High])
+			sb.Write(utf7Bytes)
+		}
+	}
+	return sb.String()
+}
+
+// DecodeString will decode the provided UTF-7 string into UTF-8.
+//
+// Any valid UTF-7 string can be decoded, not just the ones returned by
+// the EncodeString function.
+// In particular, UTF-7 strings such as "a+-b" or "a+AD0.b" can be decoded
+// even if the EncodeString function won't generate the corresponding
+// strings that way.
+//
+// Note that this function requires the string to contain only ASCII chars
+// (as per UTF-7), otherwise an error will be returned.
+// Illegal char sequences in the encoded parts of the string will also trigger
+// errors.
+func DecodeString(s string) (string, error) {
+	byteArray := []byte(s)
+
+	ranges, err := analyzeUtf7(byteArray)
+	if err != nil {
+		return "", err
+	}
+
+	var sb strings.Builder
+	sb.Grow(len(byteArray))
+
+	for _, v := range ranges {
+		if v.Name == rangeASCII {
+			// if it's an ascii range, just copy it
+			sb.Write(byteArray[v.Low:v.High])
+		} else {
+			// utf7 range
+			utf7ByteRange := byteArray[v.Low:v.High]
+			if err := convertRangeFromUtf7(utf7ByteRange, &sb); err != nil {
+				return "", err
+			}
+		}
+	}
+	return sb.String(), nil
+}
+
+// analyzeRunes will analyze the array of runes and provide a list of ranges.
+// Each range will be defined by a name and a low and high index. For example,
+// an "ascii" range could go from index 0 to 12 and "utf7" range from 12 to 25.
+// The range includes the low index but not the high "[0,12)". This means it
+// be easily extracted with something like "runes[r.Low:r.High]".
+//
+// The list of ranges will only include the following names:
+// * "ascii" for runes belonging to the "direct characters" group of UTF-7
+// (those that can be used directly without encoding them). Note that
+// it won't consider every ASCII character.
+// * "utf7" for runes that should be encoded for UTF-7.
+//
+// As said, runes in the ranges marked as "utf7" should be encoded for UTF-7,
+// while the others can be used without changes.
+//
+// This method is intended to be used to detect which ranges need to be
+// encoded to UTF-7
+func analyzeRunes(runes []rune) []Range {
+	ranges := make([]Range, 0)
+
+	var currentRange Range
+	for k, v := range runes {
+		if unicode.Is(utf7AsciiRT, v) {
+			if currentRange.Name == "" {
+				// take control of the current range
+				currentRange.Name = rangeASCII
+				currentRange.Low = k
+			} else if currentRange.Name != rangeASCII {
+				// close current range and open a new one
+				currentRange.High = k
+				ranges = append(ranges, currentRange)
+				currentRange = Range{
+					Name: rangeASCII,
+					Low:  k,
+				}
+			}
+		} else {
+			if currentRange.Name == "" {
+				// take control of the current range
+				currentRange.Name = rangeUTF7
+				currentRange.Low = k
+			} else if currentRange.Name != rangeUTF7 {
+				// close current range and open a new one
+				currentRange.High = k
+				ranges = append(ranges, currentRange)
+				currentRange = Range{
+					Name: rangeUTF7,
+					Low:  k,
+				}
+			}
+		}
+	}
+	// close the last range
+	currentRange.High = len(runes)
+	ranges = append(ranges, currentRange)
+
+	return ranges
+}
+
+// analyzeUtf7 will analyze the provided byte sequence and return a list of
+// ranges.
+// The byte sequence is considered as UTF-7, so if there is a non-ASCII char
+// in the sequence, an error will be returned (it isn't a valid UTF-7 string).
+//
+// Each returned range will have either "ascii" or "utf7" as name for the range.
+// "ascii" ranges won't require any change and can be used directly. "utf7"
+// ranges are encoded in UTF-7 and will require decoding.
+//
+// This method is intended to be used to detect which ranges need to be
+// decoded from UTF-7
+func analyzeUtf7(byteArray []byte) ([]Range, error) {
+	base64chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+	base64ByteArray := []byte(base64chars)
+
+	ranges := make([]Range, 0)
+
+	currentRange := Range{
+		Name: rangeASCII,
+		Low:  0,
+	}
+
+	for k, v := range byteArray {
+		if v > unicode.MaxASCII {
+			return nil, errors.New("Byte sequence contains a non-ASCII char")
+		}
+
+		if v == '+' && currentRange.Name != rangeUTF7 {
+			// start utf7-encoded range
+			currentRange.High = k
+			ranges = append(ranges, currentRange)
+			currentRange = Range{
+				Name: rangeUTF7,
+				Low:  k,
+			}
+		} else if v == '-' {
+			// close utf7-encoded range
+			currentRange.High = k + 1 // the '-' char is part of the range
+			ranges = append(ranges, currentRange)
+			currentRange = Range{
+				Name: rangeASCII,
+				Low:  k + 1,
+			}
+		} else if bytes.IndexByte(base64ByteArray, v) == -1 && currentRange.Name == rangeUTF7 {
+			// found invalid base64 char, so need to close the utf7 range
+			currentRange.High = k
+			ranges = append(ranges, currentRange)
+			currentRange = Range{
+				Name: rangeASCII,
+				Low:  k,
+			}
+		}
+	}
+
+	// close the last range
+	currentRange.High = len(byteArray)
+	ranges = append(ranges, currentRange)
+
+	// there might be empty ranges we need to clear
+	// empty ranges have Low = High
+	realRanges := make([]Range, 0, len(ranges))
+	for _, v := range ranges {
+		if v.Low != v.High {
+			realRanges = append(realRanges, v)
+		}
+	}
+
+	return realRanges, nil
+}
+
+// convertToUtf7 will convert the provided runes to a UTF-7 sequence of bytes.
+// The function assumes that all the provided runes must be converted to UTF-7
+func convertToUtf7(runes []rune) []byte {
+	byteArray := make([]byte, 0, len(runes)*2)
+
+	u16 := utf16.Encode(runes)
+	for _, v := range u16 {
+		byteArray = binary.BigEndian.AppendUint16(byteArray, v)
+	}
+
+	dst := make([]byte, base64.RawStdEncoding.EncodedLen(len(byteArray))+2)
+	dst[0] = '+'
+	base64.RawStdEncoding.Encode(dst[1:len(dst)-1], byteArray)
+	dst[len(dst)-1] = '-'
+	return dst
+}
+
+// convertRangeFromUtf7 will convert an utf7 byte range (enclosed in
+// the "+" and "-" chars) and write the result in the provided string builder.
+// The string builder won't be modified other than to append the result.
+// An error might be returned if there is any problem with the conversion.
+func convertRangeFromUtf7(utf7ByteRange []byte, sb *strings.Builder) error {
+	if len(utf7ByteRange) == 2 && utf7ByteRange[0] == '+' && utf7ByteRange[1] == '-' {
+		// special case for the "+-" sequence -> just write "+" as replacement
+		sb.WriteByte('+')
+	} else {
+		// utf7 range must start with "+" and should (but might not) end with "-"
+		// we need to remove those chars before decoding
+		toDecode := utf7ByteRange[1 : len(utf7ByteRange)-1]
+		if utf7ByteRange[len(utf7ByteRange)-1] != '-' {
+			toDecode = utf7ByteRange[1:]
+		}
+		runeArray, err := convertFromUtf7(toDecode)
+		if err != nil {
+			return err
+		}
+		for _, r := range runeArray {
+			sb.WriteRune(r)
+		}
+	}
+	return nil
+}
+
+// convertFromUtf7 will convert the sequence of bytes to runes. The sequence
+// of bytes is assumed to be an UTF-7 encoded sequence (without the "+" and
+// "-" limiters)
+// The returned runes should be UTF-8 encoded and can be converted to a
+// regular string easily.
+// Note that errors can be returned if the decoding process fails
+func convertFromUtf7(byteArray []byte) ([]rune, error) {
+	dst := make([]byte, base64.RawStdEncoding.DecodedLen(len(byteArray)))
+
+	_, err := base64.RawStdEncoding.Decode(dst, byteArray)
+	if err != nil {
+		return []rune{}, err
+	}
+
+	if len(dst)%2 != 0 {
+		// some data can't be represented as utf16, and can't be decoded
+		return []rune{}, errors.New("some utf7 data can't be represented as utf16")
+	}
+
+	u16array := make([]uint16, 0, len(dst)/2)
+	for i := 0; i < len(dst); i++ {
+		u16array = append(u16array, binary.BigEndian.Uint16(dst[i:i+2]))
+		i = i + 1
+	}
+	return utf16.Decode(u16array), nil
+}

Server/services/collaboration/pkg/connector/utf7/utf7_suite_test.go

@@ -0,0 +1,13 @@
+package utf7_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestUtf7(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Utf7 Suite")
+}

Server/services/collaboration/pkg/connector/utf7/utf7_test.go

@@ -0,0 +1,67 @@
+package utf7_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/qsfera/server/services/collaboration/pkg/connector/utf7"
+)
+
+var _ = Describe("Utf7", func() {
+	DescribeTable(
+		"Encode",
+		func(input, output string) {
+			Expect(utf7.EncodeString(input)).To(Equal(output))
+		},
+		Entry("regular filename", "private.txt", "private.txt"),
+		Entry("direct chars", "is-better?yes:no(3).pdf", "is-better?yes:no(3).pdf"),
+		Entry("with spaces", "a big file with spaces.docx", "a+ACA-big+ACA-file+ACA-with+ACA-spaces.docx"),
+		Entry("with symbols", "a+b=c", "a+ACs-b+AD0-c"),
+		Entry("unicode filenames", "超極秘文書.doc", "+jYVpdXnYZYdm+A-.doc"),
+		Entry("emoji and symbols", "💰🔜™️.pdf", "+2D3csNg93RwhIv4P-.pdf"),
+	)
+
+	DescribeTable(
+		"Decode",
+		func(input, output string) {
+			Expect(utf7.DecodeString(input)).To(Equal(output))
+		},
+		Entry("regular filename", "private.txt", "private.txt"),
+		Entry("direct chars", "is-better?yes:no(3).pdf", "is-better?yes:no(3).pdf"),
+		Entry("with spaces", "a+ACA-big+ACA-file+ACA-with+ACA-spaces.docx", "a big file with spaces.docx"),
+		Entry("with symbols", "a+ACs-b+AD0-c", "a+b=c"),
+		Entry("unicode filenames", "+jYVpdXnYZYdm+A-.doc", "超極秘文書.doc"),
+		Entry("emoji and symbols", "+2D3csNg93RwhIv4P-.pdf", "💰🔜™️.pdf"),
+		Entry("special case +- chars", "a+-b+AD0-c", "a+b=c"),
+		Entry("optional direct chars", "1 +- 1 +AD0- 2", "1 + 1 = 2"),
+		Entry("missing - char", "+jYVpdXnYZYdm+A.doc", "超極秘文書.doc"),
+		Entry("missing - char2", "a+AD0.b", "a=.b"),
+		Entry("missing - char end", "88+xUixVdVYwTjGlA", "88안녕하세요"),
+	)
+
+	DescribeTable(
+		"EncodeDecode",
+		func(input string) {
+			output, err := utf7.DecodeString(utf7.EncodeString(input))
+			Expect(err).To(Succeed())
+			Expect(output).To(Equal(input))
+		},
+		Entry("regular filename", "private.txt"),
+		Entry("direct chars", "is-better?yes:no(3).pdf"),
+		Entry("with spaces", "a big file with spaces.docx"),
+		Entry("with symbols", "a+b=c"),
+		Entry("unicode filenames", "超極秘文書.doc"),
+		Entry("emoji and symbols", "💰🔜™️.pdf"),
+	)
+
+	DescribeTable(
+		"DecodeFailures",
+		func(input string) {
+			out, err := utf7.DecodeString(input)
+			Expect(err).To(HaveOccurred())
+			Expect(out).To(Equal(""))
+		},
+		Entry("Non-utf16 sequence in base64", "a+-b+AD-c"),
+		Entry("Illegal base64 char", "a+-b+A.D-c"),
+		Entry("Non-ascii string", "⇉a+-b"),
+	)
+})