Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,197 @@
package convert
import (
"fmt"
"reflect"
"slices"
"strings"
"github.com/qsfera/server/pkg/ast"
)
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
return kqlExpander{}.expand(nodes, "")
}
type kqlExpander struct{}
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
for i, node := range nodes {
rnode := reflect.ValueOf(node)
// we need to ensure that the node is a pointer to an ast.Node in every case
if rnode.Kind() != reflect.Ptr {
ptr := reflect.New(rnode.Type())
ptr.Elem().Set(rnode)
rnode = ptr
cnode, ok := rnode.Interface().(ast.Node)
if !ok {
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
}
node = cnode // Update the original node to the pointer
nodes[i] = node // Update the original slice with the pointer
}
var unfoldedNodes []ast.Node
switch cnode := node.(type) {
case *ast.GroupNode:
if cnode.Key != "" { // group nodes should not get a default key
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
if err != nil {
return nil, err
}
cnode.Nodes = groupNodes
case *ast.StringNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
case *ast.DateTimeNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
case *ast.BooleanNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
if unfoldedNodes != nil {
// Insert unfolded nodes at the current index
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
// Adjust index to account for new nodes
i += len(unfoldedNodes) - 1
}
}
return nodes, nil
}
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
if defaultKey == "" {
defaultKey = "Name" // Set a default key if none is provided
}
key, ok := map[string]string{
"": defaultKey, // Default case if current is empty
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
}[current]
if !ok {
return current // Return the original key if not found
}
return key
}
func (_ kqlExpander) lowerValue(key, value string) string {
if slices.Contains([]string{"Hidden"}, key) {
return value // ignore certain keys and return the original value
}
return strings.ToLower(value)
}
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
result, ok := map[string][]ast.Node{
"MimeType:file": {
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:folder": {
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:document": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
}},
},
"MimeType:spreadsheet": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
}},
},
"MimeType:presentation": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
}},
},
"MimeType:pdf": {
&ast.StringNode{Key: key, Value: "application/pdf"},
},
"MimeType:image": {
&ast.StringNode{Key: key, Value: "image/*"},
},
"MimeType:video": {
&ast.StringNode{Key: key, Value: "video/*"},
},
"MimeType:audio": {
&ast.StringNode{Key: key, Value: "audio/*"},
},
"MimeType:archive": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tgz"},
}},
},
}[fmt.Sprintf("%s:%s", key, value)]
if !ok {
return nil
}
return result
}
@@ -0,0 +1,607 @@
package convert_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestExpandKQLAST(t *testing.T) {
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "ast.node.V -> ast.node.PTR",
Got: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: "d"},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: "g"},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
ast.GroupNode{Key: "i", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: "d"},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: "g"},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("remaps some keys", func(t *testing.T) {
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
for k, v := range map[string]string{
"": "Name", // Default to "Name" if no key is provided
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"any": "any", // Example of an unknown key that should remain unchanged
} {
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
Name: fmt.Sprintf("%s -> %s", k, v),
Got: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: k},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: k},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: k},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: k},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
}},
}},
}},
},
})
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("lowercases some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "!Hidden: StringNode -> stringnode",
Got: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "stringnode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "stringnode"},
}},
},
},
{
Name: "Hidden: StringNode -> StringNode",
Got: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "StringNode"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("unfolds some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "MimeType:unknown",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:file",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "file"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:folder",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "folder"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:document",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "document"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:spreadsheet",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:presentation",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "presentation"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:pdf",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "pdf"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:image",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "image"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "image/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:video",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "video"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "video/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:audio",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "audio"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:archive",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "archive"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
t.Run("different cases", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "use the group node key as default key",
Got: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
&ast.StringNode{Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
Want: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
}
@@ -0,0 +1,35 @@
package convert
import (
"fmt"
"github.com/qsfera/server/pkg/kql"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
)
var (
ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
)
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
kqlAst, err := kql.Builder{}.Build(kqlQuery)
if err != nil {
return nil, fmt.Errorf("failed to build query: %w", err)
}
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
}
builder, err := TranspileKQLToOpenSearch(kqlNodes)
if err != nil {
return nil, fmt.Errorf("failed to compile query: %w", err)
}
if q, ok := builder.(*osu.BoolQuery); !ok {
return osu.NewBoolQuery().Must(builder), nil
} else {
return q, nil
}
}
@@ -0,0 +1 @@
package convert_test
@@ -0,0 +1,147 @@
package convert
import (
"errors"
"fmt"
"strings"
"time"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/pkg/kql"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
)
func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) {
return kqlOpensearchTranspiler{}.Transpile(nodes)
}
type kqlOpensearchTranspiler struct{}
func (t kqlOpensearchTranspiler) Transpile(nodes []ast.Node) (osu.Builder, error) {
q, err := t.transpile(nodes)
if err != nil {
return nil, err
}
return q, nil
}
func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error) {
if len(nodes) == 0 {
return nil, fmt.Errorf("no nodes to compile")
}
if len(nodes) == 1 {
builder, err := t.toBuilder(nodes[0])
if err != nil {
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
}
return builder, nil
}
boolQueryParams := &osu.BoolQueryParams{}
boolQuery := osu.NewBoolQuery().Params(boolQueryParams)
boolQueryAdd := boolQuery.Must
for i, node := range nodes {
nextOp := t.getOperatorValueAt(nodes, i+1)
prevOp := t.getOperatorValueAt(nodes, i-1)
switch {
case nextOp == kql.BoolOR:
boolQueryAdd = boolQuery.Should
case nextOp == kql.BoolAND:
boolQueryAdd = boolQuery.Must
case prevOp == kql.BoolNOT:
boolQueryAdd = boolQuery.MustNot
}
builder, err := t.toBuilder(node)
switch {
// if the node is not known, we skip it, such as an operator node
case errors.Is(err, ErrUnsupportedNodeType):
continue
case err != nil:
return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err)
}
if _, ok := node.(*ast.OperatorNode); ok {
// operatorNodes are not builders, so we skip them
continue
}
if nextOp == kql.BoolOR {
// if there are should clauses, we set the minimum should match to 1
boolQueryParams.MinimumShouldMatch = 1
}
boolQueryAdd(builder)
}
return boolQuery, nil
}
func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) string {
if i < 0 || i >= len(nodes) {
return ""
}
if opn, ok := nodes[i].(*ast.OperatorNode); ok {
return opn.Value
}
return ""
}
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
var builder osu.Builder
switch node := node.(type) {
case *ast.BooleanNode:
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
case *ast.StringNode:
isWildcard := strings.Contains(node.Value, "*")
if isWildcard {
return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
}
totalTerms := strings.Split(node.Value, " ")
isSingleTerm := len(totalTerms) == 1
isMultiTerm := len(totalTerms) >= 1
switch {
case isSingleTerm:
return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
case isMultiTerm:
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
}
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
case *ast.DateTimeNode:
if node.Operator == nil {
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
case *ast.GroupNode:
group, err := t.transpile(node.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to build group: %w", err)
}
return group, nil
}
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
}
@@ -0,0 +1,378 @@
package convert_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestTranspileKQLToOpenSearch(t *testing.T) {
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "term query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
},
{
Name: "term query - boolean node - true",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: true},
},
},
Want: osu.NewTermQuery[bool]("Deleted").Value(true),
},
{
Name: "term query - boolean node - false",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
},
},
Want: osu.NewTermQuery[bool]("Deleted").Value(false),
},
{
Name: "match-phrase query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "open cloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
},
{
Name: "wildcard query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "open*"},
},
},
Want: osu.NewWildcardQuery("Name").Value("open*"),
},
{
Name: "bool query",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "a"},
&ast.StringNode{Key: "Name", Value: "b"},
}},
},
},
Want: osu.NewBoolQuery().Must(
osu.NewTermQuery[string]("Name").Value("a"),
osu.NewTermQuery[string]("Name").Value("b"),
),
},
{
Name: "no bool query for single term",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "any"},
}},
},
},
Want: osu.NewTermQuery[string]("Name").Value("any"),
},
{
Name: "range query >",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query >=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
// kql to os dsl - structure tests
{
Name: "[*]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
},
{
Name: "[* *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
).
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR * OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "44"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
osu.NewTermQuery[string]("age").Value("44"),
),
},
{
Name: "[* AND * OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Must(
osu.NewTermQuery[string]("a").Value("a"),
).
Should(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
},
{
Name: "[* OR * AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Must(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
).
Should(
osu.NewTermQuery[string]("a").Value("a"),
),
},
{
Name: "[* OR * AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
).
Must(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
},
{
Name: "[[* OR * OR *] AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "d", Value: "d"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
osu.NewTermQuery[string]("d").Value("d"),
),
},
{
Name: "[[* OR * OR *] AND NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "d", Value: "d"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
).MustNot(
osu.NewTermQuery[string]("d").Value("d"),
),
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("skipping test: " + test.Name)
}
dsl, err := convert.TranspileKQLToOpenSearch(test.Got.Nodes)
assert.NoError(t, err)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
})
}
}
@@ -0,0 +1,99 @@
package convert
import (
"fmt"
"strings"
"time"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/qsfera/server/pkg/conversions"
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
"github.com/qsfera/server/services/search/pkg/search"
)
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
resource, err := conversions.To[search.Resource](hit.Source)
if err != nil {
return nil, fmt.Errorf("failed to convert hit source: %w", err)
}
resourceRootID, err := storagespace.ParseID(resource.RootID)
if err != nil {
return nil, err
}
resourceID, err := storagespace.ParseID(resource.ID)
if err != nil {
return nil, err
}
resourceParentID, _ := storagespace.ParseID(resource.ParentID)
match := &searchMessage.Match{
Score: hit.Score,
Entity: &searchMessage.Entity{
Ref: &searchMessage.Reference{
ResourceId: &searchMessage.ResourceID{
StorageId: resourceRootID.GetStorageId(),
SpaceId: resourceRootID.GetSpaceId(),
OpaqueId: resourceRootID.GetOpaqueId(),
},
Path: resource.Path,
},
Id: &searchMessage.ResourceID{
StorageId: resourceID.GetStorageId(),
SpaceId: resourceID.GetSpaceId(),
OpaqueId: resourceID.GetOpaqueId(),
},
Name: resource.Name,
ParentId: &searchMessage.ResourceID{
StorageId: resourceParentID.GetStorageId(),
SpaceId: resourceParentID.GetSpaceId(),
OpaqueId: resourceParentID.GetOpaqueId(),
},
Size: resource.Size,
Type: resource.Type,
MimeType: resource.MimeType,
Deleted: resource.Deleted,
Tags: resource.Tags,
Highlights: func() string {
contentHighlights, ok := hit.Highlight["Content"]
if !ok {
return ""
}
return strings.Join(contentHighlights[:], "; ")
}(),
Audio: func() *searchMessage.Audio {
if !strings.HasPrefix(resource.MimeType, "audio/") {
return nil
}
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
return audio
}(),
Image: func() *searchMessage.Image {
image, _ := conversions.To[*searchMessage.Image](resource.Image)
return image
}(),
Location: func() *searchMessage.GeoCoordinates {
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
return geoCoordinates
}(),
Photo: func() *searchMessage.Photo {
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
return photo
}(),
},
}
if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil {
match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
}
return match, nil
}
@@ -0,0 +1,38 @@
package convert_test
import (
"encoding/json"
"testing"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/pkg/conversions"
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestOpenSearchHitToMatch(t *testing.T) {
resource := opensearchtest.Testdata.Resources.File
resource.MimeType = "audio/anything"
hit := opensearchgoAPI.SearchHit{
Score: 1.1,
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
}
match, err := convert.OpenSearchHitToMatch(hit)
assert.NoError(t, err)
assert.Equal(t, hit.Score, match.Score)
assert.Equal(t, resource.Name, match.Entity.Name)
t.Parallel()
t.Run("converts the audio field to the expected type", func(t *testing.T) {
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
assert.NoError(t, err)
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
})
}