Initial QSfera import
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/qsfera/server/pkg/ast"
|
||||
)
|
||||
|
||||
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
|
||||
return kqlExpander{}.expand(nodes, "")
|
||||
}
|
||||
|
||||
type kqlExpander struct{}
|
||||
|
||||
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
|
||||
for i, node := range nodes {
|
||||
rnode := reflect.ValueOf(node)
|
||||
|
||||
// we need to ensure that the node is a pointer to an ast.Node in every case
|
||||
if rnode.Kind() != reflect.Ptr {
|
||||
ptr := reflect.New(rnode.Type())
|
||||
ptr.Elem().Set(rnode)
|
||||
rnode = ptr
|
||||
cnode, ok := rnode.Interface().(ast.Node)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
|
||||
}
|
||||
|
||||
node = cnode // Update the original node to the pointer
|
||||
nodes[i] = node // Update the original slice with the pointer
|
||||
}
|
||||
|
||||
var unfoldedNodes []ast.Node
|
||||
switch cnode := node.(type) {
|
||||
case *ast.GroupNode:
|
||||
if cnode.Key != "" { // group nodes should not get a default key
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cnode.Nodes = groupNodes
|
||||
case *ast.StringNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
|
||||
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
|
||||
case *ast.DateTimeNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
case *ast.BooleanNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
if unfoldedNodes != nil {
|
||||
// Insert unfolded nodes at the current index
|
||||
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
|
||||
// Adjust index to account for new nodes
|
||||
i += len(unfoldedNodes) - 1
|
||||
}
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
|
||||
if defaultKey == "" {
|
||||
defaultKey = "Name" // Set a default key if none is provided
|
||||
}
|
||||
|
||||
key, ok := map[string]string{
|
||||
"": defaultKey, // Default case if current is empty
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
}[current]
|
||||
if !ok {
|
||||
return current // Return the original key if not found
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (_ kqlExpander) lowerValue(key, value string) string {
|
||||
if slices.Contains([]string{"Hidden"}, key) {
|
||||
return value // ignore certain keys and return the original value
|
||||
}
|
||||
|
||||
return strings.ToLower(value)
|
||||
}
|
||||
|
||||
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
|
||||
result, ok := map[string][]ast.Node{
|
||||
"MimeType:file": {
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:folder": {
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:document": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
},
|
||||
"MimeType:spreadsheet": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
},
|
||||
"MimeType:presentation": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
},
|
||||
"MimeType:pdf": {
|
||||
&ast.StringNode{Key: key, Value: "application/pdf"},
|
||||
},
|
||||
"MimeType:image": {
|
||||
&ast.StringNode{Key: key, Value: "image/*"},
|
||||
},
|
||||
"MimeType:video": {
|
||||
&ast.StringNode{Key: key, Value: "video/*"},
|
||||
},
|
||||
"MimeType:audio": {
|
||||
&ast.StringNode{Key: key, Value: "audio/*"},
|
||||
},
|
||||
"MimeType:archive": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tgz"},
|
||||
}},
|
||||
},
|
||||
}[fmt.Sprintf("%s:%s", key, value)]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
|
||||
|
||||
"github.com/qsfera/server/pkg/ast"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestExpandKQLAST(t *testing.T) {
|
||||
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "ast.node.V -> ast.node.PTR",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: "d"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: "g"},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: "d"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: "g"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("remaps some keys", func(t *testing.T) {
|
||||
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
|
||||
|
||||
for k, v := range map[string]string{
|
||||
"": "Name", // Default to "Name" if no key is provided
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
"any": "any", // Example of an unknown key that should remain unchanged
|
||||
} {
|
||||
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
Name: fmt.Sprintf("%s -> %s", k, v),
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: k},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: k},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: k},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("lowercases some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "!Hidden: StringNode -> stringnode",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "stringnode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "stringnode"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Hidden: StringNode -> StringNode",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unfolds some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "MimeType:unknown",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:file",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "file"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:folder",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "folder"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:document",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "document"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:spreadsheet",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:presentation",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "presentation"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:pdf",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "pdf"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:image",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "image"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "image/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:video",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "video"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "video/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:audio",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "audio"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:archive",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "archive"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different cases", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "use the group node key as default key",
|
||||
Got: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "mediatype", Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/qsfera/server/pkg/kql"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
|
||||
)
|
||||
|
||||
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
kqlAst, err := kql.Builder{}.Build(kqlQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build query: %w", err)
|
||||
}
|
||||
|
||||
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
|
||||
}
|
||||
|
||||
builder, err := TranspileKQLToOpenSearch(kqlNodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile query: %w", err)
|
||||
}
|
||||
|
||||
if q, ok := builder.(*osu.BoolQuery); !ok {
|
||||
return osu.NewBoolQuery().Must(builder), nil
|
||||
} else {
|
||||
return q, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package convert_test
|
||||
@@ -0,0 +1,147 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/ast"
|
||||
"github.com/qsfera/server/pkg/kql"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
)
|
||||
|
||||
func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) {
|
||||
return kqlOpensearchTranspiler{}.Transpile(nodes)
|
||||
}
|
||||
|
||||
type kqlOpensearchTranspiler struct{}
|
||||
|
||||
func (t kqlOpensearchTranspiler) Transpile(nodes []ast.Node) (osu.Builder, error) {
|
||||
q, err := t.transpile(nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error) {
|
||||
if len(nodes) == 0 {
|
||||
return nil, fmt.Errorf("no nodes to compile")
|
||||
}
|
||||
|
||||
if len(nodes) == 1 {
|
||||
builder, err := t.toBuilder(nodes[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
|
||||
}
|
||||
return builder, nil
|
||||
}
|
||||
|
||||
boolQueryParams := &osu.BoolQueryParams{}
|
||||
boolQuery := osu.NewBoolQuery().Params(boolQueryParams)
|
||||
boolQueryAdd := boolQuery.Must
|
||||
for i, node := range nodes {
|
||||
nextOp := t.getOperatorValueAt(nodes, i+1)
|
||||
prevOp := t.getOperatorValueAt(nodes, i-1)
|
||||
|
||||
switch {
|
||||
case nextOp == kql.BoolOR:
|
||||
boolQueryAdd = boolQuery.Should
|
||||
case nextOp == kql.BoolAND:
|
||||
boolQueryAdd = boolQuery.Must
|
||||
case prevOp == kql.BoolNOT:
|
||||
boolQueryAdd = boolQuery.MustNot
|
||||
}
|
||||
|
||||
builder, err := t.toBuilder(node)
|
||||
switch {
|
||||
// if the node is not known, we skip it, such as an operator node
|
||||
case errors.Is(err, ErrUnsupportedNodeType):
|
||||
continue
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err)
|
||||
}
|
||||
|
||||
if _, ok := node.(*ast.OperatorNode); ok {
|
||||
// operatorNodes are not builders, so we skip them
|
||||
continue
|
||||
}
|
||||
|
||||
if nextOp == kql.BoolOR {
|
||||
// if there are should clauses, we set the minimum should match to 1
|
||||
boolQueryParams.MinimumShouldMatch = 1
|
||||
}
|
||||
|
||||
boolQueryAdd(builder)
|
||||
}
|
||||
|
||||
return boolQuery, nil
|
||||
}
|
||||
|
||||
func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) string {
|
||||
if i < 0 || i >= len(nodes) {
|
||||
return ""
|
||||
}
|
||||
|
||||
if opn, ok := nodes[i].(*ast.OperatorNode); ok {
|
||||
return opn.Value
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
var builder osu.Builder
|
||||
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
if isWildcard {
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
|
||||
}
|
||||
|
||||
totalTerms := strings.Split(node.Value, " ")
|
||||
isSingleTerm := len(totalTerms) == 1
|
||||
isMultiTerm := len(totalTerms) >= 1
|
||||
switch {
|
||||
case isSingleTerm:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
|
||||
case isMultiTerm:
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
|
||||
case *ast.DateTimeNode:
|
||||
if node.Operator == nil {
|
||||
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
case *ast.GroupNode:
|
||||
group, err := t.transpile(node.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build group: %w", err)
|
||||
}
|
||||
|
||||
return group, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/pkg/ast"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
Name: "term query - string node",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - true",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: true},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[bool]("Deleted").Value(true),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - false",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[bool]("Deleted").Value(false),
|
||||
},
|
||||
{
|
||||
Name: "match-phrase query - string node",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "open cloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Name").Value("open*"),
|
||||
},
|
||||
{
|
||||
Name: "bool query",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "a"},
|
||||
&ast.StringNode{Key: "Name", Value: "b"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().Must(
|
||||
osu.NewTermQuery[string]("Name").Value("a"),
|
||||
osu.NewTermQuery[string]("Name").Value("b"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "no bool query for single term",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "any"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("any"),
|
||||
},
|
||||
{
|
||||
Name: "range query >",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.DateTimeNode{
|
||||
Key: "Mtime",
|
||||
Operator: &ast.OperatorNode{Value: ">"},
|
||||
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Want: osu.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
|
||||
},
|
||||
{
|
||||
Name: "range query >=",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.DateTimeNode{
|
||||
Key: "Mtime",
|
||||
Operator: &ast.OperatorNode{Value: ">="},
|
||||
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Want: osu.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
|
||||
},
|
||||
{
|
||||
Name: "range query <",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.DateTimeNode{
|
||||
Key: "Mtime",
|
||||
Operator: &ast.OperatorNode{Value: "<"},
|
||||
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Want: osu.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
|
||||
},
|
||||
{
|
||||
Name: "range query <=",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.DateTimeNode{
|
||||
Key: "Mtime",
|
||||
Operator: &ast.OperatorNode{Value: "<="},
|
||||
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
|
||||
},
|
||||
},
|
||||
},
|
||||
Want: osu.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
|
||||
},
|
||||
// kql to os dsl - structure tests
|
||||
{
|
||||
Name: "[*]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* AND *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* OR *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[NOT *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* NOT *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
).
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* OR * OR *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "qsfera"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "age", Value: "32"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "age", Value: "44"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("qsfera"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
osu.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* AND * OR *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "c"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Must(
|
||||
osu.NewTermQuery[string]("a").Value("a"),
|
||||
).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("b").Value("b"),
|
||||
osu.NewTermQuery[string]("c").Value("c"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* OR * AND *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "a"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "b", Value: "b"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "c", Value: "c"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Must(
|
||||
osu.NewTermQuery[string]("b").Value("b"),
|
||||
osu.NewTermQuery[string]("c").Value("c"),
|
||||
).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("a").Value("a"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[* OR * AND *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "a"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "b", Value: "b"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "c", Value: "c"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("a").Value("a"),
|
||||
).
|
||||
Must(
|
||||
osu.NewTermQuery[string]("b").Value("b"),
|
||||
osu.NewTermQuery[string]("c").Value("c"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[[* OR * OR *] AND *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "a"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "b", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "c"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "d", Value: "d"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("a").Value("a"),
|
||||
osu.NewTermQuery[string]("b").Value("b"),
|
||||
osu.NewTermQuery[string]("c").Value("c"),
|
||||
),
|
||||
osu.NewTermQuery[string]("d").Value("d"),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "[[* OR * OR *] AND NOT *]",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "a"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "b", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "c"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "d", Value: "d"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("a").Value("a"),
|
||||
osu.NewTermQuery[string]("b").Value("b"),
|
||||
osu.NewTermQuery[string]("c").Value("c"),
|
||||
),
|
||||
).MustNot(
|
||||
osu.NewTermQuery[string]("d").Value("d"),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("skipping test: " + test.Name)
|
||||
}
|
||||
|
||||
dsl, err := convert.TranspileKQLToOpenSearch(test.Got.Nodes)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
|
||||
"github.com/qsfera/server/pkg/conversions"
|
||||
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
|
||||
"github.com/qsfera/server/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
|
||||
resource, err := conversions.To[search.Resource](hit.Source)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert hit source: %w", err)
|
||||
}
|
||||
|
||||
resourceRootID, err := storagespace.ParseID(resource.RootID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resourceID, err := storagespace.ParseID(resource.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resourceParentID, _ := storagespace.ParseID(resource.ParentID)
|
||||
|
||||
match := &searchMessage.Match{
|
||||
Score: hit.Score,
|
||||
Entity: &searchMessage.Entity{
|
||||
Ref: &searchMessage.Reference{
|
||||
ResourceId: &searchMessage.ResourceID{
|
||||
StorageId: resourceRootID.GetStorageId(),
|
||||
SpaceId: resourceRootID.GetSpaceId(),
|
||||
OpaqueId: resourceRootID.GetOpaqueId(),
|
||||
},
|
||||
Path: resource.Path,
|
||||
},
|
||||
Id: &searchMessage.ResourceID{
|
||||
StorageId: resourceID.GetStorageId(),
|
||||
SpaceId: resourceID.GetSpaceId(),
|
||||
OpaqueId: resourceID.GetOpaqueId(),
|
||||
},
|
||||
Name: resource.Name,
|
||||
ParentId: &searchMessage.ResourceID{
|
||||
StorageId: resourceParentID.GetStorageId(),
|
||||
SpaceId: resourceParentID.GetSpaceId(),
|
||||
OpaqueId: resourceParentID.GetOpaqueId(),
|
||||
},
|
||||
Size: resource.Size,
|
||||
Type: resource.Type,
|
||||
MimeType: resource.MimeType,
|
||||
Deleted: resource.Deleted,
|
||||
Tags: resource.Tags,
|
||||
Highlights: func() string {
|
||||
contentHighlights, ok := hit.Highlight["Content"]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.Join(contentHighlights[:], "; ")
|
||||
}(),
|
||||
Audio: func() *searchMessage.Audio {
|
||||
if !strings.HasPrefix(resource.MimeType, "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
return audio
|
||||
}(),
|
||||
Image: func() *searchMessage.Image {
|
||||
image, _ := conversions.To[*searchMessage.Image](resource.Image)
|
||||
return image
|
||||
}(),
|
||||
Location: func() *searchMessage.GeoCoordinates {
|
||||
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
|
||||
return geoCoordinates
|
||||
}(),
|
||||
Photo: func() *searchMessage.Photo {
|
||||
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
|
||||
return photo
|
||||
}(),
|
||||
},
|
||||
}
|
||||
|
||||
if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil {
|
||||
match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
|
||||
}
|
||||
|
||||
return match, nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/pkg/conversions"
|
||||
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestOpenSearchHitToMatch(t *testing.T) {
|
||||
resource := opensearchtest.Testdata.Resources.File
|
||||
resource.MimeType = "audio/anything"
|
||||
|
||||
hit := opensearchgoAPI.SearchHit{
|
||||
Score: 1.1,
|
||||
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
|
||||
}
|
||||
match, err := convert.OpenSearchHitToMatch(hit)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, hit.Score, match.Score)
|
||||
assert.Equal(t, resource.Name, match.Entity.Name)
|
||||
t.Parallel()
|
||||
t.Run("converts the audio field to the expected type", func(t *testing.T) {
|
||||
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
|
||||
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
|
||||
audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"Content": {
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets"
|
||||
},
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Favorites": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"dario.cat/mergo"
|
||||
|
||||
"github.com/qsfera/server/pkg/conversions"
|
||||
)
|
||||
|
||||
type Builder interface {
|
||||
json.Marshaler
|
||||
Map() (map[string]any, error)
|
||||
}
|
||||
|
||||
func newBase(v ...any) (map[string]any, error) {
|
||||
base := make(map[string]any)
|
||||
for _, value := range v {
|
||||
data, err := conversions.To[map[string]any](value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert value to map: %w", err)
|
||||
}
|
||||
|
||||
if isEmpty(data) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := mergo.Merge(&base, data); err != nil {
|
||||
return nil, fmt.Errorf("failed to merge value into base: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return base, nil
|
||||
}
|
||||
|
||||
func applyValue[T any](target map[string]any, key string, v T) {
|
||||
if target == nil || isEmpty(key) || isEmpty(v) {
|
||||
return
|
||||
}
|
||||
|
||||
target[key] = v
|
||||
}
|
||||
|
||||
func applyValues[T any](target map[string]any, values map[string]T) {
|
||||
if target == nil || isEmpty(values) {
|
||||
return
|
||||
}
|
||||
|
||||
for k, v := range values {
|
||||
applyValue[T](target, k, v)
|
||||
}
|
||||
}
|
||||
|
||||
func applyBuilder(target map[string]any, key string, builder Builder) error {
|
||||
if target == nil || isEmpty(key) || isEmpty(builder) {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := builder.Map()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to map builder %s: %w", key, err)
|
||||
}
|
||||
|
||||
if !isEmpty(data) {
|
||||
target[key] = data
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyBuilders(target map[string]any, key string, bs ...Builder) error {
|
||||
if target == nil || isEmpty(key) || isEmpty(bs) {
|
||||
return nil
|
||||
}
|
||||
|
||||
builders := make([]map[string]any, 0, len(bs))
|
||||
for _, builder := range bs {
|
||||
data, err := builder.Map()
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to map builder %s: %w", key, err)
|
||||
case isEmpty(data):
|
||||
continue
|
||||
default:
|
||||
builders = append(builders, data)
|
||||
}
|
||||
}
|
||||
|
||||
if len(builders) > 0 {
|
||||
target[key] = builders
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isEmpty(x any) bool {
|
||||
switch {
|
||||
case x == nil:
|
||||
return true
|
||||
case reflect.ValueOf(x).Kind() == reflect.Bool:
|
||||
return false
|
||||
case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()):
|
||||
return true
|
||||
case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func merge[T any](vals ...T) T {
|
||||
base := make(map[string]any)
|
||||
|
||||
for _, val := range vals {
|
||||
data, err := conversions.To[map[string]any](val)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
_ = mergo.Merge(&base, data)
|
||||
}
|
||||
|
||||
data, _ := conversions.To[T](base)
|
||||
|
||||
return data
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type BoolQuery struct {
|
||||
must []Builder
|
||||
mustNot []Builder
|
||||
should []Builder
|
||||
filter []Builder
|
||||
params *BoolQueryParams
|
||||
}
|
||||
|
||||
type BoolQueryParams struct {
|
||||
MinimumShouldMatch int16 `json:"minimum_should_match,omitempty"`
|
||||
Boost float32 `json:"boost,omitempty"`
|
||||
Name string `json:"_name,omitempty"`
|
||||
}
|
||||
|
||||
func NewBoolQuery() *BoolQuery {
|
||||
return &BoolQuery{}
|
||||
}
|
||||
|
||||
func (q *BoolQuery) Params(v *BoolQueryParams) *BoolQuery {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *BoolQuery) Must(v ...Builder) *BoolQuery {
|
||||
q.must = append(q.must, v...)
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *BoolQuery) MustNot(v ...Builder) *BoolQuery {
|
||||
q.mustNot = append(q.mustNot, v...)
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *BoolQuery) Should(v ...Builder) *BoolQuery {
|
||||
q.should = append(q.should, v...)
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *BoolQuery) Filter(v ...Builder) *BoolQuery {
|
||||
q.filter = append(q.filter, v...)
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *BoolQuery) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := applyBuilders(base, "must", q.must...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := applyBuilders(base, "must_not", q.mustNot...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := applyBuilders(base, "should", q.should...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := applyBuilders(base, "filter", q.filter...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"bool": base,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *BoolQuery) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestBoolQuery(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewBoolQuery(),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "with params",
|
||||
Got: osu.NewBoolQuery().Params(&osu.BoolQueryParams{
|
||||
MinimumShouldMatch: 10,
|
||||
Boost: 10,
|
||||
Name: "some-name",
|
||||
}),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"minimum_should_match": 10,
|
||||
"boost": 10,
|
||||
"_name": "some-name",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "must",
|
||||
Got: osu.NewBoolQuery().Must(osu.NewTermQuery[string]("name").Value("tom")),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"must": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "must_not",
|
||||
Got: osu.NewBoolQuery().MustNot(osu.NewTermQuery[string]("name").Value("tom")),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"must_not": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "should",
|
||||
Got: osu.NewBoolQuery().Should(osu.NewTermQuery[string]("name").Value("tom")),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"should": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "filter",
|
||||
Got: osu.NewBoolQuery().Filter(osu.NewTermQuery[string]("name").Value("tom")),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"filter": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "full",
|
||||
Got: osu.NewBoolQuery().
|
||||
Must(osu.NewTermQuery[string]("name").Value("tom")).
|
||||
MustNot(osu.NewTermQuery[bool]("deleted").Value(true)).
|
||||
Should(osu.NewTermQuery[string]("gender").Value("male")).
|
||||
Filter(osu.NewTermQuery[int]("age").Value(42)),
|
||||
Want: map[string]any{
|
||||
"bool": map[string]any{
|
||||
"must": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"must_not": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"should": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"gender": map[string]any{
|
||||
"value": "male",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"filter": []map[string]any{
|
||||
{
|
||||
"term": map[string]any{
|
||||
"age": map[string]any{
|
||||
"value": 42,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type MatchPhraseQuery struct {
|
||||
field string
|
||||
query string
|
||||
params *MatchPhraseQueryParams
|
||||
}
|
||||
|
||||
type MatchPhraseQueryParams struct {
|
||||
Analyzer string `json:"analyzer,omitempty"`
|
||||
Slop int `json:"slop,omitempty"`
|
||||
ZeroTermsQuery string `json:"zero_terms_query,omitempty"`
|
||||
}
|
||||
|
||||
func NewMatchPhraseQuery(field string) *MatchPhraseQuery {
|
||||
return &MatchPhraseQuery{field: field}
|
||||
}
|
||||
|
||||
func (q *MatchPhraseQuery) Params(v *MatchPhraseQueryParams) *MatchPhraseQuery {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery {
|
||||
q.query = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *MatchPhraseQuery) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
applyValue(base, "query", q.query)
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"match_phrase": map[string]any{
|
||||
q.field: base,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *MatchPhraseQuery) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestNewMatchPhraseQuery(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewMatchPhraseQuery("empty"),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "with params",
|
||||
Got: osu.NewMatchPhraseQuery("name").Params(&osu.MatchPhraseQueryParams{
|
||||
Analyzer: "analyzer",
|
||||
Slop: 2,
|
||||
ZeroTermsQuery: "all",
|
||||
}),
|
||||
Want: map[string]any{
|
||||
"match_phrase": map[string]any{
|
||||
"name": map[string]any{
|
||||
"analyzer": "analyzer",
|
||||
"slop": 2,
|
||||
"zero_terms_query": "all",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "query",
|
||||
Got: osu.NewMatchPhraseQuery("name").Query("some match query"),
|
||||
Want: map[string]any{
|
||||
"match_phrase": map[string]any{
|
||||
"name": map[string]any{
|
||||
"query": "some match query",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "full",
|
||||
Got: osu.NewMatchPhraseQuery("name").Params(&osu.MatchPhraseQueryParams{
|
||||
Analyzer: "analyzer",
|
||||
Slop: 2,
|
||||
ZeroTermsQuery: "all",
|
||||
}).Query("some match query"),
|
||||
Want: map[string]any{
|
||||
"match_phrase": map[string]any{
|
||||
"name": map[string]any{
|
||||
"query": "some match query",
|
||||
"analyzer": "analyzer",
|
||||
"slop": 2,
|
||||
"zero_terms_query": "all",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
)
|
||||
|
||||
type IDsQuery struct {
|
||||
values []string
|
||||
params *IDsQueryParams
|
||||
}
|
||||
|
||||
type IDsQueryParams struct {
|
||||
Boost float32 `json:"boost,omitempty"`
|
||||
}
|
||||
|
||||
func NewIDsQuery(v ...string) *IDsQuery {
|
||||
return &IDsQuery{values: slices.Compact(v)}
|
||||
}
|
||||
|
||||
func (q *IDsQuery) Params(v *IDsQueryParams) *IDsQuery {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *IDsQuery) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
applyValue(base, "values", q.values)
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"ids": base,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *IDsQuery) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestIDsQuery(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewIDsQuery(),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "no params",
|
||||
Got: osu.NewIDsQuery("1", "2", "3", "3"),
|
||||
Want: map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{"1", "2", "3"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "ids",
|
||||
Got: osu.NewIDsQuery("1", "2", "3", "3").Params(&osu.IDsQueryParams{Boost: 1.0}),
|
||||
Want: map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{"1", "2", "3"},
|
||||
"boost": 1.0,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RangeQuery[T time.Time | string] struct {
|
||||
field string
|
||||
gt T
|
||||
gte T
|
||||
lt T
|
||||
lte T
|
||||
params *RangeQueryParams
|
||||
}
|
||||
|
||||
type RangeQueryParams struct {
|
||||
Format string `json:"format,omitempty"`
|
||||
Relation string `json:"relation,omitempty"`
|
||||
Boost float32 `json:"boost,omitempty"`
|
||||
TimeZone string `json:"time_zone,omitempty"`
|
||||
}
|
||||
|
||||
func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] {
|
||||
return &RangeQuery[T]{field: field}
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Params(v *RangeQueryParams) *RangeQuery[T] {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Gt(v T) *RangeQuery[T] {
|
||||
q.gt = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Gte(v T) *RangeQuery[T] {
|
||||
q.gte = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Lt(v T) *RangeQuery[T] {
|
||||
q.lt = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Lte(v T) *RangeQuery[T] {
|
||||
q.lte = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) Map() (map[string]any, error) {
|
||||
if !isEmpty(q.gt) && !isEmpty(q.gte) {
|
||||
return nil, errors.New("cannot set both gt and gte in RangeQuery")
|
||||
}
|
||||
|
||||
if !isEmpty(q.lt) && !isEmpty(q.lte) {
|
||||
return nil, errors.New("cannot set both lt and lte in RangeQuery")
|
||||
}
|
||||
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
applyValues(base, map[string]T{
|
||||
"gt": q.gt,
|
||||
"gte": q.gte,
|
||||
"lt": q.lt,
|
||||
"lte": q.lte,
|
||||
})
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"range": map[string]any{
|
||||
q.field: base,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *RangeQuery[T]) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestRangeQuery(t *testing.T) {
|
||||
now := time.Now()
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewRangeQuery[string]("empty"),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "gt string",
|
||||
Got: osu.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"gt": "2023-01-01T00:00:00Z",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "gt time",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Gt(now),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"gt": now,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "gte string",
|
||||
Got: osu.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"gte": "2023-01-01T00:00:00Z",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "gte time",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Gte(now),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"gte": now,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "gt & gte",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Gt(now).Gte(now),
|
||||
Want: nil,
|
||||
Err: errors.New(""),
|
||||
},
|
||||
{
|
||||
Name: "gt string",
|
||||
Got: osu.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"lt": "2023-01-01T00:00:00Z",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "lt time",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Lt(now),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"lt": now,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "lte string",
|
||||
Got: osu.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"lte": "2023-01-01T00:00:00Z",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "lte time",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Lte(now),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"lte": now,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "lt & lte",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Lt(now).Lte(now),
|
||||
Want: nil,
|
||||
Err: errors.New(""),
|
||||
},
|
||||
{
|
||||
Name: "with params",
|
||||
Got: osu.NewRangeQuery[time.Time]("created").Params(&osu.RangeQueryParams{
|
||||
Format: "strict_date_optional_time",
|
||||
Relation: "within",
|
||||
Boost: 1.0,
|
||||
TimeZone: "UTC",
|
||||
}).Lte(now).Gte(now),
|
||||
Want: map[string]any{
|
||||
"range": map[string]any{
|
||||
"created": map[string]any{
|
||||
"lte": now,
|
||||
"gte": now,
|
||||
"format": "strict_date_optional_time",
|
||||
"relation": "within",
|
||||
"boost": 1.0,
|
||||
"time_zone": "UTC",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
got, err := test.Got.MarshalJSON()
|
||||
switch {
|
||||
case test.Err != nil && test.Err.Error() == "": // Expecting any error
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, got)
|
||||
return
|
||||
case test.Err != nil && test.Err.Error() != "": // Expecting a specific error
|
||||
assert.ErrorIs(t, test.Err, err)
|
||||
assert.Nil(t, got)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), string(got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type TermQuery[T comparable] struct {
|
||||
field string
|
||||
value T
|
||||
params *TermQueryParams
|
||||
}
|
||||
|
||||
type TermQueryParams struct {
|
||||
Boost float32 `json:"boost,omitempty"`
|
||||
CaseInsensitive bool `json:"case_insensitive,omitempty"`
|
||||
Name string `json:"_name,omitempty"`
|
||||
}
|
||||
|
||||
func NewTermQuery[T comparable](field string) *TermQuery[T] {
|
||||
return &TermQuery[T]{field: field}
|
||||
}
|
||||
|
||||
func (q *TermQuery[T]) Params(v *TermQueryParams) *TermQuery[T] {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *TermQuery[T]) Value(v T) *TermQuery[T] {
|
||||
q.value = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *TermQuery[T]) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
applyValue(base, "value", q.value)
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"term": map[string]any{
|
||||
q.field: base,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *TermQuery[T]) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestTermQuery(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewTermQuery[string]("empty"),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "no params",
|
||||
Got: osu.NewTermQuery[bool]("deleted").Value(false),
|
||||
Want: map[string]any{
|
||||
"term": map[string]any{
|
||||
"deleted": map[string]any{
|
||||
"value": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "with params",
|
||||
Got: osu.NewTermQuery[bool]("deleted").Params(&osu.TermQueryParams{
|
||||
Boost: 1.0,
|
||||
CaseInsensitive: true,
|
||||
Name: "is-deleted",
|
||||
}).Value(true),
|
||||
Want: map[string]any{
|
||||
"term": map[string]any{
|
||||
"deleted": map[string]any{
|
||||
"value": true,
|
||||
"boost": 1.0,
|
||||
"case_insensitive": true,
|
||||
"_name": "is-deleted",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type WildcardQuery struct {
|
||||
field string
|
||||
value string
|
||||
params *WildcardQueryParams
|
||||
}
|
||||
|
||||
type WildcardQueryParams struct {
|
||||
Boost float32 `json:"boost,omitempty"`
|
||||
CaseInsensitive bool `json:"case_insensitive,omitempty"`
|
||||
Rewrite string `json:"rewrite,omitempty"`
|
||||
}
|
||||
|
||||
func NewWildcardQuery(field string) *WildcardQuery {
|
||||
return &WildcardQuery{field: field}
|
||||
}
|
||||
|
||||
func (q *WildcardQuery) Params(v *WildcardQueryParams) *WildcardQuery {
|
||||
q.params = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *WildcardQuery) Value(v string) *WildcardQuery {
|
||||
q.value = v
|
||||
return q
|
||||
}
|
||||
|
||||
func (q *WildcardQuery) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
applyValue(base, "value", q.value)
|
||||
|
||||
if isEmpty(base) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"wildcard": map[string]any{
|
||||
q.field: base,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *WildcardQuery) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestWildcardQuery(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "empty",
|
||||
Got: osu.NewWildcardQuery("empty"),
|
||||
Want: nil,
|
||||
},
|
||||
{
|
||||
Name: "wildcard",
|
||||
Got: osu.NewWildcardQuery("name").Params(&osu.WildcardQueryParams{
|
||||
Boost: 1.0,
|
||||
CaseInsensitive: true,
|
||||
Rewrite: "top_terms_blended_freqs_N",
|
||||
}).Value("opencl*"),
|
||||
Want: map[string]any{
|
||||
"wildcard": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "opencl*",
|
||||
"boost": 1.0,
|
||||
"case_insensitive": true,
|
||||
"rewrite": "top_terms_blended_freqs_N",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package osu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
)
|
||||
|
||||
type QueryReqBody[P any] struct {
|
||||
query Builder
|
||||
params P
|
||||
}
|
||||
|
||||
func NewQueryReqBody[P any](q Builder, p ...P) *QueryReqBody[P] {
|
||||
return &QueryReqBody[P]{query: q, params: merge(p...)}
|
||||
}
|
||||
|
||||
func (q QueryReqBody[O]) Map() (map[string]any, error) {
|
||||
base, err := newBase(q.params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := applyBuilder(base, "query", q.query); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return base, nil
|
||||
}
|
||||
|
||||
func (q QueryReqBody[O]) MarshalJSON() ([]byte, error) {
|
||||
data, err := q.Map()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
type BodyParamHighlight struct {
|
||||
HighlightOptions
|
||||
Fields map[string]HighlightOptions `json:"fields,omitempty"`
|
||||
}
|
||||
|
||||
type HighlightType string
|
||||
|
||||
const (
|
||||
HighlightTypeUnified HighlightType = "unified"
|
||||
HighlightTypeFvh HighlightType = "fvh"
|
||||
HighlightTypePlain HighlightType = "plain"
|
||||
HighlightTypeSemantic HighlightType = "semantic"
|
||||
)
|
||||
|
||||
type HighlightOptions struct {
|
||||
Type HighlightType `json:"type,omitempty"`
|
||||
FragmentSize int `json:"fragment_size,omitempty"`
|
||||
NumberOfFragments int `json:"number_of_fragments,omitempty"`
|
||||
FragmentOffset int `json:"fragment_offset,omitempty"`
|
||||
BoundaryChars string `json:"boundary_chars,omitempty"`
|
||||
BoundaryMaxScan int `json:"boundary_max_scan,omitempty"`
|
||||
BoundaryScanner string `json:"boundary_scanner,omitempty"`
|
||||
BoundaryScannerLocale string `json:"boundary_scanner_locale,omitempty"`
|
||||
Encoder string `json:"encoder,omitempty"`
|
||||
ForceSource bool `json:"force_source,omitempty"`
|
||||
Fragmenter string `json:"fragmenter,omitempty"`
|
||||
HighlightQuery Builder `json:"highlight_query,omitempty"`
|
||||
Order string `json:"order,omitempty"`
|
||||
NoMatchSize int `json:"no_match_size,omitempty"`
|
||||
RequireFieldMatch bool `json:"require_field_match,omitempty"`
|
||||
MatchedFields []string `json:"matched_fields,omitempty"`
|
||||
PhraseLimit int `json:"phrase_limit,omitempty"`
|
||||
PreTags []string `json:"pre_tags,omitempty"`
|
||||
PostTags []string `json:"post_tags,omitempty"`
|
||||
}
|
||||
|
||||
type BodyParamScript struct {
|
||||
Source string `json:"source,omitempty"`
|
||||
Lang string `json:"lang,omitempty"`
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
func BuildSearchReq(req *opensearchgoAPI.SearchReq, q Builder, p ...SearchBodyParams) (*opensearchgoAPI.SearchReq, error) {
|
||||
body, err := json.Marshal(NewQueryReqBody(q, p...))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Body = bytes.NewReader(body)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type SearchBodyParams struct {
|
||||
Highlight *BodyParamHighlight `json:"highlight,omitempty"`
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
func BuildDocumentDeleteByQueryReq(req opensearchgoAPI.DocumentDeleteByQueryReq, q Builder) (opensearchgoAPI.DocumentDeleteByQueryReq, error) {
|
||||
body, err := json.Marshal(NewQueryReqBody[any](q))
|
||||
if err != nil {
|
||||
return req, err
|
||||
}
|
||||
req.Body = bytes.NewReader(body)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
func BuildUpdateByQueryReq(req opensearchgoAPI.UpdateByQueryReq, q Builder, o ...UpdateByQueryBodyParams) (opensearchgoAPI.UpdateByQueryReq, error) {
|
||||
body, err := json.Marshal(NewQueryReqBody(q, o...))
|
||||
if err != nil {
|
||||
return req, err
|
||||
}
|
||||
req.Body = bytes.NewReader(body)
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type UpdateByQueryBodyParams struct {
|
||||
Script *BodyParamScript `json:"script,omitempty"`
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
func BuildIndicesCountReq(req *opensearchgoAPI.IndicesCountReq, q Builder) (*opensearchgoAPI.IndicesCountReq, error) {
|
||||
body, err := json.Marshal(NewQueryReqBody[any](q))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Body = bytes.NewReader(body)
|
||||
return req, nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package osu_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
|
||||
opensearchtest "github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestRequestBody(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
|
||||
{
|
||||
Name: "simple",
|
||||
Got: osu.NewQueryReqBody[any](osu.NewTermQuery[string]("name").Value("tom")),
|
||||
Want: map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"name": map[string]any{
|
||||
"value": "tom",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSearchReq(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[io.Reader, map[string]any]{
|
||||
{
|
||||
Name: "highlight",
|
||||
Got: func() io.Reader {
|
||||
req, _ := osu.BuildSearchReq(
|
||||
&opensearchgoAPI.SearchReq{},
|
||||
osu.NewTermQuery[string]("content").Value("content"),
|
||||
osu.SearchBodyParams{
|
||||
Highlight: &osu.BodyParamHighlight{
|
||||
HighlightOptions: osu.HighlightOptions{
|
||||
PreTags: []string{"<b>"},
|
||||
PostTags: []string{"</b>"},
|
||||
},
|
||||
Fields: map[string]osu.HighlightOptions{
|
||||
"content": {
|
||||
PreTags: []string{"<strong>"},
|
||||
PostTags: []string{"</strong>"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return req.Body
|
||||
}(),
|
||||
Want: map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"content": map[string]any{
|
||||
"value": "content",
|
||||
},
|
||||
},
|
||||
},
|
||||
"highlight": map[string]any{
|
||||
"pre_tags": []string{"<b>"},
|
||||
"post_tags": []string{"</b>"},
|
||||
"fields": map[string]any{
|
||||
"content": map[string]any{
|
||||
"pre_tags": []string{"<strong>"},
|
||||
"post_tags": []string{"</strong>"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
body, err := io.ReadAll(test.Got)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), string(body))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package opensearchtest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/qsfera/server/pkg/conversions"
|
||||
)
|
||||
|
||||
var TimeMustParse = func(t *testing.T, ts string) time.Time {
|
||||
tp, err := time.Parse(time.RFC3339Nano, ts)
|
||||
require.NoError(t, err, "failed to parse time %s", ts)
|
||||
|
||||
return tp
|
||||
}
|
||||
|
||||
func JSONMustMarshal(t *testing.T, data any) string {
|
||||
jsonData, err := json.Marshal(data)
|
||||
require.NoError(t, err, "failed to marshal data to JSON")
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
|
||||
ts := make([]T, len(hits))
|
||||
for i, hit := range hits {
|
||||
resource, err := conversions.To[T](hit.Source)
|
||||
require.NoError(t, err)
|
||||
ts[i] = resource
|
||||
}
|
||||
|
||||
return ts
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package opensearchtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/config"
|
||||
)
|
||||
|
||||
type TestClient struct {
|
||||
c *opensearchgoAPI.Client
|
||||
Require *testRequireClient
|
||||
}
|
||||
|
||||
func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *TestClient {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: cfg.Addresses,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "failed to create OpenSearch client")
|
||||
|
||||
return NewTestClient(t, client)
|
||||
}
|
||||
|
||||
func NewTestClient(t *testing.T, client *opensearchgoAPI.Client) *TestClient {
|
||||
tc := &TestClient{c: client}
|
||||
trc := &testRequireClient{tc: tc, t: t}
|
||||
tc.Require = trc
|
||||
|
||||
return tc
|
||||
}
|
||||
|
||||
func (tc *TestClient) Client() *opensearchgoAPI.Client {
|
||||
return tc.c
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error {
|
||||
indicesToDelete := make([]string, 0, len(indices))
|
||||
for _, index := range indices {
|
||||
exist, err := tc.IndicesExists(ctx, []string{index})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check if index %s exists: %w", index, err)
|
||||
}
|
||||
|
||||
if !exist {
|
||||
continue
|
||||
}
|
||||
|
||||
indicesToDelete = append(indicesToDelete, index)
|
||||
}
|
||||
|
||||
if len(indicesToDelete) == 0 {
|
||||
// If no indices to delete, return nil
|
||||
return nil
|
||||
}
|
||||
|
||||
return tc.IndicesDelete(ctx, indicesToDelete)
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesExists(ctx context.Context, indices []string) (bool, error) {
|
||||
if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
resp, err := tc.c.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
|
||||
Indices: indices,
|
||||
})
|
||||
switch {
|
||||
case resp != nil && resp.StatusCode == 404:
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, fmt.Errorf("failed to check if indices exist: %w", err)
|
||||
case resp != nil && resp.IsError():
|
||||
return false, fmt.Errorf("failed to check if indices exist: %s", resp.String())
|
||||
default:
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesRefresh(ctx context.Context, indices []string, allow []int) error {
|
||||
resp, err := tc.c.Indices.Refresh(ctx, &opensearchgoAPI.IndicesRefreshReq{
|
||||
Indices: indices,
|
||||
})
|
||||
|
||||
isAllowed := resp != nil
|
||||
isAllowed = isAllowed && resp.Inspect().Response != nil
|
||||
isAllowed = isAllowed && slices.Contains(allow, resp.Inspect().Response.StatusCode)
|
||||
|
||||
if err != nil && !isAllowed {
|
||||
return fmt.Errorf("failed to refresh indices %v: %w", indices, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesDelete(ctx context.Context, indices []string) error {
|
||||
if err := tc.IndicesRefresh(ctx, indices, []int{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := tc.c.Indices.Delete(ctx, opensearchgoAPI.IndicesDeleteReq{
|
||||
Indices: indices,
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to delete indices: %w", err)
|
||||
case !resp.Acknowledged:
|
||||
return errors.New("indices deletion not acknowledged")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.Reader) error {
|
||||
resp, err := tc.c.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
|
||||
Index: index,
|
||||
Body: body,
|
||||
})
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to create index %s: %w", index, err)
|
||||
case !resp.Acknowledged:
|
||||
return fmt.Errorf("index creation not acknowledged for %s", index)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) {
|
||||
if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{
|
||||
Indices: indices,
|
||||
Body: body,
|
||||
})
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
return 0, fmt.Errorf("failed to count documents in indices: %w", err)
|
||||
default:
|
||||
return resp.Count, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error {
|
||||
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{
|
||||
Index: index,
|
||||
DocumentID: id,
|
||||
Body: body,
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to create document in index %s: %w", index, err)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error {
|
||||
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{
|
||||
Index: index,
|
||||
DocumentID: id,
|
||||
Body: body,
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to update document in index %s: %w", index, err)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) {
|
||||
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
|
||||
return opensearchgoAPI.SearchHits{}, err
|
||||
}
|
||||
|
||||
resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{
|
||||
Indices: []string{index},
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err)
|
||||
}
|
||||
|
||||
return resp.Hits, nil
|
||||
}
|
||||
|
||||
type testRequireClient struct {
|
||||
tc *TestClient
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesReset(indices []string) {
|
||||
require.NoError(trc.t, trc.tc.IndicesReset(trc.t.Context(), indices))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesRefresh(indices []string, ignore []int) {
|
||||
require.NoError(trc.t, trc.tc.IndicesRefresh(trc.t.Context(), indices, ignore))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) {
|
||||
require.NoError(trc.t, trc.tc.IndicesCreate(trc.t.Context(), index, body))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesDelete(indices []string) {
|
||||
require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) {
|
||||
count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body)
|
||||
|
||||
switch {
|
||||
case expected <= 0:
|
||||
require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count)
|
||||
default:
|
||||
require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count)
|
||||
require.NoError(trc.t, err, "expected indices to have documents, but got an error")
|
||||
}
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) {
|
||||
require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) Update(index, id string, body io.Reader) {
|
||||
require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits {
|
||||
hits, err := trc.tc.Search(trc.t.Context(), index, body)
|
||||
require.NoError(trc.t, err)
|
||||
return hits
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package opensearchtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/modules/opensearch"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/config"
|
||||
"github.com/qsfera/server/services/search/pkg/config/defaults"
|
||||
"github.com/qsfera/server/services/search/pkg/config/parser"
|
||||
)
|
||||
|
||||
const (
|
||||
openSearchImage = "opensearchproject/opensearch:2"
|
||||
)
|
||||
|
||||
func SetupTests(ctx context.Context) (*config.Config, func(), error) {
|
||||
cfg, err := setupDefaultConfig()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to setup default configuration: %w", err)
|
||||
}
|
||||
|
||||
cleanupOpenSearch, err := setupOpenSearchTestContainer(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to setup OpenSearch test container: %w", err)
|
||||
}
|
||||
|
||||
return cfg, cleanupOpenSearch, nil
|
||||
}
|
||||
|
||||
func setupDefaultConfig() (*config.Config, error) {
|
||||
cfg := defaults.DefaultConfig()
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
{ // parser.Validate requires these fields to be set even if they are not used in these tests.
|
||||
cfg.TokenManager.JWTSecret = "any-jwt"
|
||||
cfg.ServiceAccount.ServiceAccountID = "any-service-account-id"
|
||||
cfg.ServiceAccount.ServiceAccountSecret = "any-service-account-secret"
|
||||
}
|
||||
|
||||
if err := parser.ParseConfig(cfg); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse default configuration: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func(), error) {
|
||||
usesExternalOpensearch := slices.Contains([]bool{
|
||||
os.Getenv("CI") == "woodpecker",
|
||||
os.Getenv("CI_SYSTEM_NAME") == "woodpecker",
|
||||
os.Getenv("USE_TESTCONTAINERS") == "false",
|
||||
}, true)
|
||||
if usesExternalOpensearch {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
containerName := fmt.Sprintf("qsfera/test/%s", openSearchImage)
|
||||
containerName = strings.Replace(containerName, "/", "__", -1)
|
||||
containerName = strings.Replace(containerName, ":", "_", -1)
|
||||
container, err := opensearch.Run(
|
||||
ctx,
|
||||
openSearchImage,
|
||||
opensearch.WithUsername(cfg.Engine.OpenSearch.Client.Username),
|
||||
opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password),
|
||||
testcontainers.WithName(containerName),
|
||||
testcontainers.WithReuseByName(containerName),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForLog("ML configuration initialized successfully").
|
||||
WithStartupTimeout(5*time.Second),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start OpenSearch container: %w", err)
|
||||
}
|
||||
|
||||
address, err := container.Address(ctx)
|
||||
if err != nil {
|
||||
_ = container.Terminate(ctx) // attempt to clean up the container
|
||||
return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err)
|
||||
}
|
||||
|
||||
// Ensure the address is set in the default configuration.
|
||||
cfg.Engine.OpenSearch.Client.Addresses = []string{address}
|
||||
|
||||
return func() {
|
||||
err := container.Terminate(ctx)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err)
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package opensearchtest
|
||||
|
||||
type TableTest[G any, W any] struct {
|
||||
Name string
|
||||
Got G
|
||||
Want W
|
||||
Err error
|
||||
Skip bool
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package opensearchtest
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/qsfera/server/services/search/pkg/search"
|
||||
)
|
||||
|
||||
//go:embed testdata/*.json
|
||||
var testdata embed.FS
|
||||
|
||||
var Testdata = struct {
|
||||
Resources resourceTestdata
|
||||
}{
|
||||
Resources: resourceTestdata{
|
||||
Root: fromTestData[search.Resource]("resource_root.json"),
|
||||
Folder: fromTestData[search.Resource]("resource_folder.json"),
|
||||
File: fromTestData[search.Resource]("resource_file.json"),
|
||||
},
|
||||
}
|
||||
|
||||
type resourceTestdata struct {
|
||||
Root search.Resource
|
||||
File search.Resource
|
||||
Folder search.Resource
|
||||
}
|
||||
|
||||
func fromTestData[D any](name string) D {
|
||||
name = path.Join("./testdata", name)
|
||||
data, err := testdata.ReadFile(name)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to read testdata file %s: %v", name, err))
|
||||
}
|
||||
|
||||
var d D
|
||||
if json.Unmarshal(data, &d) != nil {
|
||||
panic(fmt.Sprintf("failed to unmarshal testdata %s: %v", name, err))
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"ID" : "1$1!3",
|
||||
"RootID" : "1$1!1",
|
||||
"ParentID" : "1$1!2",
|
||||
"Path" : "./parent d!r/child.jpg",
|
||||
"Type" : 1,
|
||||
"Title" : "dumme title",
|
||||
"Name" : "dummy name",
|
||||
"Content" : "dummy content",
|
||||
"Size" : 42,
|
||||
"Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251",
|
||||
"MimeType" : "image/jpeg",
|
||||
"Tags" : [ "dummy" ],
|
||||
"Deleted" : false,
|
||||
"Hidden" : false,
|
||||
"audio" : {
|
||||
"album" : "Some Album",
|
||||
"albumArtist" : "Some AlbumArtist",
|
||||
"artist" : "Some Artist",
|
||||
"bitrate" : 192,
|
||||
"composers" : "Some Composers",
|
||||
"copyright" : "",
|
||||
"disc" : 2,
|
||||
"discCount" : 5,
|
||||
"duration" : 225000,
|
||||
"genre" : "Some Genre",
|
||||
"hasDrm" : false,
|
||||
"isVariableBitrate" : true,
|
||||
"title" : "Some Title",
|
||||
"track" : 34,
|
||||
"trackCount" : 99,
|
||||
"year" : 2004
|
||||
},
|
||||
"image" : {
|
||||
"height" : 100,
|
||||
"width" : 100
|
||||
},
|
||||
"location" : {
|
||||
"altitude" : 1047.7,
|
||||
"latitude" : 49.48675890884328,
|
||||
"longitude" : 11.103870357204285
|
||||
},
|
||||
"photo" : {
|
||||
"cameraMake" : "CameraMake",
|
||||
"cameraModel" : "CameraMake",
|
||||
"exposureDenominator" : 1,
|
||||
"exposureNumerator" : 1,
|
||||
"fNumber" : 1,
|
||||
"focalLength" : 1,
|
||||
"iso" : 1,
|
||||
"orientation" : 1,
|
||||
"takenDateTime" : "2018-01-01T12:34:56Z"
|
||||
}
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"ID" : "1$1!2",
|
||||
"RootID" : "1$1!1",
|
||||
"ParentID" : "1$1!1",
|
||||
"Path" : "./parent d!r",
|
||||
"Type" : 2
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"ID" : "1$1!1",
|
||||
"RootID" : "1$1!1",
|
||||
"Path" : "."
|
||||
}
|
||||
Reference in New Issue
Block a user