sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhancement(search): implement kql to os dsl range-query

Browse Source
This commit is contained in:
fschade
2025-08-28 09:30:59 +02:00
parent 48705c79f6
commit d4183807dc
13 changed files with 114 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/search/pkg/opensearch/engine_convert_test.go
+2 -2
View File
@@ -18,7 +18,7 @@ func TestSearchHitToSearchMessageMatch(t *testing.T) {
hit := opensearchgoAPI.SearchHit{
Score: 1.1,
Source: json.RawMessage(opensearchtest.ToJSON(t, resource)),
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
}
match, err := opensearch.SearchHitToSearchMessageMatch(hit)
assert.NoError(t, err)
@@ -32,6 +32,6 @@ func TestSearchHitToSearchMessageMatch(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
assert.JSONEq(t, opensearchtest.ToJSON(t, audio), opensearchtest.ToJSON(t, match.Entity.Audio))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
})
}
services/search/pkg/opensearch/engine_test.go
+7 -7
View File
@@ -37,7 +37,7 @@ func TestEngine_Search(t *testing.T) {
defer tc.Require.IndicesDelete([]string{index})
document := opensearchtest.Testdata.Resources.Full
tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document))
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1)
engine, err := opensearch.NewEngine(index, tc.Client())
@@ -58,7 +58,7 @@ func TestEngine_Search(t *testing.T) {
deletedDocument.ID = "1$2!4"
deletedDocument.Deleted = true
tc.Require.DocumentCreate(index, deletedDocument.ID, opensearchtest.ToJSON(t, deletedDocument))
tc.Require.DocumentCreate(index, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument))
tc.Require.IndicesCount([]string{index}, "", 2)
resp, err := engine.Search(t.Context(), &searchService.SearchIndexRequest{
@@ -105,7 +105,7 @@ func TestEngine_Delete(t *testing.T) {
t.Run("mark document as deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.Full
tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document))
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1)
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery(
@@ -133,7 +133,7 @@ func TestEngine_Restore(t *testing.T) {
t.Run("mark document as not deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.Full
document.Deleted = true
tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document))
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1)
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery(
@@ -160,7 +160,7 @@ func TestEngine_Purge(t *testing.T) {
t.Run("purge with full document", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.Full
tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document))
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1)
require.NoError(t, engine.Purge(document.ID))
@@ -182,14 +182,14 @@ func TestEngine_DocCount(t *testing.T) {
t.Run("ignore deleted documents", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.Full
tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document))
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1)
count, err := engine.DocCount()
require.NoError(t, err)
require.Equal(t, uint64(1), count)
tc.Require.Update(index, document.ID, opensearchtest.ToJSON(t, map[string]any{
tc.Require.Update(index, document.ID, opensearchtest.JSONMustMarshal(t, map[string]any{
"doc": map[string]any{
"Deleted": true,
},
services/search/pkg/opensearch/internal/test/helper.go
+22
View File
@@ -0,0 +1,22 @@
package opensearchtest
import (
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/require"
)
var TimeMustParse = func(t *testing.T, ts string) time.Time {
tp, err := time.Parse(time.RFC3339Nano, ts)
require.NoError(t, err, "failed to parse time %s", ts)
return tp
}
func JSONMustMarshal(t *testing.T, data any) string {
jsonData, err := json.Marshal(data)
require.NoError(t, err, "failed to marshal data to JSON")
return string(jsonData)
}
services/search/pkg/opensearch/internal/test/test.go
-13
View File
@@ -1,21 +1,8 @@
package opensearchtest
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
)
type TableTest[G any, W any] struct {
Name string
Got G
Want W
Err error
}
func ToJSON(t *testing.T, data any) string {
jsonData, err := json.Marshal(data)
require.NoError(t, err, "failed to marshal data to JSON")
return string(jsonData)
}
services/search/pkg/opensearch/kql.go
+22
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
@@ -133,6 +134,27 @@ func (k *KQL) getBuilder(node ast.Node) (Builder, error) {
default:
builder = NewMatchPhraseQuery(k.getFieldName(node.Key)).Query(node.Value)
}
case *ast.DateTimeNode:
if node.Operator == nil {
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
q := NewRangeQuery[time.Time](k.getFieldName(node.Key))
switch node.Operator.Value {
case ">":
q.Gt(node.Value)
case ">=":
q.Gte(node.Value)
case "<":
q.Lt(node.Value)
case "<=":
q.Lte(node.Value)
default:
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
return q, nil
case *ast.GroupNode:
group, err := k.compile(node.Nodes)
if err != nil {
services/search/pkg/opensearch/kql_test.go
+54 -1
View File
@@ -2,6 +2,7 @@ package opensearch_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
@@ -85,6 +86,58 @@ func TestKQL_Compile(t *testing.T) {
},
Want: opensearch.NewTermQuery[string]("Name").Value("any"),
},
{
Name: "range query >",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query >=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
// kql to os dsl - structure tests
{
Name: "[*]",
@@ -253,7 +306,7 @@ func TestKQL_Compile(t *testing.T) {
dsl, err := compiler.Compile(test.Got)
assert.NoError(t, err)
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, dsl))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
})
}
}
services/search/pkg/opensearch/os_dsl_query_bool_test.go
+1 -1
View File
@@ -152,7 +152,7 @@ func TestBoolQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go
+1 -1
View File
@@ -66,7 +66,7 @@ func TestNewMatchPhraseQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go
+1 -1
View File
@@ -30,7 +30,7 @@ func TestIDsQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go
+1 -1
View File
@@ -49,7 +49,7 @@ func TestTermQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go
+1 -1
View File
@@ -38,7 +38,7 @@ func TestWildcardQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_query_test.go
+1 -1
View File
@@ -28,7 +28,7 @@ func TestQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
services/search/pkg/opensearch/os_dsl_test.go
+1 -1
View File
@@ -30,7 +30,7 @@ func TestBuilderToBoolQuery(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, opensearch.BuilderToBoolQuery(test.Got)))
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, opensearch.BuilderToBoolQuery(test.Got)))
})
}
}