Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,251 @@
package opensearch
import (
"context"
"fmt"
"strings"
"time"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/qsfera/server/pkg/conversions"
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
searchService "github.com/qsfera/server/protogen/gen/qsfera/services/search/v0"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/search"
)
const defaultBatchSize = 50
var (
ErrUnhealthyCluster = fmt.Errorf("cluster is not healthy")
)
type Backend struct {
index string
client *opensearchgoAPI.Client
}
func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) {
pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{})
switch {
case err != nil:
return nil, fmt.Errorf("%w, failed to ping opensearch: %w", ErrUnhealthyCluster, err)
case pingResp.IsError():
return nil, fmt.Errorf("%w, failed to ping opensearch", ErrUnhealthyCluster)
}
// apply the index template
if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil {
return nil, fmt.Errorf("failed to apply index template: %w", err)
}
// first check if the cluster is healthy
resp, err := client.Cluster.Health(context.TODO(), &opensearchgoAPI.ClusterHealthReq{
Indices: []string{index},
Params: opensearchgoAPI.ClusterHealthParams{
Local: opensearchgoAPI.ToPointer(true),
Timeout: 5 * time.Second,
},
})
switch {
case err != nil:
return nil, fmt.Errorf("%w, failed to get cluster health: %w", ErrUnhealthyCluster, err)
case resp.TimedOut:
return nil, fmt.Errorf("%w, cluster health request timed out", ErrUnhealthyCluster)
case resp.Status != "green" && resp.Status != "yellow":
return nil, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status)
}
return &Backend{index: index, client: client}, nil
}
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
if err != nil {
return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
}
// filter out deleted resources
boolQuery.Filter(
osu.NewTermQuery[bool]("Deleted").Value(false),
)
if sir.Ref != nil {
// if a reference is provided, filter by the root ID
boolQuery.Filter(
osu.NewTermQuery[string]("RootID").Value(
storagespace.FormatResourceID(
&storageProvider.ResourceId{
StorageId: sir.Ref.GetResourceId().GetStorageId(),
SpaceId: sir.Ref.GetResourceId().GetSpaceId(),
OpaqueId: sir.Ref.GetResourceId().GetOpaqueId(),
},
),
),
)
}
searchParams := opensearchgoAPI.SearchParams{
SourceExcludes: []string{"Content"}, // Do not send back the full content in the search response, as it is only needed for highlighting and can be large. The highlighted snippets will be sent back in the response instead.
}
switch {
case sir.PageSize == -1:
searchParams.Size = conversions.ToPointer(1000)
case sir.PageSize == 0:
searchParams.Size = conversions.ToPointer(200)
default:
searchParams.Size = conversions.ToPointer(int(sir.PageSize))
}
req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
Indices: []string{b.index},
Params: searchParams,
},
boolQuery,
osu.SearchBodyParams{
Highlight: &osu.BodyParamHighlight{
HighlightOptions: osu.HighlightOptions{
NumberOfFragments: 2,
PreTags: []string{"<mark>"},
PostTags: []string{"</mark>"},
},
Fields: map[string]osu.HighlightOptions{
"Content": {
Type: osu.HighlightTypeFvh,
},
},
},
},
)
if err != nil {
return nil, fmt.Errorf("failed to build search request: %w", err)
}
resp, err := b.client.Search(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to search: %w", err)
}
matches := make([]*searchMessage.Match, 0, len(resp.Hits.Hits))
totalMatches := resp.Hits.Total.Value
for _, hit := range resp.Hits.Hits {
match, err := convert.OpenSearchHitToMatch(hit)
if err != nil {
return nil, fmt.Errorf("failed to convert hit to match: %w", err)
}
if sir.Ref != nil {
hitPath := strings.TrimSuffix(match.GetEntity().GetRef().GetPath(), "/")
requestedPath := utils.MakeRelativePath(sir.Ref.Path)
isRoot := hitPath == requestedPath
if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") {
totalMatches--
continue
}
}
matches = append(matches, match)
}
return &searchService.SearchIndexResponse{
Matches: matches,
TotalMatches: int32(totalMatches),
}, nil
}
func (b *Backend) DocCount() (uint64, error) {
req, err := osu.BuildIndicesCountReq(
&opensearchgoAPI.IndicesCountReq{
Indices: []string{b.index},
},
osu.NewTermQuery[bool]("Deleted").Value(false),
)
if err != nil {
return 0, fmt.Errorf("failed to build count request: %w", err)
}
resp, err := b.client.Indices.Count(context.TODO(), req)
if err != nil {
return 0, fmt.Errorf("failed to count documents: %w", err)
}
return uint64(resp.Count), nil
}
func (b *Backend) Upsert(id string, r search.Resource) error {
batch, err := b.NewBatch(defaultBatchSize)
if err != nil {
return err
}
if err := batch.Upsert(id, r); err != nil {
return err
}
return batch.Push()
}
func (b *Backend) Move(id string, parentID string, target string) error {
batch, err := b.NewBatch(defaultBatchSize)
if err != nil {
return err
}
if err := batch.Move(id, parentID, target); err != nil {
return err
}
return batch.Push()
}
func (b *Backend) Delete(id string) error {
batch, err := b.NewBatch(defaultBatchSize)
if err != nil {
return err
}
if err := batch.Delete(id); err != nil {
return err
}
return batch.Push()
}
func (b *Backend) Restore(id string) error {
batch, err := b.NewBatch(defaultBatchSize)
if err != nil {
return err
}
if err := batch.Restore(id); err != nil {
return err
}
return batch.Push()
}
func (b *Backend) Purge(id string, onlyDeleted bool) error {
batch, err := b.NewBatch(defaultBatchSize)
if err != nil {
return err
}
if err := batch.Purge(id, onlyDeleted); err != nil {
return err
}
return batch.Push()
}
func (b *Backend) NewBatch(size int) (search.BatchOperator, error) {
return NewBatch(b.client, b.index, size)
}
@@ -0,0 +1,283 @@
package opensearch_test
import (
"fmt"
"strings"
"testing"
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/require"
searchService "github.com/qsfera/server/protogen/gen/qsfera/services/search/v0"
"github.com/qsfera/server/services/search/pkg/opensearch"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
"github.com/qsfera/server/services/search/pkg/search"
)
func TestNewBackend(t *testing.T) {
t.Run("fails to create if the cluster is not healthy", func(t *testing.T) {
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
Client: opensearchgo.Config{
Addresses: []string{"http://localhost:1025"},
},
})
require.NoError(t, err, "failed to create OpenSearch client")
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
require.Nil(t, backend)
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
})
}
func TestEngine_Search(t *testing.T) {
indexName := "qsfera-test-engine-search"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
t.Run("most simple search", func(t *testing.T) {
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`"%s"`, document.Name),
})
require.NoError(t, err)
require.Len(t, resp.Matches, 1)
require.Equal(t, int32(1), resp.TotalMatches)
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
})
t.Run("ignores files that are marked as deleted", func(t *testing.T) {
deletedDocument := opensearchtest.Testdata.Resources.File
deletedDocument.ID = "1$2!4"
deletedDocument.Deleted = true
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, deletedDocument)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`"%s"`, document.Name),
})
require.NoError(t, err)
require.Len(t, resp.Matches, 1)
require.Equal(t, int32(1), resp.TotalMatches)
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
})
}
func TestEngine_Upsert(t *testing.T) {
indexName := "qsfera-test-engine-upsert"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("upsert with full document", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
require.NoError(t, backend.Upsert(document.ID, document))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
})
}
func TestEngine_Move(t *testing.T) {
indexName := "qsfera-test-engine-move"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("moves the document to a new path", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(t, map[string]any{
"query": map[string]any{
"ids": map[string]any{
"values": []string{document.ID},
},
},
})
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
require.Len(t, resources, 1)
require.Equal(t, document.Path, resources[0].Path)
document.Path = "./new/path/to/resource"
require.NoError(t, backend.Move(document.ID, document.ParentID, document.Path))
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
require.Len(t, resources, 1)
require.Equal(t, document.Path, resources[0].Path)
})
}
func TestEngine_Delete(t *testing.T) {
indexName := "qsfera-test-engine-delete"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("mark document as deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(t, map[string]any{
"query": map[string]any{
"term": map[string]any{
"Deleted": map[string]any{
"value": true,
},
},
},
})
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
require.NoError(t, backend.Delete(document.ID))
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
})
}
func TestEngine_Restore(t *testing.T) {
indexName := "qsfera-test-engine-restore"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("mark document as not deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
document.Deleted = true
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(t, map[string]any{
"query": map[string]any{
"term": map[string]any{
"Deleted": map[string]any{
"value": true,
},
},
},
})
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
require.NoError(t, backend.Restore(document.ID))
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
})
}
func TestEngine_Purge(t *testing.T) {
indexName := "qsfera-test-engine-purge"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("purge with full document", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
require.NoError(t, backend.Purge(document.ID, false))
tc.Require.IndicesCount([]string{indexName}, nil, 0)
})
t.Run("purge resource trees", func(t *testing.T) {
resourceFolder := opensearchtest.Testdata.Resources.Folder
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
resourceFile := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
require.NoError(t, backend.Purge(resourceFolder.ID, false))
tc.Require.IndicesCount([]string{indexName}, nil, 0)
})
t.Run("purge resource trees and ignores undeleted resources", func(t *testing.T) {
resourceFolder := opensearchtest.Testdata.Resources.Folder
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
resourceFile := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
require.NoError(t, backend.Delete(resourceFile.ID))
tc.Require.IndicesRefresh([]string{indexName}, nil)
require.NoError(t, backend.Purge(resourceFolder.ID, true))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
})
}
func TestEngine_DocCount(t *testing.T) {
indexName := "qsfera-test-engine-doc-count"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("ignore deleted documents", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
count, err := backend.DocCount()
require.NoError(t, err)
require.Equal(t, uint64(1), count)
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, map[string]any{
"doc": map[string]any{
"Deleted": true,
},
})))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
count, err = backend.DocCount()
require.NoError(t, err)
require.Equal(t, uint64(0), count)
})
}
@@ -0,0 +1,248 @@
package opensearch
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/utils"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/qsfera/server/pkg/conversions"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/search"
)
var _ search.BatchOperator = (*Batch)(nil) // ensure Batch implements BatchOperator
type Batch struct {
client *opensearchgoAPI.Client
index string
size int
log log.Logger
operations []any
mu sync.Mutex
}
func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, error) {
if size <= 0 {
return nil, errors.New("batch size must be greater than 0")
}
return &Batch{
client: client,
size: size,
index: index,
}, nil
}
func (b *Batch) Upsert(id string, r search.Resource) error {
return b.withSizeLimit(func() error {
body, err := conversions.To[map[string]any](r)
if err != nil {
return fmt.Errorf("failed to marshal resource: %w", err)
}
op := func() []map[string]any {
return []map[string]any{
{"index": map[string]any{"_index": b.index, "_id": id}},
body,
}
}
b.mu.Lock()
b.operations = append(b.operations, op)
b.mu.Unlock()
return nil
})
}
func (b *Batch) Move(id, parentID, location string) error {
return b.withSizeLimit(func() error {
op := func() error {
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
return &osu.BodyParamScript{
Source: `
if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; }
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath)
`,
Lang: "painless",
Params: map[string]any{
"id": id,
"parentID": parentID,
"oldPath": rootResource.Path,
"newPath": utils.MakeRelativePath(location),
"newName": path.Base(utils.MakeRelativePath(location)),
},
}
})
}
b.mu.Lock()
b.operations = append(b.operations, op)
b.mu.Unlock()
return nil
})
}
func (b *Batch) Delete(id string) error {
return b.withSizeLimit(func() error {
op := func() error {
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(_ search.Resource) *osu.BodyParamScript {
return &osu.BodyParamScript{
Source: "ctx._source.Deleted = params.deleted",
Lang: "painless",
Params: map[string]any{
"deleted": true,
},
}
})
}
b.mu.Lock()
b.operations = append(b.operations, op)
b.mu.Unlock()
return nil
})
}
func (b *Batch) Restore(id string) error {
return b.withSizeLimit(func() error {
op := func() error {
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(_ search.Resource) *osu.BodyParamScript {
return &osu.BodyParamScript{
Source: "ctx._source.Deleted = params.deleted",
Lang: "painless",
Params: map[string]any{
"deleted": false,
},
}
})
}
b.mu.Lock()
b.operations = append(b.operations, op)
b.mu.Unlock()
return nil
})
}
func (b *Batch) Purge(id string, onlyDeleted bool) error {
return b.withSizeLimit(func() error {
resource, err := searchResourceByID(context.Background(), b.client, b.index, id)
if err != nil {
return fmt.Errorf("failed to get resource: %w", err)
}
query := osu.NewBoolQuery().Must(osu.NewTermQuery[string]("Path").Value(resource.Path))
if onlyDeleted {
query.Must(osu.NewTermQuery[bool]("Deleted").Value(true))
}
req, err := osu.BuildDocumentDeleteByQueryReq(
opensearchgoAPI.DocumentDeleteByQueryReq{
Indices: []string{b.index},
Params: opensearchgoAPI.DocumentDeleteByQueryParams{
WaitForCompletion: conversions.ToPointer(true),
},
},
query,
)
if err != nil {
return fmt.Errorf("failed to build delete by query request: %w", err)
}
op := func() error {
resp, err := b.client.Document.DeleteByQuery(context.TODO(), req)
switch {
case err != nil:
return fmt.Errorf("failed to delete by query: %w", err)
case len(resp.Failures) != 0:
return fmt.Errorf("failed to delete by query, failures: %v", resp.Failures)
}
return nil
}
b.mu.Lock()
b.operations = append(b.operations, op)
b.mu.Unlock()
return nil
})
}
func (b *Batch) Push() error {
b.mu.Lock()
defer b.mu.Unlock()
defer func() { // cleanup
b.operations = nil
}()
var bulkOperations []map[string]any
pushBulkOperations := func() error {
if len(bulkOperations) == 0 {
return nil
}
var body strings.Builder
for _, operation := range bulkOperations {
part, err := json.Marshal(operation)
if err != nil {
return fmt.Errorf("failed to marshal bulk operation: %w", err)
}
body.Write(part)
body.WriteString("\n")
}
if _, err := b.client.Bulk(context.Background(), opensearchgoAPI.BulkReq{
Body: strings.NewReader(body.String()),
}); err != nil {
return fmt.Errorf("failed to execute bulk operations: %w", err)
}
bulkOperations = nil
return nil
}
// keep the order of operations in the batch intact,
// unfortunately, operations like DeleteByQuery cannot be part of the bulk API,
// so we need to push the previous bulk operations before executing such operations
// this might lead to smaller bulks than the configured size, but ensures correct order
for _, operation := range b.operations {
switch op := operation.(type) {
case func() []map[string]any:
bulkOperations = append(bulkOperations, op()...)
case func() error:
if err := pushBulkOperations(); err != nil {
return fmt.Errorf("failed to push operations: %w", err)
}
if err := op(); err != nil {
return fmt.Errorf("failed to execute operation: %w", err)
}
}
}
return pushBulkOperations()
}
func (b *Batch) withSizeLimit(f func() error) error {
if err := f(); err != nil {
return err
}
if len(b.operations) >= b.size {
return b.Push()
}
return nil
}
@@ -0,0 +1,143 @@
package opensearch
import (
"bytes"
"context"
"embed"
"errors"
"fmt"
"path"
"reflect"
"github.com/go-jose/go-jose/v3/json"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/tidwall/gjson"
)
var (
ErrManualActionRequired = errors.New("manual action required")
IndexManagerLatest = IndexIndexManagerResourceV2
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
IndexIndexManagerResourceV2 IndexManager = "resource_v2.json"
)
//go:embed internal/indexes/*.json
var indexes embed.FS
type IndexManager string
func (m IndexManager) String() string {
b, err := m.MarshalJSON()
if err != nil {
return ""
}
return string(b)
}
func (m IndexManager) MarshalJSON() ([]byte, error) {
filePath := string(m)
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
case len(body) <= 0:
return nil, fmt.Errorf("index file %s is empty", filePath)
}
return body, nil
}
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error {
localIndexB, err := m.MarshalJSON()
if err != nil {
return fmt.Errorf("failed to marshal index %s: %w", name, err)
}
indicesExistsResp, err := client.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
Indices: []string{name},
})
switch {
case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404:
break
case err != nil:
return fmt.Errorf("failed to check if index %s exists: %w", name, err)
case indicesExistsResp == nil:
return fmt.Errorf("indicesExistsResp is nil for index %s", name)
}
if indicesExistsResp.StatusCode == 200 {
resp, err := client.Indices.Get(ctx, opensearchgoAPI.IndicesGetReq{
Indices: []string{name},
})
if err != nil {
return fmt.Errorf("failed to get index %s: %w", name, err)
}
remoteIndex, ok := resp.Indices[name]
if !ok {
return fmt.Errorf("index %s not found in response", name)
}
remoteIndexB, err := json.Marshal(remoteIndex)
if err != nil {
return fmt.Errorf("failed to marshal index %s: %w", name, err)
}
localIndexJson := gjson.ParseBytes(localIndexB)
remoteIndexJson := gjson.ParseBytes(remoteIndexB)
compare := func(lvPath, rvPath string) (any, any, bool) {
lv := localIndexJson.Get(lvPath).Raw
rv := remoteIndexJson.Get(rvPath).Raw
var lvv, rvv any
if err := json.Unmarshal([]byte(lv), &lvv); err != nil {
return nil, nil, false
}
if err := json.Unmarshal([]byte(rv), &rvv); err != nil {
return nil, nil, false
}
return lv, rv, reflect.DeepEqual(lvv, rvv)
}
var errs []error
for k := range localIndexJson.Get("settings").Map() {
if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok {
errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv))
}
}
for k := range localIndexJson.Get("mappings.properties").Map() {
if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok {
errs = append(errs, fmt.Errorf("mappings.properties.%s", k))
}
}
if errs != nil {
return fmt.Errorf(
"index %s already exists and is different from the requested version, %w: %w",
name,
ErrManualActionRequired,
errors.Join(errs...),
)
}
return nil // Index is already up to date, no action needed
}
createResp, err := client.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
Index: name,
Body: bytes.NewReader(localIndexB),
})
switch {
case err != nil:
return fmt.Errorf("failed to create index %s: %w", name, err)
case !createResp.Acknowledged:
return fmt.Errorf("failed to create index %s: not acknowledged", name)
}
return nil
}
@@ -0,0 +1,63 @@
package opensearch_test
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
"github.com/tidwall/sjson"
"github.com/qsfera/server/services/search/pkg/opensearch"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestIndexManager(t *testing.T) {
t.Run("index plausibility", func(t *testing.T) {
tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{
{
Name: "empty",
Got: opensearch.IndexManagerLatest,
},
}
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
indexName := "qsfera-test-resource"
tc.Require.IndicesReset([]string{indexName})
body, err := test.Got.MarshalJSON()
require.NoError(t, err)
require.NotEmpty(t, body)
require.NotEmpty(t, test.Got.String())
require.JSONEq(t, test.Got.String(), string(body))
require.NoError(t, test.Got.Apply(t.Context(), indexName, tc.Client()))
})
}
})
t.Run("does not create index if it already exists and is up to date", func(t *testing.T) {
indexManager := opensearch.IndexManagerLatest
indexName := "qsfera-test-resource"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCreate(indexName, strings.NewReader(indexManager.String()))
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client()))
})
t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) {
indexManager := opensearch.IndexManagerLatest
indexName := "qsfera-test-resource"
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
body, err := sjson.Set(indexManager.String(), "settings.number_of_shards", "2")
require.NoError(t, err)
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired)
})
}
@@ -0,0 +1,197 @@
package convert
import (
"fmt"
"reflect"
"slices"
"strings"
"github.com/qsfera/server/pkg/ast"
)
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
return kqlExpander{}.expand(nodes, "")
}
type kqlExpander struct{}
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
for i, node := range nodes {
rnode := reflect.ValueOf(node)
// we need to ensure that the node is a pointer to an ast.Node in every case
if rnode.Kind() != reflect.Ptr {
ptr := reflect.New(rnode.Type())
ptr.Elem().Set(rnode)
rnode = ptr
cnode, ok := rnode.Interface().(ast.Node)
if !ok {
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
}
node = cnode // Update the original node to the pointer
nodes[i] = node // Update the original slice with the pointer
}
var unfoldedNodes []ast.Node
switch cnode := node.(type) {
case *ast.GroupNode:
if cnode.Key != "" { // group nodes should not get a default key
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
if err != nil {
return nil, err
}
cnode.Nodes = groupNodes
case *ast.StringNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
case *ast.DateTimeNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
case *ast.BooleanNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
if unfoldedNodes != nil {
// Insert unfolded nodes at the current index
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
// Adjust index to account for new nodes
i += len(unfoldedNodes) - 1
}
}
return nodes, nil
}
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
if defaultKey == "" {
defaultKey = "Name" // Set a default key if none is provided
}
key, ok := map[string]string{
"": defaultKey, // Default case if current is empty
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
}[current]
if !ok {
return current // Return the original key if not found
}
return key
}
func (_ kqlExpander) lowerValue(key, value string) string {
if slices.Contains([]string{"Hidden"}, key) {
return value // ignore certain keys and return the original value
}
return strings.ToLower(value)
}
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
result, ok := map[string][]ast.Node{
"MimeType:file": {
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:folder": {
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:document": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
}},
},
"MimeType:spreadsheet": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
}},
},
"MimeType:presentation": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
}},
},
"MimeType:pdf": {
&ast.StringNode{Key: key, Value: "application/pdf"},
},
"MimeType:image": {
&ast.StringNode{Key: key, Value: "image/*"},
},
"MimeType:video": {
&ast.StringNode{Key: key, Value: "video/*"},
},
"MimeType:audio": {
&ast.StringNode{Key: key, Value: "audio/*"},
},
"MimeType:archive": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tgz"},
}},
},
}[fmt.Sprintf("%s:%s", key, value)]
if !ok {
return nil
}
return result
}
@@ -0,0 +1,607 @@
package convert_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestExpandKQLAST(t *testing.T) {
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "ast.node.V -> ast.node.PTR",
Got: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: "d"},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: "g"},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
ast.GroupNode{Key: "i", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: "d"},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: "g"},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("remaps some keys", func(t *testing.T) {
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
for k, v := range map[string]string{
"": "Name", // Default to "Name" if no key is provided
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"any": "any", // Example of an unknown key that should remain unchanged
} {
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
Name: fmt.Sprintf("%s -> %s", k, v),
Got: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: k},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: k},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: k},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: k},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
}},
}},
}},
},
})
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("lowercases some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "!Hidden: StringNode -> stringnode",
Got: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "stringnode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "stringnode"},
}},
},
},
{
Name: "Hidden: StringNode -> StringNode",
Got: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "StringNode"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "StringNode"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("unfolds some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "MimeType:unknown",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:file",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "file"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:folder",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "folder"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:document",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "document"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:spreadsheet",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:presentation",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "presentation"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:pdf",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "pdf"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:image",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "image"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "image/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:video",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "video"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "video/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:audio",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "audio"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:archive",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "archive"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
t.Run("different cases", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "use the group node key as default key",
Got: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
&ast.StringNode{Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
Want: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
}
@@ -0,0 +1,35 @@
package convert
import (
"fmt"
"github.com/qsfera/server/pkg/kql"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
)
var (
ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
)
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
kqlAst, err := kql.Builder{}.Build(kqlQuery)
if err != nil {
return nil, fmt.Errorf("failed to build query: %w", err)
}
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
}
builder, err := TranspileKQLToOpenSearch(kqlNodes)
if err != nil {
return nil, fmt.Errorf("failed to compile query: %w", err)
}
if q, ok := builder.(*osu.BoolQuery); !ok {
return osu.NewBoolQuery().Must(builder), nil
} else {
return q, nil
}
}
@@ -0,0 +1 @@
package convert_test
@@ -0,0 +1,147 @@
package convert
import (
"errors"
"fmt"
"strings"
"time"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/pkg/kql"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
)
func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) {
return kqlOpensearchTranspiler{}.Transpile(nodes)
}
type kqlOpensearchTranspiler struct{}
func (t kqlOpensearchTranspiler) Transpile(nodes []ast.Node) (osu.Builder, error) {
q, err := t.transpile(nodes)
if err != nil {
return nil, err
}
return q, nil
}
func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error) {
if len(nodes) == 0 {
return nil, fmt.Errorf("no nodes to compile")
}
if len(nodes) == 1 {
builder, err := t.toBuilder(nodes[0])
if err != nil {
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
}
return builder, nil
}
boolQueryParams := &osu.BoolQueryParams{}
boolQuery := osu.NewBoolQuery().Params(boolQueryParams)
boolQueryAdd := boolQuery.Must
for i, node := range nodes {
nextOp := t.getOperatorValueAt(nodes, i+1)
prevOp := t.getOperatorValueAt(nodes, i-1)
switch {
case nextOp == kql.BoolOR:
boolQueryAdd = boolQuery.Should
case nextOp == kql.BoolAND:
boolQueryAdd = boolQuery.Must
case prevOp == kql.BoolNOT:
boolQueryAdd = boolQuery.MustNot
}
builder, err := t.toBuilder(node)
switch {
// if the node is not known, we skip it, such as an operator node
case errors.Is(err, ErrUnsupportedNodeType):
continue
case err != nil:
return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err)
}
if _, ok := node.(*ast.OperatorNode); ok {
// operatorNodes are not builders, so we skip them
continue
}
if nextOp == kql.BoolOR {
// if there are should clauses, we set the minimum should match to 1
boolQueryParams.MinimumShouldMatch = 1
}
boolQueryAdd(builder)
}
return boolQuery, nil
}
func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) string {
if i < 0 || i >= len(nodes) {
return ""
}
if opn, ok := nodes[i].(*ast.OperatorNode); ok {
return opn.Value
}
return ""
}
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
var builder osu.Builder
switch node := node.(type) {
case *ast.BooleanNode:
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
case *ast.StringNode:
isWildcard := strings.Contains(node.Value, "*")
if isWildcard {
return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
}
totalTerms := strings.Split(node.Value, " ")
isSingleTerm := len(totalTerms) == 1
isMultiTerm := len(totalTerms) >= 1
switch {
case isSingleTerm:
return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
case isMultiTerm:
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
}
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
case *ast.DateTimeNode:
if node.Operator == nil {
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
case *ast.GroupNode:
group, err := t.transpile(node.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to build group: %w", err)
}
return group, nil
}
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
}
@@ -0,0 +1,378 @@
package convert_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/pkg/ast"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestTranspileKQLToOpenSearch(t *testing.T) {
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "term query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
},
{
Name: "term query - boolean node - true",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: true},
},
},
Want: osu.NewTermQuery[bool]("Deleted").Value(true),
},
{
Name: "term query - boolean node - false",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
},
},
Want: osu.NewTermQuery[bool]("Deleted").Value(false),
},
{
Name: "match-phrase query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "open cloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
},
{
Name: "wildcard query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "open*"},
},
},
Want: osu.NewWildcardQuery("Name").Value("open*"),
},
{
Name: "bool query",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "a"},
&ast.StringNode{Key: "Name", Value: "b"},
}},
},
},
Want: osu.NewBoolQuery().Must(
osu.NewTermQuery[string]("Name").Value("a"),
osu.NewTermQuery[string]("Name").Value("b"),
),
},
{
Name: "no bool query for single term",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "any"},
}},
},
},
Want: osu.NewTermQuery[string]("Name").Value("any"),
},
{
Name: "range query >",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query >=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: ">="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<"},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <=",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.DateTimeNode{
Key: "Mtime",
Operator: &ast.OperatorNode{Value: "<="},
Value: opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00"),
},
},
},
Want: osu.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
// kql to os dsl - structure tests
{
Name: "[*]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("qsfera"),
},
{
Name: "[* *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("qsfera"),
).
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR * OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "qsfera"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "44"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("qsfera"),
osu.NewTermQuery[string]("age").Value("32"),
osu.NewTermQuery[string]("age").Value("44"),
),
},
{
Name: "[* AND * OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Must(
osu.NewTermQuery[string]("a").Value("a"),
).
Should(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
},
{
Name: "[* OR * AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Must(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
).
Should(
osu.NewTermQuery[string]("a").Value("a"),
),
},
{
Name: "[* OR * AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "c", Value: "c"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
).
Must(
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
},
{
Name: "[[* OR * OR *] AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "d", Value: "d"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
osu.NewTermQuery[string]("d").Value("d"),
),
},
{
Name: "[[* OR * OR *] AND NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "b", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "c"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "d", Value: "d"},
},
},
Want: osu.NewBoolQuery().
Must(
osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("a").Value("a"),
osu.NewTermQuery[string]("b").Value("b"),
osu.NewTermQuery[string]("c").Value("c"),
),
).MustNot(
osu.NewTermQuery[string]("d").Value("d"),
),
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("skipping test: " + test.Name)
}
dsl, err := convert.TranspileKQLToOpenSearch(test.Got.Nodes)
assert.NoError(t, err)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
})
}
}
@@ -0,0 +1,99 @@
package convert
import (
"fmt"
"strings"
"time"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/qsfera/server/pkg/conversions"
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
"github.com/qsfera/server/services/search/pkg/search"
)
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
resource, err := conversions.To[search.Resource](hit.Source)
if err != nil {
return nil, fmt.Errorf("failed to convert hit source: %w", err)
}
resourceRootID, err := storagespace.ParseID(resource.RootID)
if err != nil {
return nil, err
}
resourceID, err := storagespace.ParseID(resource.ID)
if err != nil {
return nil, err
}
resourceParentID, _ := storagespace.ParseID(resource.ParentID)
match := &searchMessage.Match{
Score: hit.Score,
Entity: &searchMessage.Entity{
Ref: &searchMessage.Reference{
ResourceId: &searchMessage.ResourceID{
StorageId: resourceRootID.GetStorageId(),
SpaceId: resourceRootID.GetSpaceId(),
OpaqueId: resourceRootID.GetOpaqueId(),
},
Path: resource.Path,
},
Id: &searchMessage.ResourceID{
StorageId: resourceID.GetStorageId(),
SpaceId: resourceID.GetSpaceId(),
OpaqueId: resourceID.GetOpaqueId(),
},
Name: resource.Name,
ParentId: &searchMessage.ResourceID{
StorageId: resourceParentID.GetStorageId(),
SpaceId: resourceParentID.GetSpaceId(),
OpaqueId: resourceParentID.GetOpaqueId(),
},
Size: resource.Size,
Type: resource.Type,
MimeType: resource.MimeType,
Deleted: resource.Deleted,
Tags: resource.Tags,
Highlights: func() string {
contentHighlights, ok := hit.Highlight["Content"]
if !ok {
return ""
}
return strings.Join(contentHighlights[:], "; ")
}(),
Audio: func() *searchMessage.Audio {
if !strings.HasPrefix(resource.MimeType, "audio/") {
return nil
}
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
return audio
}(),
Image: func() *searchMessage.Image {
image, _ := conversions.To[*searchMessage.Image](resource.Image)
return image
}(),
Location: func() *searchMessage.GeoCoordinates {
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
return geoCoordinates
}(),
Photo: func() *searchMessage.Photo {
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
return photo
}(),
},
}
if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil {
match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
}
return match, nil
}
@@ -0,0 +1,38 @@
package convert_test
import (
"encoding/json"
"testing"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/pkg/conversions"
searchMessage "github.com/qsfera/server/protogen/gen/qsfera/messages/search/v0"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/convert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestOpenSearchHitToMatch(t *testing.T) {
resource := opensearchtest.Testdata.Resources.File
resource.MimeType = "audio/anything"
hit := opensearchgoAPI.SearchHit{
Score: 1.1,
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
}
match, err := convert.OpenSearchHitToMatch(hit)
assert.NoError(t, err)
assert.Equal(t, hit.Score, match.Score)
assert.Equal(t, resource.Name, match.Entity.Name)
t.Parallel()
t.Run("converts the audio field to the expected type", func(t *testing.T) {
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
assert.NoError(t, err)
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
})
}
@@ -0,0 +1,49 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
}
}
}
}
@@ -0,0 +1,56 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"Content": {
"type": "text",
"term_vector": "with_positions_offsets"
},
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
},
"Favorites": {
"type": "keyword"
}
}
}
}
@@ -0,0 +1,128 @@
package osu
import (
"encoding/json"
"fmt"
"reflect"
"dario.cat/mergo"
"github.com/qsfera/server/pkg/conversions"
)
type Builder interface {
json.Marshaler
Map() (map[string]any, error)
}
func newBase(v ...any) (map[string]any, error) {
base := make(map[string]any)
for _, value := range v {
data, err := conversions.To[map[string]any](value)
if err != nil {
return nil, fmt.Errorf("failed to convert value to map: %w", err)
}
if isEmpty(data) {
continue
}
if err := mergo.Merge(&base, data); err != nil {
return nil, fmt.Errorf("failed to merge value into base: %w", err)
}
}
return base, nil
}
func applyValue[T any](target map[string]any, key string, v T) {
if target == nil || isEmpty(key) || isEmpty(v) {
return
}
target[key] = v
}
func applyValues[T any](target map[string]any, values map[string]T) {
if target == nil || isEmpty(values) {
return
}
for k, v := range values {
applyValue[T](target, k, v)
}
}
func applyBuilder(target map[string]any, key string, builder Builder) error {
if target == nil || isEmpty(key) || isEmpty(builder) {
return nil
}
data, err := builder.Map()
if err != nil {
return fmt.Errorf("failed to map builder %s: %w", key, err)
}
if !isEmpty(data) {
target[key] = data
}
return nil
}
func applyBuilders(target map[string]any, key string, bs ...Builder) error {
if target == nil || isEmpty(key) || isEmpty(bs) {
return nil
}
builders := make([]map[string]any, 0, len(bs))
for _, builder := range bs {
data, err := builder.Map()
switch {
case err != nil:
return fmt.Errorf("failed to map builder %s: %w", key, err)
case isEmpty(data):
continue
default:
builders = append(builders, data)
}
}
if len(builders) > 0 {
target[key] = builders
}
return nil
}
func isEmpty(x any) bool {
switch {
case x == nil:
return true
case reflect.ValueOf(x).Kind() == reflect.Bool:
return false
case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()):
return true
case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0:
return true
default:
return false
}
}
func merge[T any](vals ...T) T {
base := make(map[string]any)
for _, val := range vals {
data, err := conversions.To[map[string]any](val)
if err != nil {
continue
}
_ = mergo.Merge(&base, data)
}
data, _ := conversions.To[T](base)
return data
}
@@ -0,0 +1,87 @@
package osu
import (
"encoding/json"
)
type BoolQuery struct {
must []Builder
mustNot []Builder
should []Builder
filter []Builder
params *BoolQueryParams
}
type BoolQueryParams struct {
MinimumShouldMatch int16 `json:"minimum_should_match,omitempty"`
Boost float32 `json:"boost,omitempty"`
Name string `json:"_name,omitempty"`
}
func NewBoolQuery() *BoolQuery {
return &BoolQuery{}
}
func (q *BoolQuery) Params(v *BoolQueryParams) *BoolQuery {
q.params = v
return q
}
func (q *BoolQuery) Must(v ...Builder) *BoolQuery {
q.must = append(q.must, v...)
return q
}
func (q *BoolQuery) MustNot(v ...Builder) *BoolQuery {
q.mustNot = append(q.mustNot, v...)
return q
}
func (q *BoolQuery) Should(v ...Builder) *BoolQuery {
q.should = append(q.should, v...)
return q
}
func (q *BoolQuery) Filter(v ...Builder) *BoolQuery {
q.filter = append(q.filter, v...)
return q
}
func (q *BoolQuery) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
if err := applyBuilders(base, "must", q.must...); err != nil {
return nil, err
}
if err := applyBuilders(base, "must_not", q.mustNot...); err != nil {
return nil, err
}
if err := applyBuilders(base, "should", q.should...); err != nil {
return nil, err
}
if err := applyBuilders(base, "filter", q.filter...); err != nil {
return nil, err
}
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"bool": base,
}, nil
}
func (q *BoolQuery) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,158 @@
package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestBoolQuery(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewBoolQuery(),
Want: nil,
},
{
Name: "with params",
Got: osu.NewBoolQuery().Params(&osu.BoolQueryParams{
MinimumShouldMatch: 10,
Boost: 10,
Name: "some-name",
}),
Want: map[string]any{
"bool": map[string]any{
"minimum_should_match": 10,
"boost": 10,
"_name": "some-name",
},
},
},
{
Name: "must",
Got: osu.NewBoolQuery().Must(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"must": []map[string]any{
{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
},
},
},
{
Name: "must_not",
Got: osu.NewBoolQuery().MustNot(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"must_not": []map[string]any{
{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
},
},
},
{
Name: "should",
Got: osu.NewBoolQuery().Should(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"should": []map[string]any{
{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
},
},
},
{
Name: "filter",
Got: osu.NewBoolQuery().Filter(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"filter": []map[string]any{
{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
},
},
},
{
Name: "full",
Got: osu.NewBoolQuery().
Must(osu.NewTermQuery[string]("name").Value("tom")).
MustNot(osu.NewTermQuery[bool]("deleted").Value(true)).
Should(osu.NewTermQuery[string]("gender").Value("male")).
Filter(osu.NewTermQuery[int]("age").Value(42)),
Want: map[string]any{
"bool": map[string]any{
"must": []map[string]any{
{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
"must_not": []map[string]any{
{
"term": map[string]any{
"deleted": map[string]any{
"value": true,
},
},
},
},
"should": []map[string]any{
{
"term": map[string]any{
"gender": map[string]any{
"value": "male",
},
},
},
},
"filter": []map[string]any{
{
"term": map[string]any{
"age": map[string]any{
"value": 42,
},
},
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
@@ -0,0 +1,58 @@
package osu
import (
"encoding/json"
)
type MatchPhraseQuery struct {
field string
query string
params *MatchPhraseQueryParams
}
type MatchPhraseQueryParams struct {
Analyzer string `json:"analyzer,omitempty"`
Slop int `json:"slop,omitempty"`
ZeroTermsQuery string `json:"zero_terms_query,omitempty"`
}
func NewMatchPhraseQuery(field string) *MatchPhraseQuery {
return &MatchPhraseQuery{field: field}
}
func (q *MatchPhraseQuery) Params(v *MatchPhraseQueryParams) *MatchPhraseQuery {
q.params = v
return q
}
func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery {
q.query = v
return q
}
func (q *MatchPhraseQuery) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
applyValue(base, "query", q.query)
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"match_phrase": map[string]any{
q.field: base,
},
}, nil
}
func (q *MatchPhraseQuery) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,72 @@
package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestNewMatchPhraseQuery(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewMatchPhraseQuery("empty"),
Want: nil,
},
{
Name: "with params",
Got: osu.NewMatchPhraseQuery("name").Params(&osu.MatchPhraseQueryParams{
Analyzer: "analyzer",
Slop: 2,
ZeroTermsQuery: "all",
}),
Want: map[string]any{
"match_phrase": map[string]any{
"name": map[string]any{
"analyzer": "analyzer",
"slop": 2,
"zero_terms_query": "all",
},
},
},
},
{
Name: "query",
Got: osu.NewMatchPhraseQuery("name").Query("some match query"),
Want: map[string]any{
"match_phrase": map[string]any{
"name": map[string]any{
"query": "some match query",
},
},
},
},
{
Name: "full",
Got: osu.NewMatchPhraseQuery("name").Params(&osu.MatchPhraseQueryParams{
Analyzer: "analyzer",
Slop: 2,
ZeroTermsQuery: "all",
}).Query("some match query"),
Want: map[string]any{
"match_phrase": map[string]any{
"name": map[string]any{
"query": "some match query",
"analyzer": "analyzer",
"slop": 2,
"zero_terms_query": "all",
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
@@ -0,0 +1,49 @@
package osu
import (
"encoding/json"
"slices"
)
type IDsQuery struct {
values []string
params *IDsQueryParams
}
type IDsQueryParams struct {
Boost float32 `json:"boost,omitempty"`
}
func NewIDsQuery(v ...string) *IDsQuery {
return &IDsQuery{values: slices.Compact(v)}
}
func (q *IDsQuery) Params(v *IDsQueryParams) *IDsQuery {
q.params = v
return q
}
func (q *IDsQuery) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
applyValue(base, "values", q.values)
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"ids": base,
}, nil
}
func (q *IDsQuery) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,45 @@
package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestIDsQuery(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewIDsQuery(),
Want: nil,
},
{
Name: "no params",
Got: osu.NewIDsQuery("1", "2", "3", "3"),
Want: map[string]any{
"ids": map[string]any{
"values": []string{"1", "2", "3"},
},
},
},
{
Name: "ids",
Got: osu.NewIDsQuery("1", "2", "3", "3").Params(&osu.IDsQueryParams{Boost: 1.0}),
Want: map[string]any{
"ids": map[string]any{
"values": []string{"1", "2", "3"},
"boost": 1.0,
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
@@ -0,0 +1,92 @@
package osu
import (
"encoding/json"
"errors"
"time"
)
type RangeQuery[T time.Time | string] struct {
field string
gt T
gte T
lt T
lte T
params *RangeQueryParams
}
type RangeQueryParams struct {
Format string `json:"format,omitempty"`
Relation string `json:"relation,omitempty"`
Boost float32 `json:"boost,omitempty"`
TimeZone string `json:"time_zone,omitempty"`
}
func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] {
return &RangeQuery[T]{field: field}
}
func (q *RangeQuery[T]) Params(v *RangeQueryParams) *RangeQuery[T] {
q.params = v
return q
}
func (q *RangeQuery[T]) Gt(v T) *RangeQuery[T] {
q.gt = v
return q
}
func (q *RangeQuery[T]) Gte(v T) *RangeQuery[T] {
q.gte = v
return q
}
func (q *RangeQuery[T]) Lt(v T) *RangeQuery[T] {
q.lt = v
return q
}
func (q *RangeQuery[T]) Lte(v T) *RangeQuery[T] {
q.lte = v
return q
}
func (q *RangeQuery[T]) Map() (map[string]any, error) {
if !isEmpty(q.gt) && !isEmpty(q.gte) {
return nil, errors.New("cannot set both gt and gte in RangeQuery")
}
if !isEmpty(q.lt) && !isEmpty(q.lte) {
return nil, errors.New("cannot set both lt and lte in RangeQuery")
}
base, err := newBase(q.params)
if err != nil {
return nil, err
}
applyValues(base, map[string]T{
"gt": q.gt,
"gte": q.gte,
"lt": q.lt,
"lte": q.lte,
})
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"range": map[string]any{
q.field: base,
},
}, nil
}
func (q *RangeQuery[T]) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,164 @@
package osu_test
import (
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestRangeQuery(t *testing.T) {
now := time.Now()
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewRangeQuery[string]("empty"),
Want: nil,
},
{
Name: "gt string",
Got: osu.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"gt": "2023-01-01T00:00:00Z",
},
},
},
},
{
Name: "gt time",
Got: osu.NewRangeQuery[time.Time]("created").Gt(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"gt": now,
},
},
},
},
{
Name: "gte string",
Got: osu.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"gte": "2023-01-01T00:00:00Z",
},
},
},
},
{
Name: "gte time",
Got: osu.NewRangeQuery[time.Time]("created").Gte(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"gte": now,
},
},
},
},
{
Name: "gt & gte",
Got: osu.NewRangeQuery[time.Time]("created").Gt(now).Gte(now),
Want: nil,
Err: errors.New(""),
},
{
Name: "gt string",
Got: osu.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"lt": "2023-01-01T00:00:00Z",
},
},
},
},
{
Name: "lt time",
Got: osu.NewRangeQuery[time.Time]("created").Lt(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"lt": now,
},
},
},
},
{
Name: "lte string",
Got: osu.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"lte": "2023-01-01T00:00:00Z",
},
},
},
},
{
Name: "lte time",
Got: osu.NewRangeQuery[time.Time]("created").Lte(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"lte": now,
},
},
},
},
{
Name: "lt & lte",
Got: osu.NewRangeQuery[time.Time]("created").Lt(now).Lte(now),
Want: nil,
Err: errors.New(""),
},
{
Name: "with params",
Got: osu.NewRangeQuery[time.Time]("created").Params(&osu.RangeQueryParams{
Format: "strict_date_optional_time",
Relation: "within",
Boost: 1.0,
TimeZone: "UTC",
}).Lte(now).Gte(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
"lte": now,
"gte": now,
"format": "strict_date_optional_time",
"relation": "within",
"boost": 1.0,
"time_zone": "UTC",
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
got, err := test.Got.MarshalJSON()
switch {
case test.Err != nil && test.Err.Error() == "": // Expecting any error
assert.Error(t, err)
assert.Nil(t, got)
return
case test.Err != nil && test.Err.Error() != "": // Expecting a specific error
assert.ErrorIs(t, test.Err, err)
assert.Nil(t, got)
return
}
require.NoError(t, err)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), string(got))
})
}
}
@@ -0,0 +1,58 @@
package osu
import (
"encoding/json"
)
type TermQuery[T comparable] struct {
field string
value T
params *TermQueryParams
}
type TermQueryParams struct {
Boost float32 `json:"boost,omitempty"`
CaseInsensitive bool `json:"case_insensitive,omitempty"`
Name string `json:"_name,omitempty"`
}
func NewTermQuery[T comparable](field string) *TermQuery[T] {
return &TermQuery[T]{field: field}
}
func (q *TermQuery[T]) Params(v *TermQueryParams) *TermQuery[T] {
q.params = v
return q
}
func (q *TermQuery[T]) Value(v T) *TermQuery[T] {
q.value = v
return q
}
func (q *TermQuery[T]) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
applyValue(base, "value", q.value)
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"term": map[string]any{
q.field: base,
},
}, nil
}
func (q *TermQuery[T]) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,55 @@
package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestTermQuery(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewTermQuery[string]("empty"),
Want: nil,
},
{
Name: "no params",
Got: osu.NewTermQuery[bool]("deleted").Value(false),
Want: map[string]any{
"term": map[string]any{
"deleted": map[string]any{
"value": false,
},
},
},
},
{
Name: "with params",
Got: osu.NewTermQuery[bool]("deleted").Params(&osu.TermQueryParams{
Boost: 1.0,
CaseInsensitive: true,
Name: "is-deleted",
}).Value(true),
Want: map[string]any{
"term": map[string]any{
"deleted": map[string]any{
"value": true,
"boost": 1.0,
"case_insensitive": true,
"_name": "is-deleted",
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
@@ -0,0 +1,58 @@
package osu
import (
"encoding/json"
)
type WildcardQuery struct {
field string
value string
params *WildcardQueryParams
}
type WildcardQueryParams struct {
Boost float32 `json:"boost,omitempty"`
CaseInsensitive bool `json:"case_insensitive,omitempty"`
Rewrite string `json:"rewrite,omitempty"`
}
func NewWildcardQuery(field string) *WildcardQuery {
return &WildcardQuery{field: field}
}
func (q *WildcardQuery) Params(v *WildcardQueryParams) *WildcardQuery {
q.params = v
return q
}
func (q *WildcardQuery) Value(v string) *WildcardQuery {
q.value = v
return q
}
func (q *WildcardQuery) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
applyValue(base, "value", q.value)
if isEmpty(base) {
return nil, nil
}
return map[string]any{
"wildcard": map[string]any{
q.field: base,
},
}, nil
}
func (q *WildcardQuery) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
@@ -0,0 +1,44 @@
package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestWildcardQuery(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
Got: osu.NewWildcardQuery("empty"),
Want: nil,
},
{
Name: "wildcard",
Got: osu.NewWildcardQuery("name").Params(&osu.WildcardQueryParams{
Boost: 1.0,
CaseInsensitive: true,
Rewrite: "top_terms_blended_freqs_N",
}).Value("opencl*"),
Want: map[string]any{
"wildcard": map[string]any{
"name": map[string]any{
"value": "opencl*",
"boost": 1.0,
"case_insensitive": true,
"rewrite": "top_terms_blended_freqs_N",
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
@@ -0,0 +1,135 @@
package osu
import (
"bytes"
"encoding/json"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
)
type QueryReqBody[P any] struct {
query Builder
params P
}
func NewQueryReqBody[P any](q Builder, p ...P) *QueryReqBody[P] {
return &QueryReqBody[P]{query: q, params: merge(p...)}
}
func (q QueryReqBody[O]) Map() (map[string]any, error) {
base, err := newBase(q.params)
if err != nil {
return nil, err
}
if err := applyBuilder(base, "query", q.query); err != nil {
return nil, err
}
return base, nil
}
func (q QueryReqBody[O]) MarshalJSON() ([]byte, error) {
data, err := q.Map()
if err != nil {
return nil, err
}
return json.Marshal(data)
}
//----------------------------------------------------------------------------//
type BodyParamHighlight struct {
HighlightOptions
Fields map[string]HighlightOptions `json:"fields,omitempty"`
}
type HighlightType string
const (
HighlightTypeUnified HighlightType = "unified"
HighlightTypeFvh HighlightType = "fvh"
HighlightTypePlain HighlightType = "plain"
HighlightTypeSemantic HighlightType = "semantic"
)
type HighlightOptions struct {
Type HighlightType `json:"type,omitempty"`
FragmentSize int `json:"fragment_size,omitempty"`
NumberOfFragments int `json:"number_of_fragments,omitempty"`
FragmentOffset int `json:"fragment_offset,omitempty"`
BoundaryChars string `json:"boundary_chars,omitempty"`
BoundaryMaxScan int `json:"boundary_max_scan,omitempty"`
BoundaryScanner string `json:"boundary_scanner,omitempty"`
BoundaryScannerLocale string `json:"boundary_scanner_locale,omitempty"`
Encoder string `json:"encoder,omitempty"`
ForceSource bool `json:"force_source,omitempty"`
Fragmenter string `json:"fragmenter,omitempty"`
HighlightQuery Builder `json:"highlight_query,omitempty"`
Order string `json:"order,omitempty"`
NoMatchSize int `json:"no_match_size,omitempty"`
RequireFieldMatch bool `json:"require_field_match,omitempty"`
MatchedFields []string `json:"matched_fields,omitempty"`
PhraseLimit int `json:"phrase_limit,omitempty"`
PreTags []string `json:"pre_tags,omitempty"`
PostTags []string `json:"post_tags,omitempty"`
}
type BodyParamScript struct {
Source string `json:"source,omitempty"`
Lang string `json:"lang,omitempty"`
Params map[string]any `json:"params,omitempty"`
}
//----------------------------------------------------------------------------//
func BuildSearchReq(req *opensearchgoAPI.SearchReq, q Builder, p ...SearchBodyParams) (*opensearchgoAPI.SearchReq, error) {
body, err := json.Marshal(NewQueryReqBody(q, p...))
if err != nil {
return nil, err
}
req.Body = bytes.NewReader(body)
return req, nil
}
type SearchBodyParams struct {
Highlight *BodyParamHighlight `json:"highlight,omitempty"`
}
//----------------------------------------------------------------------------//
func BuildDocumentDeleteByQueryReq(req opensearchgoAPI.DocumentDeleteByQueryReq, q Builder) (opensearchgoAPI.DocumentDeleteByQueryReq, error) {
body, err := json.Marshal(NewQueryReqBody[any](q))
if err != nil {
return req, err
}
req.Body = bytes.NewReader(body)
return req, nil
}
//----------------------------------------------------------------------------//
func BuildUpdateByQueryReq(req opensearchgoAPI.UpdateByQueryReq, q Builder, o ...UpdateByQueryBodyParams) (opensearchgoAPI.UpdateByQueryReq, error) {
body, err := json.Marshal(NewQueryReqBody(q, o...))
if err != nil {
return req, err
}
req.Body = bytes.NewReader(body)
return req, nil
}
type UpdateByQueryBodyParams struct {
Script *BodyParamScript `json:"script,omitempty"`
}
//----------------------------------------------------------------------------//
func BuildIndicesCountReq(req *opensearchgoAPI.IndicesCountReq, q Builder) (*opensearchgoAPI.IndicesCountReq, error) {
body, err := json.Marshal(NewQueryReqBody[any](q))
if err != nil {
return nil, err
}
req.Body = bytes.NewReader(body)
return req, nil
}
@@ -0,0 +1,94 @@
package osu_test
import (
"io"
"testing"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
opensearchtest "github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
func TestRequestBody(t *testing.T) {
tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "simple",
Got: osu.NewQueryReqBody[any](osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"query": map[string]any{
"term": map[string]any{
"name": map[string]any{
"value": "tom",
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
})
}
}
func TestBuildSearchReq(t *testing.T) {
tests := []opensearchtest.TableTest[io.Reader, map[string]any]{
{
Name: "highlight",
Got: func() io.Reader {
req, _ := osu.BuildSearchReq(
&opensearchgoAPI.SearchReq{},
osu.NewTermQuery[string]("content").Value("content"),
osu.SearchBodyParams{
Highlight: &osu.BodyParamHighlight{
HighlightOptions: osu.HighlightOptions{
PreTags: []string{"<b>"},
PostTags: []string{"</b>"},
},
Fields: map[string]osu.HighlightOptions{
"content": {
PreTags: []string{"<strong>"},
PostTags: []string{"</strong>"},
},
},
},
},
)
return req.Body
}(),
Want: map[string]any{
"query": map[string]any{
"term": map[string]any{
"content": map[string]any{
"value": "content",
},
},
},
"highlight": map[string]any{
"pre_tags": []string{"<b>"},
"post_tags": []string{"</b>"},
"fields": map[string]any{
"content": map[string]any{
"pre_tags": []string{"<strong>"},
"post_tags": []string{"</strong>"},
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
body, err := io.ReadAll(test.Got)
require.NoError(t, err)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), string(body))
})
}
}
@@ -0,0 +1,36 @@
package opensearchtest
import (
"encoding/json"
"testing"
"time"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/pkg/conversions"
)
var TimeMustParse = func(t *testing.T, ts string) time.Time {
tp, err := time.Parse(time.RFC3339Nano, ts)
require.NoError(t, err, "failed to parse time %s", ts)
return tp
}
func JSONMustMarshal(t *testing.T, data any) string {
jsonData, err := json.Marshal(data)
require.NoError(t, err, "failed to marshal data to JSON")
return string(jsonData)
}
func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
ts := make([]T, len(hits))
for i, hit := range hits {
resource, err := conversions.To[T](hit.Source)
require.NoError(t, err)
ts[i] = resource
}
return ts
}
@@ -0,0 +1,256 @@
package opensearchtest
import (
"context"
"errors"
"fmt"
"io"
"slices"
"testing"
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/search/pkg/config"
)
type TestClient struct {
c *opensearchgoAPI.Client
Require *testRequireClient
}
func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *TestClient {
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
Client: opensearchgo.Config{
Addresses: cfg.Addresses,
Username: cfg.Username,
Password: cfg.Password,
},
})
require.NoError(t, err, "failed to create OpenSearch client")
return NewTestClient(t, client)
}
func NewTestClient(t *testing.T, client *opensearchgoAPI.Client) *TestClient {
tc := &TestClient{c: client}
trc := &testRequireClient{tc: tc, t: t}
tc.Require = trc
return tc
}
func (tc *TestClient) Client() *opensearchgoAPI.Client {
return tc.c
}
func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error {
indicesToDelete := make([]string, 0, len(indices))
for _, index := range indices {
exist, err := tc.IndicesExists(ctx, []string{index})
if err != nil {
return fmt.Errorf("failed to check if index %s exists: %w", index, err)
}
if !exist {
continue
}
indicesToDelete = append(indicesToDelete, index)
}
if len(indicesToDelete) == 0 {
// If no indices to delete, return nil
return nil
}
return tc.IndicesDelete(ctx, indicesToDelete)
}
func (tc *TestClient) IndicesExists(ctx context.Context, indices []string) (bool, error) {
if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
return false, err
}
resp, err := tc.c.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
Indices: indices,
})
switch {
case resp != nil && resp.StatusCode == 404:
return false, nil
case err != nil:
return false, fmt.Errorf("failed to check if indices exist: %w", err)
case resp != nil && resp.IsError():
return false, fmt.Errorf("failed to check if indices exist: %s", resp.String())
default:
return true, nil
}
}
func (tc *TestClient) IndicesRefresh(ctx context.Context, indices []string, allow []int) error {
resp, err := tc.c.Indices.Refresh(ctx, &opensearchgoAPI.IndicesRefreshReq{
Indices: indices,
})
isAllowed := resp != nil
isAllowed = isAllowed && resp.Inspect().Response != nil
isAllowed = isAllowed && slices.Contains(allow, resp.Inspect().Response.StatusCode)
if err != nil && !isAllowed {
return fmt.Errorf("failed to refresh indices %v: %w", indices, err)
}
return nil
}
func (tc *TestClient) IndicesDelete(ctx context.Context, indices []string) error {
if err := tc.IndicesRefresh(ctx, indices, []int{}); err != nil {
return err
}
resp, err := tc.c.Indices.Delete(ctx, opensearchgoAPI.IndicesDeleteReq{
Indices: indices,
})
switch {
case err != nil:
return fmt.Errorf("failed to delete indices: %w", err)
case !resp.Acknowledged:
return errors.New("indices deletion not acknowledged")
default:
return nil
}
}
func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.Reader) error {
resp, err := tc.c.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
Index: index,
Body: body,
})
switch {
case err != nil:
return fmt.Errorf("failed to create index %s: %w", index, err)
case !resp.Acknowledged:
return fmt.Errorf("index creation not acknowledged for %s", index)
default:
return nil
}
}
func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) {
if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
return 0, err
}
resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{
Indices: indices,
Body: body,
})
switch {
case err != nil:
return 0, fmt.Errorf("failed to count documents in indices: %w", err)
default:
return resp.Count, nil
}
}
func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error {
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
return err
}
_, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{
Index: index,
DocumentID: id,
Body: body,
})
switch {
case err != nil:
return fmt.Errorf("failed to create document in index %s: %w", index, err)
default:
return nil
}
}
func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error {
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
return err
}
_, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{
Index: index,
DocumentID: id,
Body: body,
})
switch {
case err != nil:
return fmt.Errorf("failed to update document in index %s: %w", index, err)
default:
return nil
}
}
func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) {
if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
return opensearchgoAPI.SearchHits{}, err
}
resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{
Indices: []string{index},
Body: body,
})
if err != nil {
return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err)
}
return resp.Hits, nil
}
type testRequireClient struct {
tc *TestClient
t *testing.T
}
func (trc *testRequireClient) IndicesReset(indices []string) {
require.NoError(trc.t, trc.tc.IndicesReset(trc.t.Context(), indices))
}
func (trc *testRequireClient) IndicesRefresh(indices []string, ignore []int) {
require.NoError(trc.t, trc.tc.IndicesRefresh(trc.t.Context(), indices, ignore))
}
func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) {
require.NoError(trc.t, trc.tc.IndicesCreate(trc.t.Context(), index, body))
}
func (trc *testRequireClient) IndicesDelete(indices []string) {
require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices))
}
func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) {
count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body)
switch {
case expected <= 0:
require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count)
default:
require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count)
require.NoError(trc.t, err, "expected indices to have documents, but got an error")
}
}
func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) {
require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body))
}
func (trc *testRequireClient) Update(index, id string, body io.Reader) {
require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body))
}
func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits {
hits, err := trc.tc.Search(trc.t.Context(), index, body)
require.NoError(trc.t, err)
return hits
}
@@ -0,0 +1,99 @@
package opensearchtest
import (
"context"
"fmt"
"os"
"slices"
"strings"
"time"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/opensearch"
"github.com/testcontainers/testcontainers-go/wait"
"github.com/qsfera/server/services/search/pkg/config"
"github.com/qsfera/server/services/search/pkg/config/defaults"
"github.com/qsfera/server/services/search/pkg/config/parser"
)
const (
openSearchImage = "opensearchproject/opensearch:2"
)
func SetupTests(ctx context.Context) (*config.Config, func(), error) {
cfg, err := setupDefaultConfig()
if err != nil {
return nil, nil, fmt.Errorf("failed to setup default configuration: %w", err)
}
cleanupOpenSearch, err := setupOpenSearchTestContainer(ctx, cfg)
if err != nil {
return nil, nil, fmt.Errorf("failed to setup OpenSearch test container: %w", err)
}
return cfg, cleanupOpenSearch, nil
}
func setupDefaultConfig() (*config.Config, error) {
cfg := defaults.DefaultConfig()
defaults.EnsureDefaults(cfg)
{ // parser.Validate requires these fields to be set even if they are not used in these tests.
cfg.TokenManager.JWTSecret = "any-jwt"
cfg.ServiceAccount.ServiceAccountID = "any-service-account-id"
cfg.ServiceAccount.ServiceAccountSecret = "any-service-account-secret"
}
if err := parser.ParseConfig(cfg); err != nil {
return nil, fmt.Errorf("failed to parse default configuration: %w", err)
}
return cfg, nil
}
func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func(), error) {
usesExternalOpensearch := slices.Contains([]bool{
os.Getenv("CI") == "woodpecker",
os.Getenv("CI_SYSTEM_NAME") == "woodpecker",
os.Getenv("USE_TESTCONTAINERS") == "false",
}, true)
if usesExternalOpensearch {
return func() {}, nil
}
containerName := fmt.Sprintf("qsfera/test/%s", openSearchImage)
containerName = strings.Replace(containerName, "/", "__", -1)
containerName = strings.Replace(containerName, ":", "_", -1)
container, err := opensearch.Run(
ctx,
openSearchImage,
opensearch.WithUsername(cfg.Engine.OpenSearch.Client.Username),
opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password),
testcontainers.WithName(containerName),
testcontainers.WithReuseByName(containerName),
testcontainers.WithWaitStrategy(
wait.ForLog("ML configuration initialized successfully").
WithStartupTimeout(5*time.Second),
),
)
if err != nil {
return nil, fmt.Errorf("failed to start OpenSearch container: %w", err)
}
address, err := container.Address(ctx)
if err != nil {
_ = container.Terminate(ctx) // attempt to clean up the container
return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err)
}
// Ensure the address is set in the default configuration.
cfg.Engine.OpenSearch.Client.Addresses = []string{address}
return func() {
err := container.Terminate(ctx)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err)
}
}, nil
}
@@ -0,0 +1,9 @@
package opensearchtest
type TableTest[G any, W any] struct {
Name string
Got G
Want W
Err error
Skip bool
}
@@ -0,0 +1,44 @@
package opensearchtest
import (
"embed"
"encoding/json"
"fmt"
"path"
"github.com/qsfera/server/services/search/pkg/search"
)
//go:embed testdata/*.json
var testdata embed.FS
var Testdata = struct {
Resources resourceTestdata
}{
Resources: resourceTestdata{
Root: fromTestData[search.Resource]("resource_root.json"),
Folder: fromTestData[search.Resource]("resource_folder.json"),
File: fromTestData[search.Resource]("resource_file.json"),
},
}
type resourceTestdata struct {
Root search.Resource
File search.Resource
Folder search.Resource
}
func fromTestData[D any](name string) D {
name = path.Join("./testdata", name)
data, err := testdata.ReadFile(name)
if err != nil {
panic(fmt.Sprintf("failed to read testdata file %s: %v", name, err))
}
var d D
if json.Unmarshal(data, &d) != nil {
panic(fmt.Sprintf("failed to unmarshal testdata %s: %v", name, err))
}
return d
}
@@ -0,0 +1,54 @@
{
"ID" : "1$1!3",
"RootID" : "1$1!1",
"ParentID" : "1$1!2",
"Path" : "./parent d!r/child.jpg",
"Type" : 1,
"Title" : "dumme title",
"Name" : "dummy name",
"Content" : "dummy content",
"Size" : 42,
"Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251",
"MimeType" : "image/jpeg",
"Tags" : [ "dummy" ],
"Deleted" : false,
"Hidden" : false,
"audio" : {
"album" : "Some Album",
"albumArtist" : "Some AlbumArtist",
"artist" : "Some Artist",
"bitrate" : 192,
"composers" : "Some Composers",
"copyright" : "",
"disc" : 2,
"discCount" : 5,
"duration" : 225000,
"genre" : "Some Genre",
"hasDrm" : false,
"isVariableBitrate" : true,
"title" : "Some Title",
"track" : 34,
"trackCount" : 99,
"year" : 2004
},
"image" : {
"height" : 100,
"width" : 100
},
"location" : {
"altitude" : 1047.7,
"latitude" : 49.48675890884328,
"longitude" : 11.103870357204285
},
"photo" : {
"cameraMake" : "CameraMake",
"cameraModel" : "CameraMake",
"exposureDenominator" : 1,
"exposureNumerator" : 1,
"fNumber" : 1,
"focalLength" : 1,
"iso" : 1,
"orientation" : 1,
"takenDateTime" : "2018-01-01T12:34:56Z"
}
}
@@ -0,0 +1,7 @@
{
"ID" : "1$1!2",
"RootID" : "1$1!1",
"ParentID" : "1$1!1",
"Path" : "./parent d!r",
"Type" : 2
}
@@ -0,0 +1,5 @@
{
"ID" : "1$1!1",
"RootID" : "1$1!1",
"Path" : "."
}
@@ -0,0 +1,77 @@
package opensearch
import (
"context"
"fmt"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/qsfera/server/services/search/pkg/search"
"github.com/qsfera/server/pkg/conversions"
"github.com/qsfera/server/services/search/pkg/opensearch/internal/osu"
)
func searchResourceByID(ctx context.Context, client *opensearchgoAPI.Client, index string, id string) (search.Resource, error) {
req, err := osu.BuildSearchReq(
&opensearchgoAPI.SearchReq{
Indices: []string{index},
},
osu.NewIDsQuery(id),
)
if err != nil {
return search.Resource{}, fmt.Errorf("failed to build search request: %w", err)
}
resp, err := client.Search(ctx, req)
switch {
case err != nil:
return search.Resource{}, fmt.Errorf("failed to search for resource: %w", err)
case resp.Hits.Total.Value == 0 || len(resp.Hits.Hits) == 0:
return search.Resource{}, fmt.Errorf("document with id %s not found", id)
}
resource, err := conversions.To[search.Resource](resp.Hits.Hits[0].Source)
if err != nil {
return search.Resource{}, fmt.Errorf("failed to convert hit source: %w", err)
}
return resource, nil
}
func updateSelfAndDescendants(ctx context.Context, client *opensearchgoAPI.Client, index string, id string, scriptProvider func(search.Resource) *osu.BodyParamScript) error {
if scriptProvider == nil {
return fmt.Errorf("script cannot be nil")
}
resource, err := searchResourceByID(context.Background(), client, index, id)
if err != nil {
return fmt.Errorf("failed to get resource: %w", err)
}
req, err := osu.BuildUpdateByQueryReq(
opensearchgoAPI.UpdateByQueryReq{
Indices: []string{index},
Params: opensearchgoAPI.UpdateByQueryParams{
WaitForCompletion: conversions.ToPointer(true),
},
},
osu.NewTermQuery[string]("Path").Value(resource.Path),
osu.UpdateByQueryBodyParams{
Script: scriptProvider(resource),
},
)
if err != nil {
return fmt.Errorf("failed to build update by query request: %w", err)
}
resp, err := client.UpdateByQuery(ctx, req)
switch {
case err != nil:
return fmt.Errorf("failed to update by query: %w", err)
case len(resp.Failures) != 0:
return fmt.Errorf("failed to update by query, failures: %v", resp.Failures)
}
return nil
}
@@ -0,0 +1,26 @@
package opensearch_test
import (
"context"
"fmt"
"os"
"testing"
"github.com/qsfera/server/services/search/pkg/config"
opensearchtest "github.com/qsfera/server/services/search/pkg/opensearch/internal/test"
)
var defaultConfig *config.Config
func TestMain(m *testing.M) {
cfg, done, err := opensearchtest.SetupTests(context.Background())
if err != nil {
_, _ = fmt.Fprintln(os.Stderr, "Failed to setup tests:", err)
os.Exit(1)
return
}
defaultConfig = cfg
code := m.Run()
done()
os.Exit(code)
}