Initial QSfera import

Server/services/search/pkg/opensearch/internal/test/helper.go

@@ -0,0 +1,36 @@
+package opensearchtest
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/qsfera/server/pkg/conversions"
+)
+
+var TimeMustParse = func(t *testing.T, ts string) time.Time {
+	tp, err := time.Parse(time.RFC3339Nano, ts)
+	require.NoError(t, err, "failed to parse time %s", ts)
+
+	return tp
+}
+
+func JSONMustMarshal(t *testing.T, data any) string {
+	jsonData, err := json.Marshal(data)
+	require.NoError(t, err, "failed to marshal data to JSON")
+	return string(jsonData)
+}
+
+func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
+	ts := make([]T, len(hits))
+	for i, hit := range hits {
+		resource, err := conversions.To[T](hit.Source)
+		require.NoError(t, err)
+		ts[i] = resource
+	}
+
+	return ts
+}

Server/services/search/pkg/opensearch/internal/test/os.go

@@ -0,0 +1,256 @@
+package opensearchtest
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+	"testing"
+
+	opensearchgo "github.com/opensearch-project/opensearch-go/v4"
+	opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/qsfera/server/services/search/pkg/config"
+)
+
+type TestClient struct {
+	c       *opensearchgoAPI.Client
+	Require *testRequireClient
+}
+
+func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *TestClient {
+	client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
+		Client: opensearchgo.Config{
+			Addresses: cfg.Addresses,
+			Username:  cfg.Username,
+			Password:  cfg.Password,
+		},
+	})
+	require.NoError(t, err, "failed to create OpenSearch client")
+
+	return NewTestClient(t, client)
+}
+
+func NewTestClient(t *testing.T, client *opensearchgoAPI.Client) *TestClient {
+	tc := &TestClient{c: client}
+	trc := &testRequireClient{tc: tc, t: t}
+	tc.Require = trc
+
+	return tc
+}
+
+func (tc *TestClient) Client() *opensearchgoAPI.Client {
+	return tc.c
+}
+
+func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error {
+	indicesToDelete := make([]string, 0, len(indices))
+	for _, index := range indices {
+		exist, err := tc.IndicesExists(ctx, []string{index})
+		if err != nil {
+			return fmt.Errorf("failed to check if index %s exists: %w", index, err)
+		}
+
+		if !exist {
+			continue
+		}
+
+		indicesToDelete = append(indicesToDelete, index)
+	}
+
+	if len(indicesToDelete) == 0 {
+		// If no indices to delete, return nil
+		return nil
+	}
+
+	return tc.IndicesDelete(ctx, indicesToDelete)
+}
+
+func (tc *TestClient) IndicesExists(ctx context.Context, indices []string) (bool, error) {
+	if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
+		return false, err
+	}
+
+	resp, err := tc.c.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
+		Indices: indices,
+	})
+	switch {
+	case resp != nil && resp.StatusCode == 404:
+		return false, nil
+	case err != nil:
+		return false, fmt.Errorf("failed to check if indices exist: %w", err)
+	case resp != nil && resp.IsError():
+		return false, fmt.Errorf("failed to check if indices exist: %s", resp.String())
+	default:
+		return true, nil
+	}
+}
+
+func (tc *TestClient) IndicesRefresh(ctx context.Context, indices []string, allow []int) error {
+	resp, err := tc.c.Indices.Refresh(ctx, &opensearchgoAPI.IndicesRefreshReq{
+		Indices: indices,
+	})
+
+	isAllowed := resp != nil
+	isAllowed = isAllowed && resp.Inspect().Response != nil
+	isAllowed = isAllowed && slices.Contains(allow, resp.Inspect().Response.StatusCode)
+
+	if err != nil && !isAllowed {
+		return fmt.Errorf("failed to refresh indices %v: %w", indices, err)
+	}
+
+	return nil
+}
+
+func (tc *TestClient) IndicesDelete(ctx context.Context, indices []string) error {
+	if err := tc.IndicesRefresh(ctx, indices, []int{}); err != nil {
+		return err
+	}
+
+	resp, err := tc.c.Indices.Delete(ctx, opensearchgoAPI.IndicesDeleteReq{
+		Indices: indices,
+	})
+	switch {
+	case err != nil:
+		return fmt.Errorf("failed to delete indices: %w", err)
+	case !resp.Acknowledged:
+		return errors.New("indices deletion not acknowledged")
+	default:
+		return nil
+	}
+}
+
+func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.Reader) error {
+	resp, err := tc.c.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
+		Index: index,
+		Body:  body,
+	})
+
+	switch {
+	case err != nil:
+		return fmt.Errorf("failed to create index %s: %w", index, err)
+	case !resp.Acknowledged:
+		return fmt.Errorf("index creation not acknowledged for %s", index)
+	default:
+		return nil
+	}
+}
+
+func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) {
+	if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil {
+		return 0, err
+	}
+
+	resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{
+		Indices: indices,
+		Body:    body,
+	})
+
+	switch {
+	case err != nil:
+		return 0, fmt.Errorf("failed to count documents in indices: %w", err)
+	default:
+		return resp.Count, nil
+	}
+}
+
+func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error {
+	if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
+		return err
+	}
+
+	_, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{
+		Index:      index,
+		DocumentID: id,
+		Body:       body,
+	})
+	switch {
+	case err != nil:
+		return fmt.Errorf("failed to create document in index %s: %w", index, err)
+	default:
+		return nil
+	}
+}
+
+func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error {
+	if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
+		return err
+	}
+
+	_, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{
+		Index:      index,
+		DocumentID: id,
+		Body:       body,
+	})
+	switch {
+	case err != nil:
+		return fmt.Errorf("failed to update document in index %s: %w", index, err)
+	default:
+		return nil
+	}
+}
+
+func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) {
+	if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
+		return opensearchgoAPI.SearchHits{}, err
+	}
+
+	resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{
+		Indices: []string{index},
+		Body:    body,
+	})
+	if err != nil {
+		return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err)
+	}
+
+	return resp.Hits, nil
+}
+
+type testRequireClient struct {
+	tc *TestClient
+	t  *testing.T
+}
+
+func (trc *testRequireClient) IndicesReset(indices []string) {
+	require.NoError(trc.t, trc.tc.IndicesReset(trc.t.Context(), indices))
+}
+
+func (trc *testRequireClient) IndicesRefresh(indices []string, ignore []int) {
+	require.NoError(trc.t, trc.tc.IndicesRefresh(trc.t.Context(), indices, ignore))
+}
+
+func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) {
+	require.NoError(trc.t, trc.tc.IndicesCreate(trc.t.Context(), index, body))
+}
+
+func (trc *testRequireClient) IndicesDelete(indices []string) {
+	require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices))
+}
+
+func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) {
+	count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body)
+
+	switch {
+	case expected <= 0:
+		require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count)
+	default:
+		require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count)
+		require.NoError(trc.t, err, "expected indices to have documents, but got an error")
+	}
+}
+
+func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) {
+	require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body))
+}
+
+func (trc *testRequireClient) Update(index, id string, body io.Reader) {
+	require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body))
+}
+
+func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits {
+	hits, err := trc.tc.Search(trc.t.Context(), index, body)
+	require.NoError(trc.t, err)
+	return hits
+}

Server/services/search/pkg/opensearch/internal/test/suite.go

@@ -0,0 +1,99 @@
+package opensearchtest
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/opensearch"
+	"github.com/testcontainers/testcontainers-go/wait"
+
+	"github.com/qsfera/server/services/search/pkg/config"
+	"github.com/qsfera/server/services/search/pkg/config/defaults"
+	"github.com/qsfera/server/services/search/pkg/config/parser"
+)
+
+const (
+	openSearchImage = "opensearchproject/opensearch:2"
+)
+
+func SetupTests(ctx context.Context) (*config.Config, func(), error) {
+	cfg, err := setupDefaultConfig()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to setup default configuration: %w", err)
+	}
+
+	cleanupOpenSearch, err := setupOpenSearchTestContainer(ctx, cfg)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to setup OpenSearch test container: %w", err)
+	}
+
+	return cfg, cleanupOpenSearch, nil
+}
+
+func setupDefaultConfig() (*config.Config, error) {
+	cfg := defaults.DefaultConfig()
+	defaults.EnsureDefaults(cfg)
+
+	{ // parser.Validate requires these fields to be set even if they are not used in these tests.
+		cfg.TokenManager.JWTSecret = "any-jwt"
+		cfg.ServiceAccount.ServiceAccountID = "any-service-account-id"
+		cfg.ServiceAccount.ServiceAccountSecret = "any-service-account-secret"
+	}
+
+	if err := parser.ParseConfig(cfg); err != nil {
+		return nil, fmt.Errorf("failed to parse default configuration: %w", err)
+	}
+
+	return cfg, nil
+}
+
+func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func(), error) {
+	usesExternalOpensearch := slices.Contains([]bool{
+		os.Getenv("CI") == "woodpecker",
+		os.Getenv("CI_SYSTEM_NAME") == "woodpecker",
+		os.Getenv("USE_TESTCONTAINERS") == "false",
+	}, true)
+	if usesExternalOpensearch {
+		return func() {}, nil
+	}
+
+	containerName := fmt.Sprintf("qsfera/test/%s", openSearchImage)
+	containerName = strings.Replace(containerName, "/", "__", -1)
+	containerName = strings.Replace(containerName, ":", "_", -1)
+	container, err := opensearch.Run(
+		ctx,
+		openSearchImage,
+		opensearch.WithUsername(cfg.Engine.OpenSearch.Client.Username),
+		opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password),
+		testcontainers.WithName(containerName),
+		testcontainers.WithReuseByName(containerName),
+		testcontainers.WithWaitStrategy(
+			wait.ForLog("ML configuration initialized successfully").
+				WithStartupTimeout(5*time.Second),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start OpenSearch container: %w", err)
+	}
+
+	address, err := container.Address(ctx)
+	if err != nil {
+		_ = container.Terminate(ctx) // attempt to clean up the container
+		return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err)
+	}
+
+	// Ensure the address is set in the default configuration.
+	cfg.Engine.OpenSearch.Client.Addresses = []string{address}
+
+	return func() {
+		err := container.Terminate(ctx)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err)
+		}
+	}, nil
+}

Server/services/search/pkg/opensearch/internal/test/test.go

@@ -0,0 +1,9 @@
+package opensearchtest
+
+type TableTest[G any, W any] struct {
+	Name string
+	Got  G
+	Want W
+	Err  error
+	Skip bool
+}

Server/services/search/pkg/opensearch/internal/test/testdata.go

@@ -0,0 +1,44 @@
+package opensearchtest
+
+import (
+	"embed"
+	"encoding/json"
+	"fmt"
+	"path"
+
+	"github.com/qsfera/server/services/search/pkg/search"
+)
+
+//go:embed testdata/*.json
+var testdata embed.FS
+
+var Testdata = struct {
+	Resources resourceTestdata
+}{
+	Resources: resourceTestdata{
+		Root:   fromTestData[search.Resource]("resource_root.json"),
+		Folder: fromTestData[search.Resource]("resource_folder.json"),
+		File:   fromTestData[search.Resource]("resource_file.json"),
+	},
+}
+
+type resourceTestdata struct {
+	Root   search.Resource
+	File   search.Resource
+	Folder search.Resource
+}
+
+func fromTestData[D any](name string) D {
+	name = path.Join("./testdata", name)
+	data, err := testdata.ReadFile(name)
+	if err != nil {
+		panic(fmt.Sprintf("failed to read testdata file %s: %v", name, err))
+	}
+
+	var d D
+	if json.Unmarshal(data, &d) != nil {
+		panic(fmt.Sprintf("failed to unmarshal testdata %s: %v", name, err))
+	}
+
+	return d
+}

Server/services/search/pkg/opensearch/internal/test/testdata/resource_file.json

@@ -0,0 +1,54 @@
+{
+  "ID" : "1$1!3",
+  "RootID" : "1$1!1",
+  "ParentID" : "1$1!2",
+  "Path" : "./parent d!r/child.jpg",
+  "Type" : 1,
+  "Title" : "dumme title",
+  "Name" : "dummy name",
+  "Content" : "dummy content",
+  "Size" : 42,
+  "Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251",
+  "MimeType" : "image/jpeg",
+  "Tags" : [ "dummy" ],
+  "Deleted" : false,
+  "Hidden" : false,
+  "audio" : {
+    "album" : "Some Album",
+    "albumArtist" : "Some AlbumArtist",
+    "artist" : "Some Artist",
+    "bitrate" : 192,
+    "composers" : "Some Composers",
+    "copyright" : "",
+    "disc" : 2,
+    "discCount" : 5,
+    "duration" : 225000,
+    "genre" : "Some Genre",
+    "hasDrm" : false,
+    "isVariableBitrate" : true,
+    "title" : "Some Title",
+    "track" : 34,
+    "trackCount" : 99,
+    "year" : 2004
+  },
+  "image" : {
+    "height" : 100,
+    "width" : 100
+  },
+  "location" : {
+    "altitude" : 1047.7,
+    "latitude" : 49.48675890884328,
+    "longitude" : 11.103870357204285
+  },
+  "photo" : {
+    "cameraMake" : "CameraMake",
+    "cameraModel" : "CameraMake",
+    "exposureDenominator" : 1,
+    "exposureNumerator" : 1,
+    "fNumber" : 1,
+    "focalLength" : 1,
+    "iso" : 1,
+    "orientation" : 1,
+    "takenDateTime" : "2018-01-01T12:34:56Z"
+  }
+}

Server/services/search/pkg/opensearch/internal/test/testdata/resource_folder.json

@@ -0,0 +1,7 @@
+{
+  "ID" : "1$1!2",
+  "RootID" : "1$1!1",
+  "ParentID" : "1$1!1",
+  "Path" : "./parent d!r",
+  "Type" : 2
+}

Server/services/search/pkg/opensearch/internal/test/testdata/resource_root.json

@@ -0,0 +1,5 @@
+{
+  "ID" : "1$1!1",
+  "RootID" : "1$1!1",
+  "Path" : "."
+}