enhancement(search): implement search backend recursive move and update restore and delete to be recursive too

services/search/pkg/opensearch/engine.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"path"
 	"strings"
 
 	storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -18,6 +20,8 @@ import (
 	"github.com/opencloud-eu/opencloud/services/search/pkg/engine"
 )
 
+var ErrNoContainerType = fmt.Errorf("not a container type")
+
 type Engine struct {
 	index  string
 	client *opensearchgoAPI.Client
@@ -119,7 +123,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
 		Body:    bytes.NewReader(body),
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to count documents: %w", err)
+		return nil, fmt.Errorf("failed to search: %w", err)
 	}
 
 	matches := make([]*searchMessage.Match, len(resp.Hits.Hits))
@@ -156,7 +160,7 @@ func (e *Engine) Upsert(id string, r engine.Resource) error {
 		return fmt.Errorf("failed to marshal resource: %w", err)
 	}
 
-	_, err = e.client.Document.Create(context.TODO(), opensearchgoAPI.DocumentCreateReq{
+	_, err = e.client.Index(context.TODO(), opensearchgoAPI.IndexReq{
 		Index:      e.index,
 		DocumentID: id,
 		Body:       bytes.NewReader(body),
@@ -169,51 +173,41 @@ func (e *Engine) Upsert(id string, r engine.Resource) error {
 }
 
 func (e *Engine) Move(id string, parentID string, target string) error {
+	resource, err := e.getResource(id)
+	if err != nil {
+		return fmt.Errorf("failed to get resource: %w", err)
+	}
+
+	oldPath := resource.Path
+	resource.Path = utils.MakeRelativePath(target)
+	resource.Name = path.Base(resource.Path)
+	resource.ParentID = parentID
+
+	if err := e.Upsert(id, resource); err != nil {
+		return fmt.Errorf("failed to upsert resource: %w", err)
+	}
+
+	descendants, err := e.getDescendants(resource.Type, resource.RootID, oldPath)
+	if err != nil && !errors.Is(err, ErrNoContainerType) {
+		return fmt.Errorf("failed to find descendants: %w", err)
+	}
+
+	for _, descendant := range descendants {
+		descendant.Path = strings.Replace(descendant.Path, oldPath, resource.Path, 1)
+		if err := e.Upsert(descendant.ID, descendant); err != nil {
+			return fmt.Errorf("failed to upsert resource: %w", err)
+		}
+	}
+
 	return nil
 }
 
 func (e *Engine) Delete(id string) error {
-	body, err := json.Marshal(map[string]any{
-		"doc": map[string]bool{
-			"Deleted": true,
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("failed to marshal body: %w", err)
-	}
-
-	_, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{
-		Index:      e.index,
-		DocumentID: id,
-		Body:       bytes.NewReader(body),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to mark document as deleted: %w", err)
-	}
-
-	return nil
+	return e.deleteResource(id, true)
 }
 
 func (e *Engine) Restore(id string) error {
-	body, err := json.Marshal(map[string]any{
-		"doc": map[string]bool{
-			"Deleted": false,
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("failed to marshal body: %w", err)
-	}
-
-	_, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{
-		Index:      e.index,
-		DocumentID: id,
-		Body:       bytes.NewReader(body),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to mark document as deleted: %w", err)
-	}
-
-	return nil
+	return e.deleteResource(id, false)
 }
 
 func (e *Engine) Purge(id string) error {
@@ -246,3 +240,115 @@ func (e *Engine) DocCount() (uint64, error) {
 
 	return uint64(resp.Count), nil
 }
+
+func (e *Engine) deleteResource(id string, deleted bool) error {
+	resource, err := e.getResource(id)
+	if err != nil {
+		return fmt.Errorf("failed to get resource: %w", err)
+	}
+
+	descendants, err := e.getDescendants(resource.Type, resource.RootID, resource.Path)
+	if err != nil && !errors.Is(err, ErrNoContainerType) {
+		return fmt.Errorf("failed to find descendants: %w", err)
+	}
+
+	body, err := json.Marshal(map[string]any{
+		"doc": map[string]bool{
+			"Deleted": deleted,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to marshal body: %w", err)
+	}
+
+	for _, resource := range append([]engine.Resource{resource}, descendants...) {
+		if resource.Deleted == deleted {
+			continue // already marked as the desired state
+		}
+
+		if _, err = e.client.Update(context.TODO(), opensearchgoAPI.UpdateReq{
+			Index:      e.index,
+			DocumentID: resource.ID,
+			Body:       bytes.NewReader(body),
+		}); err != nil {
+			return fmt.Errorf("failed to mark document as deleted: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (e *Engine) getResource(id string) (engine.Resource, error) {
+	body, err := NewRootQuery(
+		NewIDsQuery([]string{id}),
+	).MarshalJSON()
+	if err != nil {
+		return engine.Resource{}, fmt.Errorf("failed to marshal query: %w", err)
+	}
+
+	resp, err := e.client.Search(context.TODO(), &opensearchgoAPI.SearchReq{
+		Indices: []string{e.index},
+		Body:    bytes.NewReader(body),
+	})
+	switch {
+	case err != nil:
+		return engine.Resource{}, fmt.Errorf("failed to search for resource: %w", err)
+	case resp.Hits.Total.Value == 0 || len(resp.Hits.Hits) == 0:
+		return engine.Resource{}, fmt.Errorf("document with id %s not found", id)
+	}
+
+	resource, err := convert[engine.Resource](resp.Hits.Hits[0].Source)
+	if err != nil {
+		return engine.Resource{}, fmt.Errorf("failed to convert hit source: %w", err)
+	}
+
+	return resource, nil
+}
+
+func (e *Engine) getDescendants(resourceType uint64, rootID, rootPath string) ([]engine.Resource, error) {
+	switch {
+	case resourceType != uint64(storageProvider.ResourceType_RESOURCE_TYPE_CONTAINER):
+		return nil, fmt.Errorf("%w: %d", ErrNoContainerType, resourceType)
+	case rootID == "":
+		return nil, fmt.Errorf("rootID cannot be empty")
+	case rootPath == "":
+		return nil, fmt.Errorf("rootPath cannot be empty")
+	}
+
+	if !strings.HasSuffix(rootPath, "*") {
+		rootPath = strings.Join(append(strings.Split(rootPath, "/"), "*"), "/")
+	}
+
+	body, err := NewRootQuery(
+		NewBoolQuery().Must(
+			NewTermQuery[string]("RootID").Value(rootID),
+			NewWildcardQuery("Path").Value(rootPath),
+		),
+	).MarshalJSON()
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal query: %w", err)
+	}
+
+	resp, err := e.client.Search(context.TODO(), &opensearchgoAPI.SearchReq{
+		Indices: []string{e.index},
+		Body:    bytes.NewReader(body),
+	})
+	switch {
+	case err != nil:
+		return nil, fmt.Errorf("failed to search for document: %w", err)
+	case resp.Hits.Total.Value == 0 || len(resp.Hits.Hits) == 0:
+		return nil, nil // no descendants found, fin
+	}
+
+	descendants := make([]engine.Resource, resp.Hits.Total.Value)
+	for i, hit := range resp.Hits.Hits {
+		descendant, err := convert[engine.Resource](hit.Source)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert hit source %d: %w", i, err)
+		}
+
+		descendants[i] = descendant
+	}
+
+	return descendants, nil
+}

services/search/pkg/opensearch/engine_convert_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestSearchHitToSearchMessageMatch(t *testing.T) {
-	resource := opensearchtest.Testdata.Resources.Full
+	resource := opensearchtest.Testdata.Resources.File
 	resource.MimeType = "audio/anything"
 
 	hit := opensearchgoAPI.SearchHit{

services/search/pkg/opensearch/engine_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/engine"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
 )
@@ -36,15 +37,15 @@ func TestEngine_Search(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	document := opensearchtest.Testdata.Resources.Full
+	document := opensearchtest.Testdata.Resources.File
 	tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
 	tc.Require.IndicesCount([]string{index}, "", 1)
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("most simple search", func(t *testing.T) {
-		resp, err := engine.Search(t.Context(), &searchService.SearchIndexRequest{
+		resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
 			Query: fmt.Sprintf(`"%s"`, document.Name),
 		})
 		require.NoError(t, err)
@@ -54,14 +55,14 @@ func TestEngine_Search(t *testing.T) {
 	})
 
 	t.Run("ignores files that are marked as deleted", func(t *testing.T) {
-		deletedDocument := opensearchtest.Testdata.Resources.Full
+		deletedDocument := opensearchtest.Testdata.Resources.File
 		deletedDocument.ID = "1$2!4"
 		deletedDocument.Deleted = true
 
 		tc.Require.DocumentCreate(index, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument))
 		tc.Require.IndicesCount([]string{index}, "", 2)
 
-		resp, err := engine.Search(t.Context(), &searchService.SearchIndexRequest{
+		resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
 			Query: fmt.Sprintf(`"%s"`, document.Name),
 		})
 		require.NoError(t, err)
@@ -79,18 +80,59 @@ func TestEngine_Upsert(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("upsert with full document", func(t *testing.T) {
-		document := opensearchtest.Testdata.Resources.Full
-		require.NoError(t, engine.Upsert(document.ID, document))
+		document := opensearchtest.Testdata.Resources.File
+		require.NoError(t, backend.Upsert(document.ID, document))
 
 		tc.Require.IndicesCount([]string{index}, "", 1)
 	})
 }
 
-func TestEngine_Move(t *testing.T) {}
+func TestEngine_Move(t *testing.T) {
+	index := "opencloud-default-resource"
+	tc := opensearchtest.NewDefaultTestClient(t)
+	tc.Require.IndicesReset([]string{index})
+	tc.Require.IndicesCount([]string{index}, "", 0)
+
+	defer tc.Require.IndicesDelete([]string{index})
+
+	backend, err := opensearch.NewEngine(index, tc.Client())
+	require.NoError(t, err)
+
+	t.Run("moves the document to a new path", func(t *testing.T) {
+		document := opensearchtest.Testdata.Resources.File
+		tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
+		tc.Require.IndicesCount([]string{index}, "", 1)
+
+		resources := opensearchtest.SearchHitsMustBeConverted[engine.Resource](t,
+			tc.Require.Search(
+				index,
+				opensearch.NewRootQuery(
+					opensearch.NewIDsQuery([]string{document.ID}),
+				).String(),
+			).Hits,
+		)
+		require.Len(t, resources, 1)
+		require.Equal(t, document.Path, resources[0].Path)
+
+		document.Path = "./new/path/to/resource"
+		require.NoError(t, backend.Move(document.ID, document.ParentID, document.Path))
+
+		resources = opensearchtest.SearchHitsMustBeConverted[engine.Resource](t,
+			tc.Require.Search(
+				index,
+				opensearch.NewRootQuery(
+					opensearch.NewIDsQuery([]string{document.ID}),
+				).String(),
+			).Hits,
+		)
+		require.Len(t, resources, 1)
+		require.Equal(t, document.Path, resources[0].Path)
+	})
+}
 
 func TestEngine_Delete(t *testing.T) {
 	index := "opencloud-default-resource"
@@ -100,11 +142,11 @@ func TestEngine_Delete(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("mark document as deleted", func(t *testing.T) {
-		document := opensearchtest.Testdata.Resources.Full
+		document := opensearchtest.Testdata.Resources.File
 		tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
 		tc.Require.IndicesCount([]string{index}, "", 1)
 
@@ -112,7 +154,7 @@ func TestEngine_Delete(t *testing.T) {
 			opensearch.NewTermQuery[bool]("Deleted").Value(true),
 		).String(), 0)
 
-		require.NoError(t, engine.Delete(document.ID))
+		require.NoError(t, backend.Delete(document.ID))
 		tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery(
 			opensearch.NewTermQuery[bool]("Deleted").Value(true),
 		).String(), 1)
@@ -127,11 +169,11 @@ func TestEngine_Restore(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("mark document as not deleted", func(t *testing.T) {
-		document := opensearchtest.Testdata.Resources.Full
+		document := opensearchtest.Testdata.Resources.File
 		document.Deleted = true
 		tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
 		tc.Require.IndicesCount([]string{index}, "", 1)
@@ -140,7 +182,7 @@ func TestEngine_Restore(t *testing.T) {
 			opensearch.NewTermQuery[bool]("Deleted").Value(true),
 		).String(), 1)
 
-		require.NoError(t, engine.Restore(document.ID))
+		require.NoError(t, backend.Restore(document.ID))
 		tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery(
 			opensearch.NewTermQuery[bool]("Deleted").Value(true),
 		).String(), 0)
@@ -155,15 +197,15 @@ func TestEngine_Purge(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("purge with full document", func(t *testing.T) {
-		document := opensearchtest.Testdata.Resources.Full
+		document := opensearchtest.Testdata.Resources.File
 		tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
 		tc.Require.IndicesCount([]string{index}, "", 1)
 
-		require.NoError(t, engine.Purge(document.ID))
+		require.NoError(t, backend.Purge(document.ID))
 
 		tc.Require.IndicesCount([]string{index}, "", 0)
 	})
@@ -177,15 +219,15 @@ func TestEngine_DocCount(t *testing.T) {
 
 	defer tc.Require.IndicesDelete([]string{index})
 
-	engine, err := opensearch.NewEngine(index, tc.Client())
+	backend, err := opensearch.NewEngine(index, tc.Client())
 	require.NoError(t, err)
 
 	t.Run("ignore deleted documents", func(t *testing.T) {
-		document := opensearchtest.Testdata.Resources.Full
+		document := opensearchtest.Testdata.Resources.File
 		tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document))
 		tc.Require.IndicesCount([]string{index}, "", 1)
 
-		count, err := engine.DocCount()
+		count, err := backend.DocCount()
 		require.NoError(t, err)
 		require.Equal(t, uint64(1), count)
 
@@ -197,7 +239,7 @@ func TestEngine_DocCount(t *testing.T) {
 
 		tc.Require.IndicesCount([]string{index}, "", 1)
 
-		count, err = engine.DocCount()
+		count, err = backend.DocCount()
 		require.NoError(t, err)
 		require.Equal(t, uint64(0), count)
 	})

services/search/pkg/opensearch/internal/indices/resource_v1.json

@@ -21,6 +21,9 @@
         "MimeType": {
           "type": "wildcard"
         },
+        "Path": {
+          "type": "wildcard"
+        },
         "Deleted": {
           "type": "boolean"
         },

services/search/pkg/opensearch/internal/test/helper.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 	"time"
 
+	opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,3 +22,30 @@ func JSONMustMarshal(t *testing.T, data any) string {
 	require.NoError(t, err, "failed to marshal data to JSON")
 	return string(jsonData)
 }
+
+func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
+	return lo.ReduceRight(hits, func(agg []T, item opensearchgoAPI.SearchHit, _ int) []T {
+		resource, err := convert[T](item.Source)
+		require.NoError(t, err)
+		return append(agg, resource)
+	}, []T{})
+}
+
+func convert[T any](v any) (T, error) {
+	var t T
+
+	if v == nil {
+		return t, nil
+	}
+
+	j, err := json.Marshal(v)
+	if err != nil {
+		return t, err
+	}
+
+	if err := json.Unmarshal(j, &t); err != nil {
+		return t, err
+	}
+
+	return t, nil
+}

services/search/pkg/opensearch/internal/test/os.go

@@ -188,6 +188,22 @@ func (tc *TestClient) Update(ctx context.Context, index string, id, body string)
 	}
 }
 
+func (tc *TestClient) Search(ctx context.Context, index string, body string) (opensearchgoAPI.SearchHits, error) {
+	if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil {
+		return opensearchgoAPI.SearchHits{}, err
+	}
+
+	resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{
+		Indices: []string{index},
+		Body:    strings.NewReader(body),
+	})
+	if err != nil {
+		return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err)
+	}
+
+	return resp.Hits, nil
+}
+
 type testRequireClient struct {
 	tc *TestClient
 	t  *testing.T
@@ -228,3 +244,9 @@ func (trc *testRequireClient) DocumentCreate(index string, id, body string) {
 func (trc *testRequireClient) Update(index string, id, body string) {
 	require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body))
 }
+
+func (trc *testRequireClient) Search(index string, body string) opensearchgoAPI.SearchHits {
+	hits, err := trc.tc.Search(trc.t.Context(), index, body)
+	require.NoError(trc.t, err)
+	return hits
+}

services/search/pkg/opensearch/internal/test/testdata.go

@@ -13,12 +13,12 @@ var Testdata = struct {
 	Resources resourceTestdata
 }{
 	Resources: resourceTestdata{
-		Full: loadTestdata[engine.Resource]("resource_full.json"),
+		File: loadTestdata[engine.Resource]("resource_file.json"),
 	},
 }
 
 type resourceTestdata struct {
-	Full engine.Resource
+	File engine.Resource
 }
 
 func loadTestdata[D any](name string) D {