sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a flag to the reindex command to force a full reindex

Browse Source 
That can be helpful when the search service configuration has changed,
e.g. by enabling TIKA. Previously files that had already been indexed
were not indexed again and thus were no part of the fulltext index.

Fixes #2285
Fixes #2578
This commit is contained in:
André Duffeck
2026-04-14 08:20:28 +02:00
parent 366296eba6
commit 60bcc6b0f2
10 changed files with 112 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/antivirus/pkg/scanners/mocks/scanner.go
+1 -1
View File
@@ -6,7 +6,7 @@ package mocks
import (
"github.com/opencloud-eu/icap-client"
"github.com/stretchr/testify/mock"
mock "github.com/stretchr/testify/mock"
)
// NewScanner creates a new instance of Scanner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
services/search/README.md
+7
View File
@@ -118,6 +118,13 @@ It can also be used to re-index all spaces:
opencloud search index --all-spaces
```
Please note that a reindex only picks up new files. Files that have already been indexed are not indexed again, even if the configuration or the whole extractor has been changed. To force a full reindex you need to use the `force-reindex` flag:
```shell
opencloud search index --all-spaces --force-reindex
```
## Metrics
The search service exposes the following prometheus metrics at `<debug_endpoint>/metrics` (as configured using the `SEARCH_DEBUG_ADDR` env var):
services/search/pkg/command/index.go
+8 -1
View File
@@ -29,6 +29,7 @@ func Index(cfg *config.Config) *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
allSpacesFlag, _ := cmd.Flags().GetBool("all-spaces")
spaceFlag, _ := cmd.Flags().GetString("space")
forceReindexFlag, _ := cmd.Flags().GetBool("force-reindex")
if spaceFlag == "" && !allSpacesFlag {
return errors.New("either --space or --all-spaces is required")
}
@@ -48,7 +49,8 @@ func Index(cfg *config.Config) *cobra.Command {
c := searchsvc.NewSearchProviderService("eu.opencloud.api.search", grpcClient)
_, err = c.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
SpaceId: spaceFlag,
SpaceId: spaceFlag,
ForceReindex: forceReindexFlag,
}, func(opts *client.CallOptions) { opts.RequestTimeout = 10 * time.Minute })
if err != nil {
fmt.Println("failed to index space: " + err.Error())
@@ -68,6 +70,11 @@ func Index(cfg *config.Config) *cobra.Command {
false,
"index all spaces instead. This or --space is required.",
)
indexCmd.Flags().Bool(
"force-rescan",
false,
"force a rescan of all files, even if they are already indexed. This will make the indexing process much slower, but ensures that the index is up-to-date using the current search service configuration.",
)
return indexCmd
}
services/search/pkg/search/mocks/searcher.go
+14 -8
View File
@@ -40,16 +40,16 @@ func (_m *Searcher) EXPECT() *Searcher_Expecter {
}
// IndexSpace provides a mock function for the type Searcher
func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId) error {
ret := _mock.Called(rID)
func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error {
ret := _mock.Called(rID, forceRescan)
if len(ret) == 0 {
panic("no return value specified for IndexSpace")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId) error); ok {
r0 = returnFunc(rID)
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId, bool) error); ok {
r0 = returnFunc(rID, forceRescan)
} else {
r0 = ret.Error(0)
}
@@ -63,18 +63,24 @@ type Searcher_IndexSpace_Call struct {
// IndexSpace is a helper method to define mock.On call
// - rID *providerv1beta1.StorageSpaceId
func (_e *Searcher_Expecter) IndexSpace(rID interface{}) *Searcher_IndexSpace_Call {
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID)}
// - forceRescan bool
func (_e *Searcher_Expecter) IndexSpace(rID interface{}, forceRescan interface{}) *Searcher_IndexSpace_Call {
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID, forceRescan)}
}
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId)) *Searcher_IndexSpace_Call {
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool)) *Searcher_IndexSpace_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 *providerv1beta1.StorageSpaceId
if args[0] != nil {
arg0 = args[0].(*providerv1beta1.StorageSpaceId)
}
var arg1 bool
if args[1] != nil {
arg1 = args[1].(bool)
}
run(
arg0,
arg1,
)
})
return _c
@@ -85,7 +91,7 @@ func (_c *Searcher_IndexSpace_Call) Return(err error) *Searcher_IndexSpace_Call
return _c
}
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(rID *providerv1beta1.StorageSpaceId) error) *Searcher_IndexSpace_Call {
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error) *Searcher_IndexSpace_Call {
_c.Call.Return(run)
return _c
}
services/search/pkg/search/service.go
+7 -2
View File
@@ -46,7 +46,7 @@ const (
type Searcher interface {
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
IndexSpace(rID *provider.StorageSpaceId) error
IndexSpace(rID *provider.StorageSpaceId, forceRescan bool) error
PurgeDeleted(spaceID *provider.StorageSpaceId) error
TrashItem(rID *provider.ResourceId)
@@ -443,7 +443,7 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
}
// IndexSpace (re)indexes all resources of a given space.
func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool) error {
ownerCtx, err := getAuthContext(s.serviceAccountID, s.gatewaySelector, s.serviceAccountSecret, s.logger)
if err != nil {
return err
@@ -501,6 +501,11 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
}
s.logger.Debug().Str("path", ref.Path).Msg("Walking tree")
if forceRescan {
s.doUpsertItem(ref, batch)
return nil
}
searchRes, err := s.engine.Search(ownerCtx, &searchsvc.SearchIndexRequest{
Query: "id:" + storagespace.FormatResourceID(info.Id) + ` mtime>=` + utils.TSToTime(info.Mtime).Format(time.RFC3339Nano),
})
services/search/pkg/service/event/service.go
+1 -1
View File
@@ -74,7 +74,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
}
svc.indexSpaceDebouncer = NewSpaceDebouncer(time.Duration(debounceDuration)*time.Millisecond, 30*time.Second, func(id *provider.StorageSpaceId) {
if err := svc.index.IndexSpace(id); err != nil {
if err := svc.index.IndexSpace(id, false); err != nil {
svc.log.Error().Err(err).Interface("spaceID", id).Msg("error while indexing a space")
}
}, svc.log)
services/search/pkg/service/grpc/v0/service.go
+2 -2
View File
@@ -121,7 +121,7 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s
// IndexSpace (re)indexes all resources of a given space.
func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, _ *searchsvc.IndexSpaceResponse) error {
if in.GetSpaceId() != "" {
return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.GetSpaceId()})
return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.GetSpaceId()}, in.GetForceReindex())
}
// index all spaces instead
@@ -145,7 +145,7 @@ func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest,
}
for _, space := range resp.GetStorageSpaces() {
if err := s.searcher.IndexSpace(space.GetId()); err != nil {
if err := s.searcher.IndexSpace(space.GetId(), in.GetForceReindex()); err != nil {
return err
}
}