sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into search

Browse Source 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2022-04-14 11:56:26 +00:00
parent 2de90423b7 a332ba8c00
commit 37f64eb0e8
972 changed files with 2895 additions and 2949 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/search/cmd/search/main.go
+14
View File
@@ -0,0 +1,14 @@
package main
import (
"os"
"github.com/owncloud/ocis/extensions/search/pkg/command"
"github.com/owncloud/ocis/extensions/search/pkg/config/defaults"
)
func main() {
if err := command.Execute(defaults.DefaultConfig()); err != nil {
os.Exit(1)
}
}
extensions/search/pkg/command/health.go
+53
View File
@@ -0,0 +1,53 @@
package command
import (
"fmt"
"net/http"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/extensions/search/pkg/config/parser"
"github.com/owncloud/ocis/extensions/search/pkg/logging"
"github.com/urfave/cli/v2"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "health",
Usage: "check health status",
Category: "info",
Before: func(c *cli.Context) error {
return parser.ParseConfig(cfg)
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
resp, err := http.Get(
fmt.Sprintf(
"http://%s/healthz",
cfg.Debug.Addr,
),
)
if err != nil {
logger.Fatal().
Err(err).
Msg("Failed to request health check")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Fatal().
Int("code", resp.StatusCode).
Msg("Health seems to be in bad state")
}
logger.Debug().
Int("code", resp.StatusCode).
Msg("Health got a good state")
return nil
},
}
}
extensions/search/pkg/command/root.go
+64
View File
@@ -0,0 +1,64 @@
package command
import (
"context"
"os"
"github.com/owncloud/ocis/ocis-pkg/clihelper"
"github.com/thejerf/suture/v4"
"github.com/owncloud/ocis/extensions/search/pkg/config"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/urfave/cli/v2"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) cli.Commands {
return []*cli.Command{
// start this service
Server(cfg),
// interaction with this service
// infos about this service
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the ocis-search command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cli.App{
Name: "ocis-search",
Usage: "Serve search API for oCIS",
Commands: GetCommands(cfg),
})
cli.HelpFlag = &cli.BoolFlag{
Name: "help,h",
Usage: "Show the help",
}
return app.Run(os.Args)
}
// SutureService allows for the search command to be embedded and supervised by a suture supervisor tree.
type SutureService struct {
cfg *config.Config
}
// NewSutureService creates a new search.SutureService
func NewSutureService(cfg *ociscfg.Config) suture.Service {
cfg.Search.Commons = cfg.Commons
return SutureService{
cfg: cfg.Search,
}
}
func (s SutureService) Serve(ctx context.Context) error {
s.cfg.Context = ctx
if err := Execute(s.cfg); err != nil {
return err
}
return nil
}
extensions/search/pkg/command/server.go
+79
View File
@@ -0,0 +1,79 @@
package command
import (
"context"
"fmt"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/extensions/search/pkg/config/parser"
"github.com/owncloud/ocis/extensions/search/pkg/logging"
"github.com/owncloud/ocis/extensions/search/pkg/metrics"
"github.com/owncloud/ocis/extensions/search/pkg/server/debug"
"github.com/owncloud/ocis/extensions/search/pkg/server/grpc"
"github.com/owncloud/ocis/extensions/search/pkg/tracing"
"github.com/owncloud/ocis/ocis-pkg/version"
"github.com/urfave/cli/v2"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "server",
Usage: fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name),
Category: "server",
Before: func(c *cli.Context) error {
return parser.ParseConfig(cfg)
},
Action: func(c *cli.Context) error {
logger := logging.Configure(cfg.Service.Name, cfg.Log)
err := tracing.Configure(cfg)
if err != nil {
return err
}
gr := run.Group{}
ctx, cancel := func() (context.Context, context.CancelFunc) {
if cfg.Context == nil {
return context.WithCancel(context.Background())
}
return context.WithCancel(cfg.Context)
}()
defer cancel()
mtrcs := metrics.New()
mtrcs.BuildInfo.WithLabelValues(version.String).Set(1)
grpcServer := grpc.Server(
grpc.Config(cfg),
grpc.Logger(logger),
grpc.Name(cfg.Service.Name),
grpc.Context(ctx),
grpc.Metrics(mtrcs),
)
gr.Add(grpcServer.Run, func(_ error) {
logger.Info().Str("server", "grpc").Msg("shutting down server")
cancel()
})
server, err := debug.Server(
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(server.ListenAndServe, func(_ error) {
_ = server.Shutdown(ctx)
cancel()
})
return gr.Run()
},
}
}
extensions/search/pkg/command/version.go
+50
View File
@@ -0,0 +1,50 @@
package command
import (
"fmt"
"os"
"github.com/owncloud/ocis/ocis-pkg/registry"
"github.com/owncloud/ocis/ocis-pkg/version"
tw "github.com/olekukonko/tablewriter"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/urfave/cli/v2"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "version",
Usage: "print the version of this binary and the running extension instances",
Category: "info",
Action: func(c *cli.Context) error {
fmt.Println("Version: " + version.String)
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Println("")
reg := registry.GetRegistry()
services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name)
if err != nil {
fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err))
return err
}
if len(services) == 0 {
fmt.Println("No running " + cfg.Service.Name + " service found.")
return nil
}
table := tw.NewWriter(os.Stdout)
table.SetHeader([]string{"Version", "Address", "Id"})
table.SetAutoFormatHeaders(false)
for _, s := range services {
for _, n := range s.Nodes {
table.Append([]string{s.Version, n.Address, n.Id})
}
}
table.Render()
return nil
},
}
}
extensions/search/pkg/config/config.go
+25
View File
@@ -0,0 +1,25 @@
package config
import (
"context"
"github.com/owncloud/ocis/ocis-pkg/shared"
)
// Config combines all available configuration parts.
type Config struct {
*shared.Commons `ocisConfig:"-" yaml:"-"`
Service Service `ocisConfig:"-" yaml:"-"`
Tracing *Tracing `ocisConfig:"tracing"`
Log *Log `ocisConfig:"log"`
Debug Debug `ocisConfig:"debug"`
GRPC GRPC `ocisConfig:"grpc"`
Reva Reva `ocisConfig:"reva"`
TokenManager TokenManager `ocisConfig:"token_manager"`
Context context.Context `ocisConfig:"-" yaml:"-"`
}
extensions/search/pkg/config/debug.go
+9
View File
@@ -0,0 +1,9 @@
package config
// Debug defines the available debug configuration.
type Debug struct {
Addr string `ocisConfig:"addr" env:"SEARCH_DEBUG_ADDR"`
Token string `ocisConfig:"token" env:"SEARCH_DEBUG_TOKEN"`
Pprof bool `ocisConfig:"pprof" env:"SEARCH_DEBUG_PPROF"`
Zpages bool `ocisConfig:"zpages" env:"SEARCH_DEBUG_ZPAGES"`
}
extensions/search/pkg/config/defaults/defaultconfig.go
+56
View File
@@ -0,0 +1,56 @@
package defaults
import (
"github.com/owncloud/ocis/extensions/search/pkg/config"
)
func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9224",
Token: "",
},
GRPC: config.GRPC{
Addr: "127.0.0.1:9220",
Namespace: "com.owncloud.api",
},
Service: config.Service{
Name: "search",
},
Reva: config.Reva{
Address: "127.0.0.1:9142",
},
TokenManager: config.TokenManager{
JWTSecret: "Pive-Fumkiu4",
},
}
}
func EnsureDefaults(cfg *config.Config) {
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Log = &config.Log{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Log == nil {
cfg.Log = &config.Log{}
}
// provide with defaults for shared tracing, since we need a valid destination address for BindEnv.
if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil {
cfg.Tracing = &config.Tracing{
Enabled: cfg.Commons.Tracing.Enabled,
Type: cfg.Commons.Tracing.Type,
Endpoint: cfg.Commons.Tracing.Endpoint,
Collector: cfg.Commons.Tracing.Collector,
}
} else if cfg.Tracing == nil {
cfg.Tracing = &config.Tracing{}
}
}
func Sanitize(cfg *config.Config) {
// no http endpoint to be sanitized
}
extensions/search/pkg/config/grpc.go
+7
View File
@@ -0,0 +1,7 @@
package config
// GRPC defines the available grpc configuration.
type GRPC struct {
Addr string `ocisConfig:"addr" env:"ACCOUNTS_GRPC_ADDR" desc:"The address of the grpc service."`
Namespace string `ocisConfig:"-" yaml:"-"`
}
extensions/search/pkg/config/http.go
+8
View File
@@ -0,0 +1,8 @@
package config
// HTTP defines the available http configuration.
type HTTP struct {
Addr string `ocisConfig:"addr" env:"SEARCH_HTTP_ADDR"`
Namespace string `ocisConfig:"-" yaml:"-"`
Root string `ocisConfig:"root" env:"SEARCH_HTTP_ROOT"`
}
extensions/search/pkg/config/log.go
+9
View File
@@ -0,0 +1,9 @@
package config
// Log defines the available log configuration.
type Log struct {
Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;SEARCH_LOG_LEVEL"`
Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;SEARCH_LOG_PRETTY"`
Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;SEARCH_LOG_COLOR"`
File string `mapstructure:"file" env:"OCIS_LOG_FILE;SEARCH_LOG_FILE"`
}
extensions/search/pkg/config/parser/parse.go
+33
View File
@@ -0,0 +1,33 @@
package parser
import (
"errors"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/extensions/search/pkg/config/defaults"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/config/envdecode"
)
// ParseConfig loads accounts configuration from known paths.
func ParseConfig(cfg *config.Config) error {
_, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
defaults.EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
defaults.Sanitize(cfg)
return nil
}
extensions/search/pkg/config/reva.go
+11
View File
@@ -0,0 +1,11 @@
package config
// Reva defines all available REVA configuration.
type Reva struct {
Address string `ocisConfig:"address" env:"REVA_GATEWAY"`
}
// TokenManager is the config for using the reva token manager
type TokenManager struct {
JWTSecret string `ocisConfig:"jwt_secret" env:"OCIS_JWT_SECRET;SEARCH_JWT_SECRET"`
}
extensions/search/pkg/config/service.go
+6
View File
@@ -0,0 +1,6 @@
package config
// Service defines the available service configuration.
type Service struct {
Name string `ocisConfig:"-" yaml:"-"`
}
extensions/search/pkg/config/tracing.go
+9
View File
@@ -0,0 +1,9 @@
package config
// Tracing defines the available tracing configuration.
type Tracing struct {
Enabled bool `ocisConfig:"enabled" env:"OCIS_TRACING_ENABLED;SEARCH_TRACING_ENABLED"`
Type string `ocisConfig:"type" env:"OCIS_TRACING_TYPE;SEARCH_TRACING_TYPE"`
Endpoint string `ocisConfig:"endpoint" env:"OCIS_TRACING_ENDPOINT;SEARCH_TRACING_ENDPOINT"`
Collector string `ocisConfig:"collector" env:"OCIS_TRACING_COLLECTOR;SEARCH_TRACING_COLLECTOR"`
}
extensions/search/pkg/logging/logging.go
+17
View File
@@ -0,0 +1,17 @@
package logging
import (
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// LoggerFromConfig initializes a service-specific logger instance.
func Configure(name string, cfg *config.Log) log.Logger {
return log.NewLogger(
log.Name(name),
log.Level(cfg.Level),
log.Pretty(cfg.Pretty),
log.Color(cfg.Color),
log.File(cfg.File),
)
}
extensions/search/pkg/metrics/metrics.go
+33
View File
@@ -0,0 +1,33 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// Namespace defines the namespace for the defines metrics.
Namespace = "ocis"
// Subsystem defines the subsystem for the defines metrics.
Subsystem = "search"
)
// Metrics defines the available metrics of this service.
type Metrics struct {
// Counter *prometheus.CounterVec
BuildInfo *prometheus.GaugeVec
}
// New initializes the available metrics.
func New() *Metrics {
m := &Metrics{
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "build_info",
Help: "Build information",
}, []string{"version"}),
}
_ = prometheus.Register(m.BuildInfo)
// TODO: implement metrics
return m
}
extensions/search/pkg/search/index/index.go
+145
View File
@@ -0,0 +1,145 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package index
import (
"context"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/mapping"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
type indexDocument struct {
RootID string
Path string
ID string
Name string
Size uint64
}
// Index represents a bleve based search index
type Index struct {
bleveIndex bleve.Index
}
// NewPersisted returns a new instance of Index with the data being persisted in the given directory
func NewPersisted(path string) (*Index, error) {
bi, err := bleve.New(path, BuildMapping())
if err != nil {
return nil, err
}
return New(bi)
}
// New returns a new instance of Index using the given bleve Index as the backend
func New(bleveIndex bleve.Index) (*Index, error) {
return &Index{
bleveIndex: bleveIndex,
}, nil
}
// Add adds a new entity to the Index
func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error {
entity := toEntity(ref, ri)
return i.bleveIndex.Index(idToBleveId(ri.Id), entity)
}
// Remove removes an entity from the index
func (i *Index) Remove(ri *sprovider.ResourceInfo) error {
return i.bleveIndex.Delete(idToBleveId(ri.Id))
}
// Search searches the index according to the criteria specified in the given SearchIndexRequest
func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) {
query := bleve.NewConjunctionQuery(
bleve.NewQueryStringQuery(req.Query),
bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space
bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space
)
bleveReq := bleve.NewSearchRequest(query)
bleveReq.Fields = []string{"*"}
res, err := i.bleveIndex.Search(bleveReq)
if err != nil {
return nil, err
}
matches := []*searchmsg.Match{}
for _, h := range res.Hits {
match, err := fromFields(h.Fields)
if err != nil {
return nil, err
}
matches = append(matches, match)
}
return &searchsvc.SearchIndexResponse{
Matches: matches,
}, nil
}
// BuildMapping builds a bleve index mapping which can be used for indexing
func BuildMapping() mapping.IndexMapping {
indexMapping := bleve.NewIndexMapping()
indexMapping.DefaultAnalyzer = keyword.Name
return indexMapping
}
func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument {
return &indexDocument{
RootID: idToBleveId(ref.ResourceId),
Path: ref.Path,
ID: idToBleveId(ri.Id),
Name: ri.Path,
Size: ri.Size,
}
}
func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) {
rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2)
IDParts := strings.SplitN(fields["ID"].(string), "!", 2)
return &searchmsg.Match{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rootIDParts[0],
OpaqueId: rootIDParts[1],
},
Path: fields["Path"].(string),
},
Id: &searchmsg.ResourceID{
StorageId: IDParts[0],
OpaqueId: IDParts[1],
},
Name: fields["Name"].(string),
Size: uint64(fields["Size"].(float64)),
},
}, nil
}
func idToBleveId(id *sprovider.ResourceId) string {
return id.StorageId + "!" + id.OpaqueId
}
extensions/search/pkg/search/index/index_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package index_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestIndex(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Index Suite")
}
extensions/search/pkg/search/index/index_test.go
+221
View File
@@ -0,0 +1,221 @@
package index_test
import (
"context"
"github.com/blevesearch/bleve/v2"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/owncloud/ocis/extensions/search/pkg/search/index"
searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Index", func() {
var (
i *index.Index
bleveIndex bleve.Index
ref *sprovider.Reference
ri *sprovider.ResourceInfo
ctx context.Context
)
BeforeEach(func() {
var err error
bleveIndex, err = bleve.NewMemOnly(index.BuildMapping())
Expect(err).ToNot(HaveOccurred())
i, err = index.New(bleveIndex)
Expect(err).ToNot(HaveOccurred())
ref = &sprovider.Reference{
ResourceId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "rootopaqueid",
},
Path: "./foo.pdf",
}
ri = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "opaqueid",
},
Path: "foo.pdf",
Size: 12345,
}
})
Describe("New", func() {
It("returns a new index instance", func() {
i, err := index.New(bleveIndex)
Expect(err).ToNot(HaveOccurred())
Expect(i).ToNot(BeNil())
})
})
Describe("NewPersisted", func() {
It("returns a new index instance", func() {
i, err := index.NewPersisted("")
Expect(err).ToNot(HaveOccurred())
Expect(i).ToNot(BeNil())
})
})
Describe("Search", func() {
Context("with a file in the root of the space", func() {
BeforeEach(func() {
err := i.Add(ref, ri)
Expect(err).ToNot(HaveOccurred())
})
It("scopes the search to the specified space", func() {
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: "differentstorageid",
OpaqueId: "differentopaqueid",
},
},
Query: "foo.pdf",
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(0))
})
It("finds files by name, prefix or substring match", func() {
queries := []string{"foo.pdf", "foo*", "*oo.p*"}
for _, query := range queries {
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: ref.ResourceId.StorageId,
OpaqueId: ref.ResourceId.OpaqueId,
},
},
Query: query,
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query)
Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(ref.ResourceId.OpaqueId))
Expect(res.Matches[0].Entity.Ref.Path).To(Equal(ref.Path))
Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId))
Expect(res.Matches[0].Entity.Name).To(Equal(ri.Path))
Expect(res.Matches[0].Entity.Size).To(Equal(ri.Size))
}
})
Context("and an additional file in a subdirectory", func() {
var (
nestedRef *sprovider.Reference
nestedRI *sprovider.ResourceInfo
)
BeforeEach(func() {
nestedRef = &sprovider.Reference{
ResourceId: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "rootopaqueid",
},
Path: "./nested/nestedpdf.pdf",
}
nestedRI = &sprovider.ResourceInfo{
Id: &sprovider.ResourceId{
StorageId: "storageid",
OpaqueId: "nestedopaqueid",
},
Path: "nestedpdf.pdf",
Size: 12345,
}
err := i.Add(nestedRef, nestedRI)
Expect(err).ToNot(HaveOccurred())
})
It("finds files living deeper in the tree by filename, prefix or substring match", func() {
queries := []string{"nestedpdf.pdf", "nested*", "*tedpdf.*"}
for _, query := range queries {
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: ref.ResourceId.StorageId,
OpaqueId: ref.ResourceId.OpaqueId,
},
},
Query: query,
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query)
Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(nestedRef.ResourceId.OpaqueId))
Expect(res.Matches[0].Entity.Ref.Path).To(Equal(nestedRef.Path))
Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(nestedRI.Id.OpaqueId))
Expect(res.Matches[0].Entity.Name).To(Equal(nestedRI.Path))
Expect(res.Matches[0].Entity.Size).To(Equal(nestedRI.Size))
}
})
It("does not find the higher levels when limiting the searched directory", func() {
res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: ref.ResourceId.StorageId,
OpaqueId: ref.ResourceId.OpaqueId,
},
Path: "./nested/",
},
Query: "foo.pdf",
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(0))
})
})
})
})
Describe("Index", func() {
It("adds a resourceInfo to the index", func() {
err := i.Add(ref, ri)
Expect(err).ToNot(HaveOccurred())
count, err := bleveIndex.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
query := bleve.NewMatchQuery("foo.pdf")
res, err := bleveIndex.Search(bleve.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
})
It("updates an existing resource in the index", func() {
err := i.Add(ref, ri)
Expect(err).ToNot(HaveOccurred())
count, _ := bleveIndex.DocCount()
Expect(count).To(Equal(uint64(1)))
err = i.Add(ref, ri)
Expect(err).ToNot(HaveOccurred())
count, _ = bleveIndex.DocCount()
Expect(count).To(Equal(uint64(1)))
})
})
Describe("Remove", func() {
It("removes a resource from the index", func() {
err := i.Add(ref, ri)
Expect(err).ToNot(HaveOccurred())
count, _ := bleveIndex.DocCount()
Expect(count).To(Equal(uint64(1)))
err = i.Remove(ri)
Expect(err).ToNot(HaveOccurred())
count, _ = bleveIndex.DocCount()
Expect(count).To(Equal(uint64(0)))
})
})
})
extensions/search/pkg/search/index/mocks/BleveIndex.go
+415
View File
@@ -0,0 +1,415 @@
// Code generated by mockery v2.10.0. DO NOT EDIT.
package mocks
import (
context "context"
bleve "github.com/blevesearch/bleve/v2"
index "github.com/blevesearch/bleve_index_api"
mapping "github.com/blevesearch/bleve/v2/mapping"
mock "github.com/stretchr/testify/mock"
)
// BleveIndex is an autogenerated mock type for the BleveIndex type
type BleveIndex struct {
mock.Mock
}
// Advanced provides a mock function with given fields:
func (_m *BleveIndex) Advanced() (index.Index, error) {
ret := _m.Called()
var r0 index.Index
if rf, ok := ret.Get(0).(func() index.Index); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(index.Index)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Batch provides a mock function with given fields: b
func (_m *BleveIndex) Batch(b *bleve.Batch) error {
ret := _m.Called(b)
var r0 error
if rf, ok := ret.Get(0).(func(*bleve.Batch) error); ok {
r0 = rf(b)
} else {
r0 = ret.Error(0)
}
return r0
}
// Close provides a mock function with given fields:
func (_m *BleveIndex) Close() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Delete provides a mock function with given fields: id
func (_m *BleveIndex) Delete(id string) error {
ret := _m.Called(id)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(id)
} else {
r0 = ret.Error(0)
}
return r0
}
// DeleteInternal provides a mock function with given fields: key
func (_m *BleveIndex) DeleteInternal(key []byte) error {
ret := _m.Called(key)
var r0 error
if rf, ok := ret.Get(0).(func([]byte) error); ok {
r0 = rf(key)
} else {
r0 = ret.Error(0)
}
return r0
}
// DocCount provides a mock function with given fields:
func (_m *BleveIndex) DocCount() (uint64, error) {
ret := _m.Called()
var r0 uint64
if rf, ok := ret.Get(0).(func() uint64); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(uint64)
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Document provides a mock function with given fields: id
func (_m *BleveIndex) Document(id string) (index.Document, error) {
ret := _m.Called(id)
var r0 index.Document
if rf, ok := ret.Get(0).(func(string) index.Document); ok {
r0 = rf(id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(index.Document)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FieldDict provides a mock function with given fields: field
func (_m *BleveIndex) FieldDict(field string) (index.FieldDict, error) {
ret := _m.Called(field)
var r0 index.FieldDict
if rf, ok := ret.Get(0).(func(string) index.FieldDict); ok {
r0 = rf(field)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(index.FieldDict)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(field)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FieldDictPrefix provides a mock function with given fields: field, termPrefix
func (_m *BleveIndex) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {
ret := _m.Called(field, termPrefix)
var r0 index.FieldDict
if rf, ok := ret.Get(0).(func(string, []byte) index.FieldDict); ok {
r0 = rf(field, termPrefix)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(index.FieldDict)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, []byte) error); ok {
r1 = rf(field, termPrefix)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FieldDictRange provides a mock function with given fields: field, startTerm, endTerm
func (_m *BleveIndex) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
ret := _m.Called(field, startTerm, endTerm)
var r0 index.FieldDict
if rf, ok := ret.Get(0).(func(string, []byte, []byte) index.FieldDict); ok {
r0 = rf(field, startTerm, endTerm)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(index.FieldDict)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(string, []byte, []byte) error); ok {
r1 = rf(field, startTerm, endTerm)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Fields provides a mock function with given fields:
func (_m *BleveIndex) Fields() ([]string, error) {
ret := _m.Called()
var r0 []string
if rf, ok := ret.Get(0).(func() []string); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetInternal provides a mock function with given fields: key
func (_m *BleveIndex) GetInternal(key []byte) ([]byte, error) {
ret := _m.Called(key)
var r0 []byte
if rf, ok := ret.Get(0).(func([]byte) []byte); ok {
r0 = rf(key)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
var r1 error
if rf, ok := ret.Get(1).(func([]byte) error); ok {
r1 = rf(key)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Index provides a mock function with given fields: id, data
func (_m *BleveIndex) Index(id string, data interface{}) error {
ret := _m.Called(id, data)
var r0 error
if rf, ok := ret.Get(0).(func(string, interface{}) error); ok {
r0 = rf(id, data)
} else {
r0 = ret.Error(0)
}
return r0
}
// Mapping provides a mock function with given fields:
func (_m *BleveIndex) Mapping() mapping.IndexMapping {
ret := _m.Called()
var r0 mapping.IndexMapping
if rf, ok := ret.Get(0).(func() mapping.IndexMapping); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(mapping.IndexMapping)
}
}
return r0
}
// Name provides a mock function with given fields:
func (_m *BleveIndex) Name() string {
ret := _m.Called()
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// NewBatch provides a mock function with given fields:
func (_m *BleveIndex) NewBatch() *bleve.Batch {
ret := _m.Called()
var r0 *bleve.Batch
if rf, ok := ret.Get(0).(func() *bleve.Batch); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*bleve.Batch)
}
}
return r0
}
// Search provides a mock function with given fields: req
func (_m *BleveIndex) Search(req *bleve.SearchRequest) (*bleve.SearchResult, error) {
ret := _m.Called(req)
var r0 *bleve.SearchResult
if rf, ok := ret.Get(0).(func(*bleve.SearchRequest) *bleve.SearchResult); ok {
r0 = rf(req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*bleve.SearchResult)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*bleve.SearchRequest) error); ok {
r1 = rf(req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SearchInContext provides a mock function with given fields: ctx, req
func (_m *BleveIndex) SearchInContext(ctx context.Context, req *bleve.SearchRequest) (*bleve.SearchResult, error) {
ret := _m.Called(ctx, req)
var r0 *bleve.SearchResult
if rf, ok := ret.Get(0).(func(context.Context, *bleve.SearchRequest) *bleve.SearchResult); ok {
r0 = rf(ctx, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*bleve.SearchResult)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *bleve.SearchRequest) error); ok {
r1 = rf(ctx, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SetInternal provides a mock function with given fields: key, val
func (_m *BleveIndex) SetInternal(key []byte, val []byte) error {
ret := _m.Called(key, val)
var r0 error
if rf, ok := ret.Get(0).(func([]byte, []byte) error); ok {
r0 = rf(key, val)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetName provides a mock function with given fields: _a0
func (_m *BleveIndex) SetName(_a0 string) {
_m.Called(_a0)
}
// Stats provides a mock function with given fields:
func (_m *BleveIndex) Stats() *bleve.IndexStat {
ret := _m.Called()
var r0 *bleve.IndexStat
if rf, ok := ret.Get(0).(func() *bleve.IndexStat); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*bleve.IndexStat)
}
}
return r0
}
// StatsMap provides a mock function with given fields:
func (_m *BleveIndex) StatsMap() map[string]interface{} {
ret := _m.Called()
var r0 map[string]interface{}
if rf, ok := ret.Get(0).(func() map[string]interface{}); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[string]interface{})
}
}
return r0
}
extensions/search/pkg/search/mocks/IndexClient.go
+39
View File
@@ -0,0 +1,39 @@
// Code generated by mockery v2.10.0. DO NOT EDIT.
package mocks
import (
context "context"
mock "github.com/stretchr/testify/mock"
v0 "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
// IndexClient is an autogenerated mock type for the IndexClient type
type IndexClient struct {
mock.Mock
}
// Search provides a mock function with given fields: ctx, req
func (_m *IndexClient) Search(ctx context.Context, req *v0.SearchIndexRequest) (*v0.SearchIndexResponse, error) {
ret := _m.Called(ctx, req)
var r0 *v0.SearchIndexResponse
if rf, ok := ret.Get(0).(func(context.Context, *v0.SearchIndexRequest) *v0.SearchIndexResponse); ok {
r0 = rf(ctx, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v0.SearchIndexResponse)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *v0.SearchIndexRequest) error); ok {
r1 = rf(ctx, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
extensions/search/pkg/search/mocks/ProviderClient.go
+39
View File
@@ -0,0 +1,39 @@
// Code generated by mockery v2.10.0. DO NOT EDIT.
package mocks
import (
context "context"
mock "github.com/stretchr/testify/mock"
v0 "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
// ProviderClient is an autogenerated mock type for the ProviderClient type
type ProviderClient struct {
mock.Mock
}
// Search provides a mock function with given fields: ctx, req
func (_m *ProviderClient) Search(ctx context.Context, req *v0.SearchRequest) (*v0.SearchResponse, error) {
ret := _m.Called(ctx, req)
var r0 *v0.SearchResponse
if rf, ok := ret.Get(0).(func(context.Context, *v0.SearchRequest) *v0.SearchResponse); ok {
r0 = rf(ctx, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v0.SearchResponse)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *v0.SearchRequest) error); ok {
r1 = rf(ctx, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
extensions/search/pkg/search/provider/provider_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package provider_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestProvider(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Provider Suite")
}
extensions/search/pkg/search/provider/searchprovider.go
+89
View File
@@ -0,0 +1,89 @@
package provider
import (
"context"
"strings"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/owncloud/ocis/extensions/search/pkg/search"
searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
type Provider struct {
gwClient gateway.GatewayAPIClient
indexClient search.IndexClient
}
func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient) *Provider {
return &Provider{
gwClient: gwClient,
indexClient: indexClient,
}
}
func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
if req.Query == "" {
return nil, errtypes.PreconditionFailed("empty query provided")
}
listSpacesRes, err := p.gwClient.ListStorageSpaces(ctx, &providerv1beta1.ListStorageSpacesRequest{
Opaque: &typesv1beta1.Opaque{Map: map[string]*typesv1beta1.OpaqueEntry{
"path": {
Decoder: "plain",
Value: []byte("/"),
},
}},
})
if err != nil {
return nil, err
}
matches := []*searchmsg.Match{}
for _, space := range listSpacesRes.StorageSpaces {
pathPrefix := ""
if space.SpaceType == "grant" {
gpRes, err := p.gwClient.GetPath(ctx, &providerv1beta1.GetPathRequest{
ResourceId: space.Root,
})
if err != nil {
return nil, err
}
if gpRes.Status.Code != rpcv1beta1.Code_CODE_OK {
return nil, errtypes.NewErrtypeFromStatus(gpRes.Status)
}
pathPrefix = utils.MakeRelativePath(gpRes.Path)
}
res, err := p.indexClient.Search(ctx, &searchsvc.SearchIndexRequest{
Query: req.Query,
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: space.Root.StorageId,
OpaqueId: space.Root.OpaqueId,
},
Path: pathPrefix,
},
})
if err != nil {
return nil, err
}
for _, match := range res.Matches {
if pathPrefix != "" {
match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, pathPrefix))
}
matches = append(matches, match)
}
}
return &searchsvc.SearchResponse{
Matches: matches,
}, nil
}
extensions/search/pkg/search/provider/searchprovider_test.go
+252
View File
@@ -0,0 +1,252 @@
package provider_test
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/mock"
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/v2/pkg/rgrpc/status"
cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks"
"github.com/owncloud/ocis/extensions/search/pkg/search/mocks"
provider "github.com/owncloud/ocis/extensions/search/pkg/search/provider"
searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
var _ = Describe("Searchprovider", func() {
var (
p *provider.Provider
gwClient *cs3mocks.GatewayAPIClient
indexClient *mocks.IndexClient
ctx context.Context
otherUser = &userv1beta1.User{
Id: &userv1beta1.UserId{
OpaqueId: "otheruser",
},
}
personalSpace = &sprovider.StorageSpace{
Opaque: &typesv1beta1.Opaque{
Map: map[string]*typesv1beta1.OpaqueEntry{
"path": {
Decoder: "plain",
Value: []byte("/foo"),
},
},
},
Id: &sprovider.StorageSpaceId{OpaqueId: "personalspace"},
Root: &sprovider.ResourceId{OpaqueId: "personalspaceroot"},
Name: "personalspace",
}
)
BeforeEach(func() {
ctx = context.Background()
gwClient = &cs3mocks.GatewayAPIClient{}
indexClient = &mocks.IndexClient{}
p = provider.New(gwClient, indexClient)
})
Describe("New", func() {
It("returns a new instance", func() {
p := provider.New(gwClient, indexClient)
Expect(p).ToNot(BeNil())
})
})
Describe("Search", func() {
It("fails when an empty query is given", func() {
res, err := p.Search(ctx, &searchsvc.SearchRequest{
Query: "",
})
Expect(err).To(HaveOccurred())
Expect(res).To(BeNil())
})
Context("with a personal space", func() {
BeforeEach(func() {
gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool {
p := string(req.Opaque.Map["path"].Value)
return p == "/"
})).Return(&sprovider.ListStorageSpacesResponse{
Status: status.NewOK(ctx),
StorageSpaces: []*sprovider.StorageSpace{personalSpace},
}, nil)
indexClient.On("Search", mock.Anything, mock.Anything).Return(&searchsvc.SearchIndexResponse{
Matches: []*searchmsg.Match{
{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: personalSpace.Root.StorageId,
OpaqueId: personalSpace.Root.OpaqueId,
},
Path: "./path/to/Foo.pdf",
},
Id: &searchmsg.ResourceID{
StorageId: personalSpace.Root.StorageId,
OpaqueId: "foo-id",
},
Name: "Foo.pdf",
},
},
},
}, nil)
})
It("searches the personal user space", func() {
res, err := p.Search(ctx, &searchsvc.SearchRequest{
Query: "foo",
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(1))
match := res.Matches[0]
Expect(match.Entity.Id.OpaqueId).To(Equal("foo-id"))
Expect(match.Entity.Name).To(Equal("Foo.pdf"))
Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(personalSpace.Root.OpaqueId))
Expect(match.Entity.Ref.Path).To(Equal("./path/to/Foo.pdf"))
indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
return req.Query == "foo" && req.Ref.ResourceId.OpaqueId == personalSpace.Root.OpaqueId && req.Ref.Path == ""
}))
})
})
Context("with received shares", func() {
var (
grantSpace *sprovider.StorageSpace
)
BeforeEach(func() {
grantSpace = &sprovider.StorageSpace{
SpaceType: "grant",
Owner: otherUser,
Id: &sprovider.StorageSpaceId{OpaqueId: "otherspaceroot!otherspacegrant"},
Root: &sprovider.ResourceId{StorageId: "otherspaceroot", OpaqueId: "otherspacegrant"},
Name: "grantspace",
}
gwClient.On("GetPath", mock.Anything, mock.Anything).Return(&sprovider.GetPathResponse{
Status: status.NewOK(ctx),
Path: "/grant/path",
}, nil)
})
It("searches the received spaces (grants)", func() {
gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool {
p := string(req.Opaque.Map["path"].Value)
return p == "/"
})).Return(&sprovider.ListStorageSpacesResponse{
Status: status.NewOK(ctx),
StorageSpaces: []*sprovider.StorageSpace{grantSpace},
}, nil)
indexClient.On("Search", mock.Anything, mock.Anything).Return(&searchsvc.SearchIndexResponse{
Matches: []*searchmsg.Match{
{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: grantSpace.Root.StorageId,
OpaqueId: grantSpace.Root.OpaqueId,
},
Path: "./grant/path/to/Shared.pdf",
},
Id: &searchmsg.ResourceID{
StorageId: grantSpace.Root.StorageId,
OpaqueId: "grant-shared-id",
},
Name: "Shared.pdf",
},
},
},
}, nil)
res, err := p.Search(ctx, &searchsvc.SearchRequest{
Query: "foo",
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(1))
match := res.Matches[0]
Expect(match.Entity.Id.OpaqueId).To(Equal("grant-shared-id"))
Expect(match.Entity.Name).To(Equal("Shared.pdf"))
Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(grantSpace.Root.OpaqueId))
Expect(match.Entity.Ref.Path).To(Equal("./to/Shared.pdf"))
indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
return req.Query == "foo" && req.Ref.ResourceId.OpaqueId == grantSpace.Root.OpaqueId && req.Ref.Path == "./grant/path"
}))
})
It("finds matches in both the personal space AND the grant", func() {
gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool {
p := string(req.Opaque.Map["path"].Value)
return p == "/"
})).Return(&sprovider.ListStorageSpacesResponse{
Status: status.NewOK(ctx),
StorageSpaces: []*sprovider.StorageSpace{personalSpace, grantSpace},
}, nil)
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
return req.Ref.ResourceId.OpaqueId == grantSpace.Root.OpaqueId
})).Return(&searchsvc.SearchIndexResponse{
Matches: []*searchmsg.Match{
{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: grantSpace.Root.StorageId,
OpaqueId: grantSpace.Root.OpaqueId,
},
Path: "./grant/path/to/Shared.pdf",
},
Id: &searchmsg.ResourceID{
StorageId: grantSpace.Root.StorageId,
OpaqueId: "grant-shared-id",
},
Name: "Shared.pdf",
},
},
},
}, nil)
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
return req.Ref.ResourceId.OpaqueId == personalSpace.Root.OpaqueId
})).Return(&searchsvc.SearchIndexResponse{
Matches: []*searchmsg.Match{
{
Entity: &searchmsg.Entity{
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: personalSpace.Root.StorageId,
OpaqueId: personalSpace.Root.OpaqueId,
},
Path: "./path/to/Foo.pdf",
},
Id: &searchmsg.ResourceID{
StorageId: personalSpace.Root.StorageId,
OpaqueId: "foo-id",
},
Name: "Foo.pdf",
},
},
},
}, nil)
res, err := p.Search(ctx, &searchsvc.SearchRequest{
Query: "foo",
})
Expect(err).ToNot(HaveOccurred())
Expect(res).ToNot(BeNil())
Expect(len(res.Matches)).To(Equal(2))
ids := []string{res.Matches[0].Entity.Id.OpaqueId, res.Matches[1].Entity.Id.OpaqueId}
Expect(ids).To(ConsistOf("foo-id", "grant-shared-id"))
})
})
})
})
extensions/search/pkg/search/search.go
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2018-2022 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package search
import (
"context"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
//go:generate mockery --name=ProviderClient
//go:generate mockery --name=IndexClient
// ProviderClient is the interface to the search provider service
type ProviderClient interface {
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
}
// IndexClient is the interface to the search index
type IndexClient interface {
Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error)
}
extensions/search/pkg/search/search_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package search_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestSearch(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Search Suite")
}
extensions/search/pkg/server/debug/option.go
+50
View File
@@ -0,0 +1,50 @@
package debug
import (
"context"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Context context.Context
Config *config.Config
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
extensions/search/pkg/server/debug/server.go
+59
View File
@@ -0,0 +1,59 @@
package debug
import (
"io"
"net/http"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/service/debug"
"github.com/owncloud/ocis/ocis-pkg/version"
)
// Server initializes the debug service and server.
func Server(opts ...Option) (*http.Server, error) {
options := newOptions(opts...)
return debug.NewService(
debug.Logger(options.Logger),
debug.Name(options.Config.Service.Name),
debug.Version(version.String),
debug.Address(options.Config.Debug.Addr),
debug.Token(options.Config.Debug.Token),
debug.Pprof(options.Config.Debug.Pprof),
debug.Zpages(options.Config.Debug.Zpages),
debug.Health(health(options.Config)),
debug.Ready(ready(options.Config)),
), nil
}
// health implements the health check.
func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
// TODO: check if services are up and running
_, err := io.WriteString(w, http.StatusText(http.StatusOK))
// io.WriteString should not fail but if it does we want to know.
if err != nil {
panic(err)
}
}
}
// ready implements the ready check.
func ready(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
// TODO: check if services are up and running
_, err := io.WriteString(w, http.StatusText(http.StatusOK))
// io.WriteString should not fail but if it does we want to know.
if err != nil {
panic(err)
}
}
}
extensions/search/pkg/server/grpc/option.go
+85
View File
@@ -0,0 +1,85 @@
package grpc
import (
"context"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/extensions/search/pkg/metrics"
svc "github.com/owncloud/ocis/extensions/search/pkg/service/v0"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/urfave/cli/v2"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Flags []cli.Flag
Handler *svc.Service
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Name provides a name for the service.
func Name(val string) Option {
return func(o *Options) {
o.Name = val
}
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
return func(o *Options) {
o.Metrics = val
}
}
// Flags provides a function to set the flags option.
func Flags(val []cli.Flag) Option {
return func(o *Options) {
o.Flags = append(o.Flags, val...)
}
}
// Handler provides a function to set the handler option.
func Handler(val *svc.Service) Option {
return func(o *Options) {
o.Handler = val
}
}
extensions/search/pkg/server/grpc/server.go
+39
View File
@@ -0,0 +1,39 @@
package grpc
import (
svc "github.com/owncloud/ocis/extensions/search/pkg/service/v0"
"github.com/owncloud/ocis/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/ocis-pkg/version"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
// Server initializes a new go-micro service ready to run
func Server(opts ...Option) grpc.Service {
options := newOptions(opts...)
service := grpc.NewService(
grpc.Name(options.Config.Service.Name),
grpc.Context(options.Context),
grpc.Address(options.Config.GRPC.Addr),
grpc.Namespace(options.Config.GRPC.Namespace),
grpc.Logger(options.Logger),
grpc.Flags(options.Flags...),
grpc.Version(version.String),
)
handle, err := svc.NewHandler(
svc.Config(options.Config),
svc.Logger(options.Logger),
)
if err != nil {
options.Logger.Error().
Err(err).
Msg("Error initializing search service")
return grpc.Service{}
}
_ = searchsvc.RegisterSearchProviderHandler(
service.Server(),
handle,
)
return service
}
extensions/search/pkg/service/v0/option.go
+39
View File
@@ -0,0 +1,39 @@
package service
import (
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Config *config.Config
}
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the Logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Config provides a function to set the Config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
extensions/search/pkg/service/v0/service.go
+66
View File
@@ -0,0 +1,66 @@
package service
import (
"context"
"github.com/blevesearch/bleve/v2"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/owncloud/ocis/extensions/search/pkg/config"
"github.com/owncloud/ocis/extensions/search/pkg/search"
"github.com/owncloud/ocis/extensions/search/pkg/search/index"
searchprovider "github.com/owncloud/ocis/extensions/search/pkg/search/provider"
"github.com/owncloud/ocis/ocis-pkg/log"
searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
)
// NewHandler returns a service implementation for Service.
func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) {
options := newOptions(opts...)
logger := options.Logger
cfg := options.Config
bleveIndex, err := bleve.NewMemOnly(index.BuildMapping())
if err != nil {
return nil, err
}
index, err := index.New(bleveIndex)
if err != nil {
return nil, err
}
gwclient, err := pool.GetGatewayServiceClient(cfg.Reva.Address)
if err != nil {
logger.Fatal().Err(err).Str("addr", cfg.Reva.Address).Msg("could not get reva client")
}
provider := searchprovider.New(gwclient, index)
return &Service{
id: cfg.GRPC.Namespace + "." + cfg.Service.Name,
log: logger,
Config: cfg,
provider: provider,
}, nil
}
// Service implements the searchServiceHandler interface
type Service struct {
id string
log log.Logger
Config *config.Config
provider search.ProviderClient
}
func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *searchsvc.SearchResponse) error {
res, err := s.provider.Search(ctx, &searchsvc.SearchRequest{
Query: in.Query,
})
if err != nil {
return nil
}
out.Matches = res.Matches
out.NextPageToken = res.NextPageToken
return nil
}
extensions/search/pkg/tracing/tracing.go
+23
View File
@@ -0,0 +1,23 @@
package tracing
import (
"github.com/owncloud/ocis/extensions/search/pkg/config"
pkgtrace "github.com/owncloud/ocis/ocis-pkg/tracing"
"go.opentelemetry.io/otel/trace"
)
var (
// TraceProvider is the global trace provider for the proxy service.
TraceProvider = trace.NewNoopTracerProvider()
)
func Configure(cfg *config.Config) error {
var err error
if cfg.Tracing.Enabled {
if TraceProvider, err = pkgtrace.GetTraceProvider(cfg.Tracing.Endpoint, cfg.Tracing.Collector, cfg.Service.Name, cfg.Tracing.Type); err != nil {
return err
}
}
return nil
}