Initial QSfera import

Server/services/search/pkg/command/health.go

@@ -0,0 +1,53 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/services/search/pkg/config"
+	"github.com/qsfera/server/services/search/pkg/config/parser"
+
+	"github.com/spf13/cobra"
+)
+
+// Health is the entrypoint for the health command.
+func Health(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "health",
+		Short: "check health status",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return parser.ParseConfig(cfg)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
+
+			resp, err := http.Get(
+				fmt.Sprintf(
+					"http://%s/healthz",
+					cfg.Debug.Addr,
+				),
+			)
+
+			if err != nil {
+				logger.Fatal().
+					Err(err).
+					Msg("Failed to request health check")
+			}
+
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				logger.Fatal().
+					Int("code", resp.StatusCode).
+					Msg("Health seems to be in bad state")
+			}
+
+			logger.Debug().
+				Int("code", resp.StatusCode).
+				Msg("Health got a good state")
+
+			return nil
+		},
+	}
+}

Server/services/search/pkg/command/index.go

@@ -0,0 +1,99 @@
+package command
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/qsfera/server/pkg/config/configlog"
+	searchsvc "github.com/qsfera/server/protogen/gen/qsfera/services/search/v0"
+	"github.com/qsfera/server/services/search/pkg/config"
+	"github.com/qsfera/server/services/search/pkg/config/parser"
+
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// Index is the entrypoint for the server command.
+func Index(cfg *config.Config) *cobra.Command {
+	indexCmd := &cobra.Command{
+		Use:     "index",
+		Short:   "index the files for one one more users",
+		Aliases: []string{"i"},
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			allSpacesFlag, _ := cmd.Flags().GetBool("all-spaces")
+			spaceFlag, _ := cmd.Flags().GetString("space")
+			forceRescanFlag, _ := cmd.Flags().GetBool("force-rescan")
+			endpointFlag, _ := cmd.Flags().GetString("endpoint")
+			insecureFlag, _ := cmd.Flags().GetBool("insecure")
+			if spaceFlag == "" && !allSpacesFlag {
+				return errors.New("either --space or --all-spaces is required")
+			}
+
+			var dialOpts []grpc.DialOption
+			if cfg.GRPCClientTLS.Mode == "insecure" || insecureFlag {
+				dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+			} else {
+				dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
+					MinVersion: tls.VersionTLS12,
+				})))
+			}
+
+			conn, err := grpc.NewClient(endpointFlag, dialOpts...)
+			if err != nil {
+				return fmt.Errorf("failed to dial %s: %w", endpointFlag, err)
+			}
+			defer conn.Close()
+
+			c := searchsvc.NewSearchProviderClient(conn)
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+			defer cancel()
+
+			_, err = c.IndexSpace(ctx, &searchsvc.IndexSpaceRequest{
+				SpaceId:      spaceFlag,
+				ForceReindex: forceRescanFlag,
+			})
+			if err != nil {
+				fmt.Println("failed to index space: " + err.Error())
+				return err
+			}
+			return nil
+		},
+	}
+	indexCmd.Flags().StringP(
+		"space",
+		"s",
+		"",
+		"the id of the space to travers and index the files of. This or --all-spaces is required.")
+
+	indexCmd.Flags().Bool(
+		"all-spaces",
+		false,
+		"index all spaces instead. This or --space is required.",
+	)
+	indexCmd.Flags().Bool(
+		"force-rescan",
+		false,
+		"force a rescan of all files, even if they are already indexed. This will make the indexing process much slower, but ensures that the index is up-to-date using the current search service configuration.",
+	)
+	indexCmd.Flags().String(
+		"endpoint",
+		"127.0.0.1:9220",
+		"the address of the search service gRPC endpoint.",
+	)
+	indexCmd.Flags().Bool(
+		"insecure",
+		false,
+		"disable TLS for the gRPC connection.",
+	)
+
+	return indexCmd
+}

Server/services/search/pkg/command/root.go

@@ -0,0 +1,37 @@
+package command
+
+import (
+	"os"
+
+	"github.com/qsfera/server/pkg/clihelper"
+	"github.com/qsfera/server/services/search/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+// GetCommands provides all commands for this service
+func GetCommands(cfg *config.Config) []*cobra.Command {
+	return []*cobra.Command{
+		// start this service
+		Server(cfg),
+
+		// interaction with this service
+		Index(cfg),
+
+		// infos about this service
+		Health(cfg),
+		Version(cfg),
+	}
+}
+
+// Execute is the entry point for the qsfera-search command.
+func Execute(cfg *config.Config) error {
+	app := clihelper.DefaultApp(&cobra.Command{
+		Use:   "search",
+		Short: "Serve search API for КуСфера",
+	})
+	app.AddCommand(GetCommands(cfg)...)
+	app.SetArgs(os.Args[1:])
+
+	return app.ExecuteContext(cfg.Context)
+}

Server/services/search/pkg/command/server.go

@@ -0,0 +1,240 @@
+package command
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+
+	"github.com/qsfera/server/pkg/config/configlog"
+	"github.com/qsfera/server/pkg/generators"
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/pkg/registry"
+	"github.com/qsfera/server/pkg/runner"
+	ogrpc "github.com/qsfera/server/pkg/service/grpc"
+	"github.com/qsfera/server/pkg/tracing"
+	"github.com/qsfera/server/pkg/version"
+	"github.com/qsfera/server/services/search/pkg/bleve"
+	"github.com/qsfera/server/services/search/pkg/config"
+	"github.com/qsfera/server/services/search/pkg/config/parser"
+	"github.com/qsfera/server/services/search/pkg/content"
+	"github.com/qsfera/server/services/search/pkg/metrics"
+	"github.com/qsfera/server/services/search/pkg/opensearch"
+	bleveQuery "github.com/qsfera/server/services/search/pkg/query/bleve"
+	"github.com/qsfera/server/services/search/pkg/search"
+	"github.com/qsfera/server/services/search/pkg/server/debug"
+	"github.com/qsfera/server/services/search/pkg/server/grpc"
+	svcEvent "github.com/qsfera/server/services/search/pkg/service/event"
+
+	"github.com/opencloud-eu/reva/v2/pkg/events/raw"
+	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
+	opensearchgo "github.com/opensearch-project/opensearch-go/v4"
+	opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/spf13/cobra"
+)
+
+// Server is the entrypoint for the server command.
+func Server(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "server",
+		Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
+			traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
+			if err != nil {
+				return err
+			}
+
+			cfg.GrpcClient, err = ogrpc.NewClient(
+				append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(traceProvider))...,
+			)
+			if err != nil {
+				return err
+			}
+
+			var cancel context.CancelFunc
+			if cfg.Context == nil {
+				cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
+				defer cancel()
+			}
+			ctx := cfg.Context
+
+			mtrcs := metrics.New()
+			mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
+
+			// initialize search engine
+			var eng search.Engine
+			switch cfg.Engine.Type {
+			case "bleve":
+				idx, err := bleve.NewIndex(cfg.Engine.Bleve.Datapath)
+				if err != nil {
+					return err
+				}
+
+				defer func() {
+					if err = idx.Close(); err != nil {
+						logger.Error().Err(err).Msg("could not close bleve index")
+					}
+				}()
+
+				eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, logger)
+			case "open-search":
+				clientConfig := opensearchgo.Config{
+					Addresses:             cfg.Engine.OpenSearch.Client.Addresses,
+					Username:              cfg.Engine.OpenSearch.Client.Username,
+					Password:              cfg.Engine.OpenSearch.Client.Password,
+					Header:                cfg.Engine.OpenSearch.Client.Header,
+					RetryOnStatus:         cfg.Engine.OpenSearch.Client.RetryOnStatus,
+					DisableRetry:          cfg.Engine.OpenSearch.Client.DisableRetry,
+					EnableRetryOnTimeout:  cfg.Engine.OpenSearch.Client.EnableRetryOnTimeout,
+					MaxRetries:            cfg.Engine.OpenSearch.Client.MaxRetries,
+					CompressRequestBody:   cfg.Engine.OpenSearch.Client.CompressRequestBody,
+					DiscoverNodesOnStart:  cfg.Engine.OpenSearch.Client.DiscoverNodesOnStart,
+					DiscoverNodesInterval: cfg.Engine.OpenSearch.Client.DiscoverNodesInterval,
+					EnableMetrics:         cfg.Engine.OpenSearch.Client.EnableMetrics,
+					EnableDebugLogger:     cfg.Engine.OpenSearch.Client.EnableDebugLogger,
+					Transport: &http.Transport{
+						TLSClientConfig: &tls.Config{
+							MinVersion:         tls.VersionTLS12,
+							InsecureSkipVerify: cfg.Engine.OpenSearch.Client.Insecure,
+						},
+					},
+				}
+
+				if cfg.Engine.OpenSearch.Client.CACert != "" {
+					certBytes, err := os.ReadFile(cfg.Engine.OpenSearch.Client.CACert)
+					if err != nil {
+						return fmt.Errorf("failed to read CA cert: %w", err)
+					}
+					clientConfig.CACert = certBytes
+				}
+
+				client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{Client: clientConfig})
+				if err != nil {
+					return fmt.Errorf("failed to create OpenSearch client: %w", err)
+				}
+
+				openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client)
+				if err != nil {
+					return fmt.Errorf("failed to create OpenSearch backend: %w", err)
+				}
+
+				eng = openSearchBackend
+			default:
+				return fmt.Errorf("unknown search engine: %s", cfg.Engine.Type)
+			}
+
+			// initialize gateway selector
+			selector, err := pool.GatewaySelector(cfg.Reva.Address, pool.WithRegistry(registry.GetRegistry()), pool.WithTracerProvider(traceProvider))
+			if err != nil {
+				logger.Fatal().Err(err).Msg("could not get reva gateway selector")
+				return err
+			}
+
+			// initialize search content extractor
+			var extractor content.Extractor
+			switch cfg.Extractor.Type {
+			case "basic":
+				if extractor, err = content.NewBasicExtractor(logger); err != nil {
+					return err
+				}
+			case "tika":
+				if extractor, err = content.NewTikaExtractor(selector, logger, cfg); err != nil {
+					return err
+				}
+			default:
+				return fmt.Errorf("unknown search extractor: %s", cfg.Extractor.Type)
+			}
+
+			ss := search.NewService(selector, eng, extractor, mtrcs, logger, cfg)
+
+			// setup the servers
+			gr := runner.NewGroup()
+
+			if !cfg.GRPC.Disabled {
+				grpcServer, err := grpc.Server(
+					grpc.Config(cfg),
+					grpc.Logger(logger),
+					grpc.Name(cfg.Service.Name),
+					grpc.Context(ctx),
+					grpc.Metrics(mtrcs),
+					grpc.JWTSecret(cfg.TokenManager.JWTSecret),
+					grpc.TraceProvider(traceProvider),
+					grpc.GatewaySelector(selector),
+					grpc.Searcher(ss),
+				)
+				if err != nil {
+					logger.Error().Err(err).Str("transport", "grpc").Msg("Failed to initialize server")
+					return err
+				}
+
+				gr.Add(runner.NewGoMicroGrpcServerRunner(cfg.Service.Name+".grpc", grpcServer))
+			} else {
+				logger.Info().Msg("gRPC server disabled, not starting gRPC service")
+			}
+
+			if !cfg.Events.Disabled {
+				connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
+				bus, err := raw.FromConfig(context.Background(), connName, raw.Config{
+					Endpoint:             cfg.Events.Endpoint,
+					Cluster:              cfg.Events.Cluster,
+					EnableTLS:            cfg.Events.EnableTLS,
+					TLSInsecure:          cfg.Events.TLSInsecure,
+					TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
+					AuthUsername:         cfg.Events.AuthUsername,
+					AuthPassword:         cfg.Events.AuthPassword,
+					MaxAckPending:        cfg.Events.MaxAckPending,
+					AckWait:              cfg.Events.AckWait,
+				})
+				if err != nil {
+					logger.Error().Err(err).Msg("Failed to create event bus client")
+					return err
+				}
+
+				eventSvc, err := svcEvent.New(ctx, bus, logger, traceProvider, mtrcs, ss, cfg.Events.DebounceDuration, cfg.Events.NumConsumers, cfg.Events.AsyncUploads)
+				if err != nil {
+					logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
+					return err
+				}
+
+				gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
+					return eventSvc.Run()
+				}, func() {
+					eventSvc.Close()
+				}))
+			} else {
+				logger.Info().Msg("event listening disabled, not starting event service")
+			}
+
+			// always start a debug server
+			{
+				debugServer, err := debug.Server(
+					debug.Logger(logger),
+					debug.Context(ctx),
+					debug.Config(cfg),
+				)
+				if err != nil {
+					logger.Error().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
+					return err
+				}
+
+				gr.Add(runner.NewGolangHttpServerRunner(cfg.Service.Name+".debug", debugServer))
+			}
+
+			grResults := gr.Run(ctx)
+
+			// return the first non-nil error found in the results
+			for _, grResult := range grResults {
+				if grResult.RunnerError != nil {
+					return grResult.RunnerError
+				}
+			}
+			return nil
+		},
+	}
+}

Server/services/search/pkg/command/version.go

@@ -0,0 +1,49 @@
+package command
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/qsfera/server/pkg/registry"
+	"github.com/qsfera/server/pkg/version"
+	"github.com/qsfera/server/services/search/pkg/config"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/olekukonko/tablewriter/tw"
+	"github.com/spf13/cobra"
+)
+
+// Version prints the service versions of all running instances.
+func Version(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "version",
+		Short: "print the version of this binary and the running service instances",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fmt.Println("Version: " + version.GetString())
+			fmt.Printf("Compiled: %s\n", version.Compiled())
+			fmt.Println("")
+
+			reg := registry.GetRegistry()
+			services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name)
+			if err != nil {
+				fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err))
+				return err
+			}
+
+			if len(services) == 0 {
+				fmt.Println("No running " + cfg.Service.Name + " service found.")
+				return nil
+			}
+
+			table := tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
+			table.Header([]string{"Version", "Address", "Id"})
+			for _, s := range services {
+				for _, n := range s.Nodes {
+					table.Append([]string{s.Version, n.Address, n.Id})
+				}
+			}
+			table.Render()
+			return nil
+		},
+	}
+}