Initial QSfera import
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
SHELL := bash
|
||||
NAME := eventhistory
|
||||
|
||||
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
|
||||
include ../../.bingo/Variables.mk
|
||||
endif
|
||||
|
||||
include ../../.make/default.mk
|
||||
include ../../.make/go.mk
|
||||
include ../../.make/release.mk
|
||||
include ../../.make/docs.mk
|
||||
@@ -0,0 +1,34 @@
|
||||
# Eventhistory
|
||||
|
||||
The `eventhistory` consumes all events from the configured event system like NATS, stores them and allows other services to retrieve them via an event ID.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running the eventhistory service without an event system like NATS is not possible.
|
||||
|
||||
## Consuming
|
||||
|
||||
The `eventhistory` services consumes all events from the configured event system.
|
||||
|
||||
## Storing
|
||||
|
||||
The `eventhistory` service stores each consumed event via the configured store in `EVENTHISTORY_STORE`. Possible stores are:
|
||||
- `memory`: Basic in-memory store and the default.
|
||||
- `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
|
||||
- `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
|
||||
- `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
|
||||
|
||||
Other store types may work but are not supported currently.
|
||||
|
||||
Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
|
||||
|
||||
Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
|
||||
|
||||
Store specific notes:
|
||||
- When using `redis-sentinel`, the Redis master to use is configured via e.g. `OC_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
|
||||
- When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
## Retrieving
|
||||
|
||||
Other services can call the `eventhistory` service via a gRPC call to retrieve events. The request must contain the event ID that should be retrieved.
|
||||
@@ -0,0 +1,19 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Health is the entrypoint for the health command.
|
||||
func Health(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "health",
|
||||
Short: "Check health status",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/qsfera/server/pkg/clihelper"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// GetCommands provides all commands for this service
|
||||
func GetCommands(cfg *config.Config) []*cobra.Command {
|
||||
return []*cobra.Command{
|
||||
// start this service
|
||||
Server(cfg),
|
||||
|
||||
// interaction with this service
|
||||
|
||||
// infos about this service
|
||||
Health(cfg),
|
||||
Version(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// Execute is the entry point for the eventhistory command.
|
||||
func Execute(cfg *config.Config) error {
|
||||
app := clihelper.DefaultApp(&cobra.Command{
|
||||
Use: "eventhistory",
|
||||
Short: "starts eventhistory service",
|
||||
})
|
||||
app.AddCommand(GetCommands(cfg)...)
|
||||
app.SetArgs(os.Args[1:])
|
||||
|
||||
return app.ExecuteContext(cfg.Context)
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/signal"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/generators"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/pkg/runner"
|
||||
ogrpc "github.com/qsfera/server/pkg/service/grpc"
|
||||
"github.com/qsfera/server/pkg/tracing"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config/parser"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/metrics"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/server/debug"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/server/grpc"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "server",
|
||||
Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnFatal(parser.ParseConfig(cfg))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
|
||||
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.GrpcClient, err = ogrpc.NewClient(
|
||||
append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(traceProvider))...,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var cancel context.CancelFunc
|
||||
if cfg.Context == nil {
|
||||
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
|
||||
defer cancel()
|
||||
}
|
||||
ctx := cfg.Context
|
||||
|
||||
m := metrics.New()
|
||||
m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
gr := runner.NewGroup()
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
consumer, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
st := store.Create(
|
||||
store.Store(cfg.Store.Store),
|
||||
store.TTL(cfg.Store.TTL),
|
||||
microstore.Nodes(cfg.Store.Nodes...),
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
)
|
||||
|
||||
service := grpc.NewService(
|
||||
grpc.Logger(logger),
|
||||
grpc.Context(ctx),
|
||||
grpc.Config(cfg),
|
||||
grpc.Name(cfg.Service.Name),
|
||||
grpc.Namespace(cfg.GRPC.Namespace),
|
||||
grpc.Address(cfg.GRPC.Addr),
|
||||
grpc.Metrics(m),
|
||||
grpc.Consumer(consumer),
|
||||
grpc.Persistence(st),
|
||||
grpc.TraceProvider(traceProvider),
|
||||
)
|
||||
|
||||
gr.Add(runner.NewGoMicroGrpcServerRunner(cfg.Service.Name+".grpc", service))
|
||||
|
||||
{
|
||||
debugServer, err := debug.Server(
|
||||
debug.Logger(logger),
|
||||
debug.Context(ctx),
|
||||
debug.Config(cfg),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.NewGolangHttpServerRunner(cfg.Service.Name+".debug", debugServer))
|
||||
}
|
||||
|
||||
grResults := gr.Run(ctx)
|
||||
|
||||
// return the first non-nil error found in the results
|
||||
for _, grResult := range grResults {
|
||||
if grResult.RunnerError != nil {
|
||||
return grResult.RunnerError
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Version prints the service versions of all running instances.
|
||||
func Version(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print the version of this binary and the running service instances",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"go-micro.dev/v4/client"
|
||||
)
|
||||
|
||||
// Config combines all available configuration parts.
|
||||
type Config struct {
|
||||
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;EVENTHISTORY_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
GRPC GRPCConfig `yaml:"grpc"`
|
||||
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
GrpcClient client.Client `yaml:"-"`
|
||||
|
||||
Events Events `yaml:"events"`
|
||||
Store Store `yaml:"store"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// GRPCConfig defines the available grpc configuration.
|
||||
type GRPCConfig struct {
|
||||
Addr string `yaml:"addr" env:"EVENTHISTORY_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
}
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;EVENTHISTORY_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;EVENTHISTORY_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;EVENTHISTORY_STORE_TTL" desc:"Time to live for events in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;EVENTHISTORY_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;EVENTHISTORY_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;EVENTHISTORY_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;EVENTHISTORY_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;EVENTHISTORY_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;EVENTHISTORY_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. Will be seen as empty if NOTIFICATIONS_EVENTS_TLS_INSECURE is provided." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;EVENTHISTORY_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_EVENTS_AUTH_USERNAME;EVENTHISTORY_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;EVENTHISTORY_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package config
|
||||
|
||||
// Debug defines the available debug configuration.
|
||||
type Debug struct {
|
||||
Addr string `yaml:"addr" env:"EVENTHISTORY_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
|
||||
Token string `yaml:"token" env:"EVENTHISTORY_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
|
||||
Pprof bool `yaml:"pprof" env:"EVENTHISTORY_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
|
||||
Zpages bool `yaml:"zpages" env:"EVENTHISTORY_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/pkg/structs"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
)
|
||||
|
||||
// FullDefaultConfig returns the full default config
|
||||
func FullDefaultConfig() *config.Config {
|
||||
cfg := DefaultConfig()
|
||||
EnsureDefaults(cfg)
|
||||
Sanitize(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultConfig return the default configuration
|
||||
func DefaultConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Debug: config.Debug{
|
||||
Addr: "127.0.0.1:9270",
|
||||
Token: "",
|
||||
Pprof: false,
|
||||
Zpages: false,
|
||||
},
|
||||
Service: config.Service{
|
||||
Name: "eventhistory",
|
||||
},
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
Cluster: "qsfera-cluster",
|
||||
EnableTLS: false,
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{"127.0.0.1:9233"},
|
||||
Database: "eventhistory",
|
||||
Table: "",
|
||||
TTL: 0,
|
||||
},
|
||||
GRPC: config.GRPCConfig{
|
||||
Addr: "127.0.0.1:9274",
|
||||
Namespace: "qsfera.api",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDefaults ensures the config contains default values
|
||||
func EnsureDefaults(cfg *config.Config) {
|
||||
if cfg.LogLevel == "" {
|
||||
cfg.LogLevel = "error"
|
||||
}
|
||||
|
||||
if cfg.GRPCClientTLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS)
|
||||
}
|
||||
|
||||
if cfg.GRPC.TLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPC.TLS = structs.CopyOrZeroValue(cfg.Commons.GRPCServiceTLS)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize sanitizes the config
|
||||
func Sanitize(cfg *config.Config) {
|
||||
// nothing to sanitize here atm
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config/defaults"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/envdecode"
|
||||
)
|
||||
|
||||
// ParseConfig loads configuration from known paths.
|
||||
func ParseConfig(cfg *config.Config) error {
|
||||
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
// load all env variables relevant to the config in the current context.
|
||||
if err := envdecode.Decode(cfg); err != nil {
|
||||
// no environment variable set for this config is an expected "error"
|
||||
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
defaults.Sanitize(cfg)
|
||||
|
||||
return Validate(cfg)
|
||||
}
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package config
|
||||
|
||||
// Service defines the available service configuration.
|
||||
type Service struct {
|
||||
Name string `yaml:"-"`
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package metrics
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
var (
|
||||
// Namespace defines the namespace for the defines metrics.
|
||||
Namespace = "qsfera"
|
||||
|
||||
// Subsystem defines the subsystem for the defines metrics.
|
||||
Subsystem = "eventhistory"
|
||||
)
|
||||
|
||||
// Metrics defines the available metrics of this service.
|
||||
type Metrics struct {
|
||||
BuildInfo *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// New initializes the available metrics.
|
||||
func New() *Metrics {
|
||||
m := &Metrics{
|
||||
BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "build_info",
|
||||
Help: "Build information",
|
||||
}, []string{"version"}),
|
||||
}
|
||||
|
||||
_ = prometheus.Register(
|
||||
m.BuildInfo,
|
||||
)
|
||||
|
||||
// TODO: implement metrics
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/checks"
|
||||
"github.com/qsfera/server/pkg/handlers"
|
||||
"github.com/qsfera/server/pkg/service/debug"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
)
|
||||
|
||||
// Server initializes the debug service and server.
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
healthHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr))
|
||||
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
debug.Name(options.Config.Service.Name),
|
||||
debug.Version(version.GetString()),
|
||||
debug.Address(options.Config.Debug.Addr),
|
||||
debug.Token(options.Config.Debug.Token),
|
||||
debug.Pprof(options.Config.Debug.Pprof),
|
||||
debug.Zpages(options.Config.Debug.Zpages),
|
||||
debug.Health(handlers.NewCheckHandler(healthHandlerConfiguration)),
|
||||
debug.Ready(handlers.NewCheckHandler(readyHandlerConfiguration)),
|
||||
), nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Address string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Namespace string
|
||||
Persistence store.Store
|
||||
Consumer events.Consumer
|
||||
TraceProvider trace.TracerProvider
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Name provides a name for the service.
|
||||
func Name(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Name = val
|
||||
}
|
||||
}
|
||||
|
||||
// Address provides an address for the service.
|
||||
func Address(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Address = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides a function to set the namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// Persistence provides a function to configure the store
|
||||
func Persistence(store store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.Persistence = store
|
||||
}
|
||||
}
|
||||
|
||||
// Consumer provides a function to configure the consumer
|
||||
func Consumer(consumer events.Consumer) Option {
|
||||
return func(o *Options) {
|
||||
o.Consumer = consumer
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to configure the trace provider
|
||||
func TraceProvider(traceProvider trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
if traceProvider != nil {
|
||||
o.TraceProvider = traceProvider
|
||||
} else {
|
||||
o.TraceProvider = trace.NewNoopTracerProvider()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/pkg/service/grpc"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
ehsvc "github.com/qsfera/server/protogen/gen/qsfera/services/eventhistory/v0"
|
||||
svc "github.com/qsfera/server/services/eventhistory/pkg/service"
|
||||
)
|
||||
|
||||
// NewService initializes the grpc service and server.
|
||||
func NewService(opts ...Option) grpc.Service {
|
||||
options := newOptions(opts...)
|
||||
|
||||
service, err := grpc.NewServiceWithClient(
|
||||
options.Config.GrpcClient,
|
||||
grpc.TLSEnabled(options.Config.GRPC.TLS.Enabled),
|
||||
grpc.TLSCert(
|
||||
options.Config.GRPC.TLS.Cert,
|
||||
options.Config.GRPC.TLS.Key,
|
||||
),
|
||||
grpc.Logger(options.Logger),
|
||||
grpc.Namespace(options.Namespace),
|
||||
grpc.Name(options.Name),
|
||||
grpc.Version(version.GetString()),
|
||||
grpc.Address(options.Address),
|
||||
grpc.Context(options.Context),
|
||||
grpc.Version(version.GetString()),
|
||||
grpc.TraceProvider(options.TraceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
eh, err := svc.NewEventHistoryService(options.Config, options.Consumer, options.Persistence, options.Logger)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("Error creating event history service")
|
||||
return grpc.Service{}
|
||||
}
|
||||
|
||||
_ = ehsvc.RegisterEventHistoryServiceHandler(
|
||||
service.Server(),
|
||||
eh,
|
||||
)
|
||||
|
||||
return service
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
ehmsg "github.com/qsfera/server/protogen/gen/qsfera/messages/eventhistory/v0"
|
||||
ehsvc "github.com/qsfera/server/protogen/gen/qsfera/services/eventhistory/v0"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// StoreEvent is data structure in the store
|
||||
type StoreEvent struct {
|
||||
ID string
|
||||
Type string
|
||||
Event []byte
|
||||
}
|
||||
|
||||
// EventHistoryService is the service responsible for event history
|
||||
type EventHistoryService struct {
|
||||
ch <-chan events.Event
|
||||
store store.Store
|
||||
cfg *config.Config
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
// NewEventHistoryService returns an EventHistory service
|
||||
func NewEventHistoryService(cfg *config.Config, consumer events.Consumer, store store.Store, log log.Logger) (*EventHistoryService, error) {
|
||||
if consumer == nil || store == nil {
|
||||
return nil, fmt.Errorf("need non nil consumer (%v) and store (%v) to work properly", consumer, store)
|
||||
}
|
||||
|
||||
ch, err := events.ConsumeAll(consumer, "evhistory")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
eh := &EventHistoryService{ch: ch, store: store, cfg: cfg, log: log}
|
||||
go eh.StoreEvents()
|
||||
|
||||
return eh, nil
|
||||
}
|
||||
|
||||
// StoreEvents consumes all events and stores them in the store. Will block
|
||||
func (eh *EventHistoryService) StoreEvents() {
|
||||
for event := range eh.ch {
|
||||
ev, err := json.Marshal(StoreEvent{
|
||||
ID: event.ID,
|
||||
Type: event.Type,
|
||||
Event: event.Event.([]byte),
|
||||
})
|
||||
if err != nil {
|
||||
eh.log.Error().Err(err).Str("eventid", event.ID).Msg("could not marshal event")
|
||||
continue
|
||||
}
|
||||
if err := eh.store.Write(&store.Record{
|
||||
Key: event.ID,
|
||||
Value: ev,
|
||||
Expiry: eh.cfg.Store.TTL,
|
||||
Metadata: map[string]any{
|
||||
"type": event.Type,
|
||||
},
|
||||
}); err != nil {
|
||||
// we can't store. That's it for us.
|
||||
eh.log.Error().Err(err).Str("eventid", event.ID).Msg("could not store event")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetEvents allows retrieving events from the eventstore by id
|
||||
func (eh *EventHistoryService) GetEvents(ctx context.Context, req *ehsvc.GetEventsRequest, resp *ehsvc.GetEventsResponse) error {
|
||||
for _, id := range req.Ids {
|
||||
ev, err := eh.getEvent(id)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp.Events = append(resp.Events, ev)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetEventsForUser allows retrieving events from the eventstore by userID
|
||||
// This function will match all events that contains the user ID between two non-word characters.
|
||||
// The reasoning behind this is that events put the userID in many different fields, which can differ
|
||||
// per event type. This function will match all events that contain the userID by using a regex.
|
||||
// This should also cover future events that might contain the userID in a different field.
|
||||
func (eh *EventHistoryService) GetEventsForUser(ctx context.Context, req *ehsvc.GetEventsForUserRequest, resp *ehsvc.GetEventsResponse) error {
|
||||
idx, err := eh.store.List(store.ListPrefix(""))
|
||||
if err != nil {
|
||||
eh.log.Error().Err(err).Msg("could not list events")
|
||||
return err
|
||||
}
|
||||
|
||||
// Match all events that contains the user ID between two non-word characters.
|
||||
userID, err := regexp.Compile(fmt.Sprintf(`\W%s\W`, req.UserID))
|
||||
if err != nil {
|
||||
eh.log.Error().Err(err).Str("userID", req.UserID).Msg("could not compile regex")
|
||||
return err
|
||||
}
|
||||
|
||||
for _, i := range idx {
|
||||
e, err := eh.getEvent(i)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if userID.Match(e.Event) {
|
||||
resp.Events = append(resp.Events, e)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (eh *EventHistoryService) getEvent(id string) (*ehmsg.Event, error) {
|
||||
evs, err := eh.store.Read(id)
|
||||
if err != nil {
|
||||
if !errors.Is(err, store.ErrNotFound) {
|
||||
eh.log.Error().Err(err).Str("eventid", id).Msg("could not read event")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(evs) == 0 {
|
||||
return nil, store.ErrNotFound
|
||||
}
|
||||
|
||||
var ev StoreEvent
|
||||
if err := json.Unmarshal(evs[0].Value, &ev); err != nil {
|
||||
eh.log.Error().Err(err).Str("eventid", id).Msg("could not unmarshal event")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ehmsg.Event{
|
||||
Id: ev.ID,
|
||||
Event: ev.Event,
|
||||
Type: ev.Type,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
ehsvc "github.com/qsfera/server/protogen/gen/qsfera/services/eventhistory/v0"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/config"
|
||||
"github.com/qsfera/server/services/eventhistory/pkg/service"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
microevents "go-micro.dev/v4/events"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
var _ = Describe("EventHistoryService", func() {
|
||||
var (
|
||||
cfg = &config.Config{}
|
||||
|
||||
eh *service.EventHistoryService
|
||||
bus testBus
|
||||
sto microstore.Store
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
sto = store.Create()
|
||||
bus = testBus(make(chan events.Event))
|
||||
eh, err = service.NewEventHistoryService(cfg, bus, sto, log.Logger{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
close(bus)
|
||||
})
|
||||
|
||||
It("Records events, stores them and allows them to be retrieved", func() {
|
||||
id := bus.Publish(events.UploadReady{})
|
||||
|
||||
// service will store eventually
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
resp := &ehsvc.GetEventsResponse{}
|
||||
err := eh.GetEvents(context.Background(), &ehsvc.GetEventsRequest{Ids: []string{id}}, resp)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp).ToNot(BeNil())
|
||||
|
||||
Expect(len(resp.Events)).To(Equal(1))
|
||||
Expect(resp.Events[0].Id).To(Equal(id))
|
||||
})
|
||||
|
||||
It("Gets all events", func() {
|
||||
ids := make([]string, 3)
|
||||
ids[0] = bus.Publish(events.UploadReady{
|
||||
ExecutingUser: &userv1beta1.User{
|
||||
Id: &userv1beta1.UserId{
|
||||
OpaqueId: "test-id",
|
||||
},
|
||||
},
|
||||
Failed: false,
|
||||
Timestamp: utils.TimeToTS(time.Time{}),
|
||||
})
|
||||
ids[1] = bus.Publish(events.UserCreated{
|
||||
UserID: "another-id",
|
||||
})
|
||||
ids[2] = bus.Publish(events.UserDeleted{
|
||||
Executant: &userv1beta1.UserId{
|
||||
OpaqueId: "another-id",
|
||||
},
|
||||
UserID: "test-id",
|
||||
})
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
resp := &ehsvc.GetEventsResponse{}
|
||||
err := eh.GetEventsForUser(context.Background(), &ehsvc.GetEventsForUserRequest{UserID: "test-id"}, resp)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp).ToNot(BeNil())
|
||||
|
||||
// Events don't always come back in the same order as they were sent, so we need to sort them and
|
||||
// do the same for the expected IDs as well.
|
||||
expectedIDs := []string{ids[0], ids[2]}
|
||||
sort.Strings(expectedIDs)
|
||||
var gotIDs []string
|
||||
for _, ev := range resp.Events {
|
||||
gotIDs = append(gotIDs, ev.Id)
|
||||
}
|
||||
sort.Strings(gotIDs)
|
||||
|
||||
Expect(len(gotIDs)).To(Equal(len(expectedIDs)))
|
||||
Expect(gotIDs[0]).To(Equal(expectedIDs[0]))
|
||||
Expect(gotIDs[1]).To(Equal(expectedIDs[1]))
|
||||
})
|
||||
})
|
||||
|
||||
type testBus chan events.Event
|
||||
|
||||
func (tb testBus) Consume(_ string, _ ...microevents.ConsumeOption) (<-chan microevents.Event, error) {
|
||||
ch := make(chan microevents.Event)
|
||||
go func() {
|
||||
for ev := range tb {
|
||||
b, _ := json.Marshal(ev.Event)
|
||||
ch <- microevents.Event{
|
||||
Payload: b,
|
||||
Metadata: map[string]string{
|
||||
events.MetadatakeyEventID: ev.ID,
|
||||
events.MetadatakeyEventType: ev.Type,
|
||||
},
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (tb testBus) Publish(e any) string {
|
||||
ev := events.Event{
|
||||
ID: uuid.New().String(),
|
||||
Type: reflect.TypeOf(e).String(),
|
||||
Event: e,
|
||||
}
|
||||
|
||||
tb <- ev
|
||||
return ev.ID
|
||||
}
|
||||
Reference in New Issue
Block a user