Initial QSfera import
This commit is contained in:
Курнат Андрей
@@ -0,0 +1,18 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Health is the entrypoint for the health command.
|
||||
func Health(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "health",
|
||||
Short: "Check health status",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/configlog"
|
||||
"github.com/qsfera/server/pkg/generators"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config/parser"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// RestartPostprocessing cli command to restart postprocessing
|
||||
func RestartPostprocessing(cfg *config.Config) *cobra.Command {
|
||||
restartPostprocessingCmd := &cobra.Command{
|
||||
Use: "resume",
|
||||
Aliases: []string{"restart"},
|
||||
Short: "resume postprocessing for an uploadID",
|
||||
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return configlog.ReturnFatal(parser.ParseConfig(cfg))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
stream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
|
||||
Endpoint: cfg.Postprocessing.Events.Endpoint,
|
||||
Cluster: cfg.Postprocessing.Events.Cluster,
|
||||
EnableTLS: cfg.Postprocessing.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Postprocessing.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Postprocessing.Events.AuthUsername,
|
||||
AuthPassword: cfg.Postprocessing.Events.AuthPassword,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid, _ := cmd.Flags().GetString("upload-id")
|
||||
step := ""
|
||||
if uid == "" {
|
||||
step, _ = cmd.Flags().GetString("step")
|
||||
}
|
||||
|
||||
restart, _ := cmd.Flags().GetBool("restart")
|
||||
var ev events.Unmarshaller
|
||||
switch {
|
||||
case restart:
|
||||
ev = events.RestartPostprocessing{
|
||||
UploadID: uid,
|
||||
Timestamp: utils.TSNow(),
|
||||
}
|
||||
default:
|
||||
ev = events.ResumePostprocessing{
|
||||
UploadID: uid,
|
||||
Step: events.Postprocessingstep(step),
|
||||
Timestamp: utils.TSNow(),
|
||||
}
|
||||
}
|
||||
|
||||
return events.Publish(context.Background(), stream, ev)
|
||||
},
|
||||
}
|
||||
|
||||
restartPostprocessingCmd.Flags().StringP(
|
||||
"upload-id",
|
||||
"u",
|
||||
"",
|
||||
"the uploadid to resume. Ignored if unset.",
|
||||
)
|
||||
restartPostprocessingCmd.Flags().StringP(
|
||||
"step",
|
||||
"s",
|
||||
"finished",
|
||||
"resume all uploads in the given postprocessing step. Ignored if upload-id is set.",
|
||||
)
|
||||
restartPostprocessingCmd.Flags().BoolP(
|
||||
"restart",
|
||||
"r",
|
||||
false,
|
||||
"restart postprocessing for the given uploadID. Ignores the step flag.",
|
||||
)
|
||||
|
||||
return restartPostprocessingCmd
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/qsfera/server/pkg/clihelper"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// GetCommands provides all commands for this service
|
||||
func GetCommands(cfg *config.Config) []*cobra.Command {
|
||||
return []*cobra.Command{
|
||||
// start this service
|
||||
Server(cfg),
|
||||
|
||||
// interaction with this service
|
||||
RestartPostprocessing(cfg),
|
||||
|
||||
// infos about this service
|
||||
Health(cfg),
|
||||
Version(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
// Execute is the entry point for the postprocessing command.
|
||||
func Execute(cfg *config.Config) error {
|
||||
app := clihelper.DefaultApp(&cobra.Command{
|
||||
Use: "postprocessing",
|
||||
Short: "starts postprocessing service",
|
||||
})
|
||||
app.AddCommand(GetCommands(cfg)...)
|
||||
app.SetArgs(os.Args[1:])
|
||||
|
||||
return app.ExecuteContext(cfg.Context)
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/pkg/runner"
|
||||
"github.com/qsfera/server/pkg/tracing"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config/parser"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/server/debug"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/service"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
)
|
||||
|
||||
// Server is the entrypoint for the server command.
|
||||
func Server(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "server",
|
||||
Short: fmt.Sprintf("start %s service without runtime (unsupervised mode)", cfg.Service.Name),
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
err := parser.ParseConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Printf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return err
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
|
||||
|
||||
var cancel context.CancelFunc
|
||||
if cfg.Context == nil {
|
||||
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
|
||||
defer cancel()
|
||||
}
|
||||
ctx := cfg.Context
|
||||
|
||||
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gr := runner.NewGroup()
|
||||
{
|
||||
st := store.Create(
|
||||
store.Store(cfg.Store.Store),
|
||||
store.TTL(cfg.Store.TTL),
|
||||
microstore.Nodes(cfg.Store.Nodes...),
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
)
|
||||
|
||||
svc, err := service.NewPostprocessingService(ctx, logger, st, traceProvider, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
|
||||
return svc.Run()
|
||||
}, func() {
|
||||
svc.Close()
|
||||
}))
|
||||
}
|
||||
|
||||
{
|
||||
debugServer, err := debug.Server(
|
||||
debug.Logger(logger),
|
||||
debug.Context(ctx),
|
||||
debug.Config(cfg),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.NewGolangHttpServerRunner("postprocessing_debug", debugServer))
|
||||
}
|
||||
|
||||
grResults := gr.Run(ctx)
|
||||
|
||||
// return the first non-nil error found in the results
|
||||
for _, grResult := range grResults {
|
||||
if grResult.RunnerError != nil {
|
||||
return grResult.RunnerError
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Version prints the service versions of all running instances.
|
||||
func Version(cfg *config.Config) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print the version of this binary and the running extension instances",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// not implemented
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
)
|
||||
|
||||
// Config combines all available configuration parts.
|
||||
type Config struct {
|
||||
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;POSTPROCESSING_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
Store Store `yaml:"store"`
|
||||
Postprocessing Postprocessing `yaml:"postprocessing"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// Postprocessing defines the config options for the postprocessing service.
|
||||
type Postprocessing struct {
|
||||
Events Events `yaml:"events"`
|
||||
Workers int `yaml:"workers" env:"POSTPROCESSING_WORKERS" desc:"The number of concurrent go routines that fetch events from the event queue." introductionVersion:"1.0.0"`
|
||||
|
||||
Steps []string `yaml:"steps" env:"POSTPROCESSING_STEPS" desc:"A list of postprocessing steps processed in order of their appearance. Currently supported values by the system are: 'virusscan', 'policies' and 'delay'. Custom steps are allowed. See the documentation for instructions. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Delayprocessing time.Duration `yaml:"delayprocessing" env:"POSTPROCESSING_DELAY" desc:"After uploading a file but before making it available for download, a delay step can be added. Intended for developing purposes only. If a duration is set but the keyword 'delay' is not explicitely added to 'POSTPROCESSING_STEPS', the delay step will be processed as last step. In such a case, a log entry will be written on service startup to remind the admin about that situation. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
|
||||
RetryBackoffDuration time.Duration `yaml:"retry_backoff_duration" env:"POSTPROCESSING_RETRY_BACKOFF_DURATION" desc:"The base for the exponential backoff duration before retrying a failed postprocessing step. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
MaxRetries int `yaml:"max_retries" env:"POSTPROCESSING_MAX_RETRIES" desc:"The maximum number of retries for a failed postprocessing step." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POSTPROCESSING_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POSTPROCESSING_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POSTPROCESSING_EVENTS_TLS_INSECURE" desc:"Whether the КуСфера server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;POSTPROCESSING_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided POSTPROCESSING_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;POSTPROCESSING_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_EVENTS_AUTH_USERNAME;POSTPROCESSING_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;POSTPROCESSING_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
|
||||
MaxAckPending int `yaml:"max_ack_pending" env:"SEARCH_EVENTS_MAX_ACK_PENDING" desc:"The maximum number of unacknowledged messages. This is used to limit the number of messages that can be in flight at the same time." introductionVersion:"4.0.0"`
|
||||
AckWait time.Duration `yaml:"ack_wait" env:"SEARCH_EVENTS_ACK_WAIT" desc:"The time to wait for an ack before the message is redelivered. This is used to ensure that messages are not lost if the consumer crashes." introductionVersion:"4.0.0"`
|
||||
}
|
||||
|
||||
// Debug defines the available debug configuration.
|
||||
type Debug struct {
|
||||
Addr string `yaml:"addr" env:"POSTPROCESSING_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
|
||||
Token string `yaml:"token" env:"POSTPROCESSING_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
|
||||
Pprof bool `yaml:"pprof" env:"POSTPROCESSING_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
|
||||
Zpages bool `yaml:"zpages" env:"POSTPROCESSING_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;POSTPROCESSING_STORE" desc:"The type of the store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;POSTPROCESSING_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"POSTPROCESSING_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"POSTPROCESSING_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;POSTPROCESSING_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;POSTPROCESSING_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;POSTPROCESSING_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
)
|
||||
|
||||
// FullDefaultConfig returns a full sanitized config
|
||||
func FullDefaultConfig() *config.Config {
|
||||
cfg := DefaultConfig()
|
||||
EnsureDefaults(cfg)
|
||||
Sanitize(cfg)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultConfig is the default configuration
|
||||
func DefaultConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Debug: config.Debug{
|
||||
Addr: "127.0.0.1:9255",
|
||||
Token: "",
|
||||
Pprof: false,
|
||||
Zpages: false,
|
||||
},
|
||||
Service: config.Service{
|
||||
Name: "postprocessing",
|
||||
},
|
||||
Postprocessing: config.Postprocessing{
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
Cluster: "qsfera-cluster",
|
||||
MaxAckPending: 10_000,
|
||||
AckWait: 1 * time.Minute,
|
||||
},
|
||||
Workers: 3,
|
||||
RetryBackoffDuration: 5 * time.Second,
|
||||
MaxRetries: 14,
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{"127.0.0.1:9233"},
|
||||
Database: "postprocessing",
|
||||
Table: "",
|
||||
TTL: 7 * 24 * time.Hour,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDefaults ensures defaults on a config
|
||||
func EnsureDefaults(cfg *config.Config) {
|
||||
if cfg.LogLevel == "" {
|
||||
cfg.LogLevel = "error"
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize does nothing atm
|
||||
func Sanitize(cfg *config.Config) {
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
occfg "github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config/defaults"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/envdecode"
|
||||
)
|
||||
|
||||
// ParseConfig loads configuration from known paths.
|
||||
func ParseConfig(cfg *config.Config) error {
|
||||
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
// load all env variables relevant to the config in the current context.
|
||||
if err := envdecode.Decode(cfg); err != nil {
|
||||
// no environment variable set for this config is an expected "error"
|
||||
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
defaults.Sanitize(cfg)
|
||||
|
||||
return Validate(cfg)
|
||||
}
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Postprocessing.Delayprocessing != 0 {
|
||||
if !contains(cfg.Postprocessing.Steps, events.PPStepDelay) {
|
||||
if len(cfg.Postprocessing.Steps) > 0 {
|
||||
s := strings.Join(append(cfg.Postprocessing.Steps, string(events.PPStepDelay)), ",")
|
||||
fmt.Printf("Added delay step to the list of postprocessing steps. NOTE: Use envvar `POSTPROCESSING_STEPS=%s` to suppress this message and choose the order of postprocessing steps.\n", s)
|
||||
}
|
||||
|
||||
cfg.Postprocessing.Steps = append(cfg.Postprocessing.Steps, string(events.PPStepDelay))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(all []string, candidate events.Postprocessingstep) bool {
|
||||
for _, s := range all {
|
||||
if s == string(candidate) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package config
|
||||
|
||||
// Service defines the available service configuration.
|
||||
type Service struct {
|
||||
Name string `yaml:"-"`
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
// Namespace defines the namespace for the defines metrics.
|
||||
Namespace = "qsfera"
|
||||
|
||||
// Subsystem defines the subsystem for the defines metrics.
|
||||
Subsystem = "postprocessing"
|
||||
|
||||
buildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "build_info",
|
||||
Help: "Build information",
|
||||
}, []string{"version"})
|
||||
eventsOutstandingAcks = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "events_outstanding_acks",
|
||||
Help: "Number of outstanding acks for events",
|
||||
})
|
||||
eventsUnprocessed = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "events_unprocessed",
|
||||
Help: "Number of unprocessed events",
|
||||
})
|
||||
eventsRedelivered = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "events_redelivered",
|
||||
Help: "Number of redelivered events",
|
||||
})
|
||||
finished = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "finished",
|
||||
Help: "Number of finished postprocessing events",
|
||||
}, []string{"status"})
|
||||
duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "duration_seconds",
|
||||
Help: "Duration of postprocessing operations in seconds",
|
||||
Buckets: []float64{0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1200},
|
||||
}, []string{"status"})
|
||||
)
|
||||
|
||||
// Metrics defines the available metrics of this service.
|
||||
type Metrics struct {
|
||||
// Counter *prometheus.CounterVec
|
||||
BuildInfo *prometheus.GaugeVec
|
||||
EventsOutstandingAcks prometheus.Gauge
|
||||
EventsUnprocessed prometheus.Gauge
|
||||
EventsRedelivered prometheus.Gauge
|
||||
Finished *prometheus.CounterVec
|
||||
Duration *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
// New initializes the available metrics.
|
||||
func New() *Metrics {
|
||||
m := &Metrics{
|
||||
BuildInfo: buildInfo,
|
||||
EventsOutstandingAcks: eventsOutstandingAcks,
|
||||
EventsUnprocessed: eventsUnprocessed,
|
||||
EventsRedelivered: eventsRedelivered,
|
||||
Finished: finished,
|
||||
Duration: duration,
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package postprocessing
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
)
|
||||
|
||||
// Postprocessing handles postprocessing of a file
|
||||
type Postprocessing struct {
|
||||
ID string
|
||||
URL string
|
||||
User *user.User
|
||||
ImpersonatingUser *user.User
|
||||
Filename string
|
||||
Filesize uint64
|
||||
ResourceID *provider.ResourceId
|
||||
Steps []events.Postprocessingstep
|
||||
Status Status
|
||||
Failures int
|
||||
InitiatorID string
|
||||
Finished bool
|
||||
StartTime time.Time
|
||||
|
||||
config config.Postprocessing
|
||||
}
|
||||
|
||||
// Status is helper struct to show current postprocessing status
|
||||
type Status struct {
|
||||
CurrentStep events.Postprocessingstep
|
||||
Outcome events.PostprocessingOutcome
|
||||
}
|
||||
|
||||
// New returns a new postprocessing instance
|
||||
func New(config config.Postprocessing) *Postprocessing {
|
||||
return &Postprocessing{
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
// Init is the first step of the postprocessing
|
||||
func (pp *Postprocessing) Init(_ events.BytesReceived) any {
|
||||
if len(pp.Steps) == 0 {
|
||||
return pp.finished(events.PPOutcomeContinue)
|
||||
}
|
||||
|
||||
return pp.step(pp.Steps[0])
|
||||
}
|
||||
|
||||
// NextStep returns the next postprocessing step
|
||||
func (pp *Postprocessing) NextStep(ev events.PostprocessingStepFinished) any {
|
||||
switch ev.Outcome {
|
||||
case events.PPOutcomeContinue:
|
||||
return pp.next(ev.FinishedStep)
|
||||
case events.PPOutcomeRetry:
|
||||
pp.Failures++
|
||||
if pp.Failures > pp.config.MaxRetries {
|
||||
return pp.finished(events.PPOutcomeAbort)
|
||||
}
|
||||
return pp.retry()
|
||||
default:
|
||||
return pp.finished(ev.Outcome)
|
||||
}
|
||||
}
|
||||
|
||||
// CurrentStep returns the current postprocessing step
|
||||
func (pp *Postprocessing) CurrentStep() any {
|
||||
if pp.Status.CurrentStep == events.PPStepFinished {
|
||||
return pp.finished(pp.Status.Outcome)
|
||||
}
|
||||
return pp.step(pp.Status.CurrentStep)
|
||||
}
|
||||
|
||||
// Delay will sleep the configured time then continue
|
||||
func (pp *Postprocessing) Delay(f func(next any)) {
|
||||
next := pp.next(events.PPStepDelay)
|
||||
go func() {
|
||||
time.Sleep(pp.config.Delayprocessing)
|
||||
f(next)
|
||||
}()
|
||||
}
|
||||
|
||||
// BackoffDuration calculates the duration for exponential backoff based on the number of failures.
|
||||
func (pp *Postprocessing) BackoffDuration() time.Duration {
|
||||
return pp.config.RetryBackoffDuration * time.Duration(math.Pow(2, float64(pp.Failures-1)))
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) next(current events.Postprocessingstep) any {
|
||||
l := len(pp.Steps)
|
||||
for i, s := range pp.Steps {
|
||||
if s == current && i+1 < l {
|
||||
return pp.step(pp.Steps[i+1])
|
||||
}
|
||||
}
|
||||
return pp.finished(events.PPOutcomeContinue)
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) step(next events.Postprocessingstep) events.StartPostprocessingStep {
|
||||
pp.Status.CurrentStep = next
|
||||
return events.StartPostprocessingStep{
|
||||
UploadID: pp.ID,
|
||||
URL: pp.URL,
|
||||
ExecutingUser: pp.User,
|
||||
Filename: pp.Filename,
|
||||
Filesize: pp.Filesize,
|
||||
ResourceID: pp.ResourceID,
|
||||
StepToStart: next,
|
||||
ImpersonatingUser: pp.ImpersonatingUser,
|
||||
}
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events.PostprocessingFinished {
|
||||
pp.Status.CurrentStep = events.PPStepFinished
|
||||
pp.Status.Outcome = outcome
|
||||
return events.PostprocessingFinished{
|
||||
UploadID: pp.ID,
|
||||
ExecutingUser: pp.User,
|
||||
Filename: pp.Filename,
|
||||
Outcome: outcome,
|
||||
ImpersonatingUser: pp.ImpersonatingUser,
|
||||
}
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) retry() events.PostprocessingRetry {
|
||||
pp.Status.Outcome = events.PPOutcomeRetry
|
||||
return events.PostprocessingRetry{
|
||||
UploadID: pp.ID,
|
||||
ExecutingUser: pp.User,
|
||||
Filename: pp.Filename,
|
||||
Failures: pp.Failures,
|
||||
BackoffDuration: pp.BackoffDuration(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// Context provides a function to set the context option.
|
||||
func Context(val context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = val
|
||||
}
|
||||
}
|
||||
|
||||
// Config provides a function to set the config option.
|
||||
func Config(val *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = val
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/qsfera/server/pkg/checks"
|
||||
"github.com/qsfera/server/pkg/handlers"
|
||||
"github.com/qsfera/server/pkg/service/debug"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
)
|
||||
|
||||
// Server initializes the debug service and server.
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Postprocessing.Events.Endpoint))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
debug.Name(options.Config.Service.Name),
|
||||
debug.Version(version.GetString()),
|
||||
debug.Address(options.Config.Debug.Addr),
|
||||
debug.Token(options.Config.Debug.Token),
|
||||
debug.Pprof(options.Config.Debug.Pprof),
|
||||
debug.Zpages(options.Config.Debug.Zpages),
|
||||
debug.Ready(handlers.NewCheckHandler(readyHandlerConfiguration)),
|
||||
), nil
|
||||
}
|
||||
@@ -0,0 +1,452 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/generators"
|
||||
"github.com/qsfera/server/pkg/log"
|
||||
"github.com/qsfera/server/pkg/version"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/config"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/metrics"
|
||||
"github.com/qsfera/server/services/postprocessing/pkg/postprocessing"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// PostprocessingService is an instance of the service handling postprocessing of files
|
||||
type PostprocessingService struct {
|
||||
ctx context.Context
|
||||
log log.Logger
|
||||
events <-chan raw.Event
|
||||
pub events.Publisher
|
||||
steps []events.Postprocessingstep
|
||||
store store.Store
|
||||
c config.Postprocessing
|
||||
tp trace.TracerProvider
|
||||
metrics *metrics.Metrics
|
||||
stopCh chan struct{}
|
||||
stopped atomic.Bool
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrFatal is returned when a fatal error occurs and we want to exit.
|
||||
ErrFatal = errors.New("fatal error")
|
||||
// ErrEvent is returned when something went wrong with a specific event.
|
||||
ErrEvent = errors.New("event error")
|
||||
// ErrNotFound is returned when a postprocessing is not found in the store.
|
||||
ErrNotFound = errors.New("postprocessing not found")
|
||||
)
|
||||
|
||||
// NewPostprocessingService returns a new instance of a postprocessing service
|
||||
func NewPostprocessingService(ctx context.Context, logger log.Logger, sto store.Store, tp trace.TracerProvider, cfg *config.Config) (*PostprocessingService, error) {
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
pub, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
|
||||
Endpoint: cfg.Postprocessing.Events.Endpoint,
|
||||
Cluster: cfg.Postprocessing.Events.Cluster,
|
||||
EnableTLS: cfg.Postprocessing.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Postprocessing.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Postprocessing.Events.AuthUsername,
|
||||
AuthPassword: cfg.Postprocessing.Events.AuthPassword,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
raw, err := raw.FromConfig(ctx, connName, raw.Config{
|
||||
Endpoint: cfg.Postprocessing.Events.Endpoint,
|
||||
Cluster: cfg.Postprocessing.Events.Cluster,
|
||||
EnableTLS: cfg.Postprocessing.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Postprocessing.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Postprocessing.Events.AuthUsername,
|
||||
AuthPassword: cfg.Postprocessing.Events.AuthPassword,
|
||||
MaxAckPending: cfg.Postprocessing.Events.MaxAckPending,
|
||||
AckWait: cfg.Postprocessing.Events.AckWait,
|
||||
})
|
||||
|
||||
evs, err := raw.Consume("postprocessing-pull",
|
||||
events.BytesReceived{},
|
||||
events.StartPostprocessingStep{},
|
||||
events.UploadReady{},
|
||||
events.PostprocessingStepFinished{},
|
||||
events.ResumePostprocessing{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m := metrics.New()
|
||||
m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
monitorMetrics(ctx, raw, "postprocessing-pull", m, logger)
|
||||
|
||||
return &PostprocessingService{
|
||||
ctx: ctx,
|
||||
log: logger,
|
||||
events: evs,
|
||||
pub: pub,
|
||||
steps: getSteps(cfg.Postprocessing),
|
||||
store: sto,
|
||||
c: cfg.Postprocessing,
|
||||
tp: tp,
|
||||
metrics: m,
|
||||
stopCh: make(chan struct{}, 1),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run to fulfil Runner interface
|
||||
func (pps *PostprocessingService) Run() error {
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
for range pps.c.Workers {
|
||||
wg.Go(func() {
|
||||
|
||||
EventLoop:
|
||||
for {
|
||||
select {
|
||||
case <-pps.stopCh:
|
||||
// stop requested
|
||||
// TODO: we might need a way to unsubscribe from the event channel, otherwise
|
||||
// we'll be leaking a goroutine in reva that will be stuck waiting for
|
||||
// someone to read from the event channel.
|
||||
// Note: redis implementation seems to have a timeout, so the goroutine
|
||||
// will exit if there is nobody processing the events and the timeout
|
||||
// is reached. The behavior is unclear with natsjs
|
||||
break EventLoop
|
||||
case e, ok := <-pps.events:
|
||||
if !ok {
|
||||
// event channel is closed, so nothing more to do
|
||||
break EventLoop
|
||||
}
|
||||
|
||||
err := pps.processEvent(e)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, ErrFatal):
|
||||
pps.log.Fatal().Err(err).Msg("fatal error - exiting")
|
||||
case errors.Is(err, ErrEvent):
|
||||
pps.log.Error().Err(err).Msg("continuing")
|
||||
default:
|
||||
pps.log.Fatal().Err(err).Msg("unknown error - exiting")
|
||||
}
|
||||
}
|
||||
|
||||
if pps.stopped.Load() {
|
||||
// if stopped, don't process any more events
|
||||
break EventLoop
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close will make the postprocessing service to stop processing, so the `Run`
|
||||
// method can finish.
|
||||
// TODO: Underlying services can't be stopped. This means that some goroutines
|
||||
// will get stuck trying to push events through a channel nobody is reading
|
||||
// from, so resources won't be freed and there will be memory leaks. For now,
|
||||
// if the service is stopped, you should close the app soon after.
|
||||
func (pps *PostprocessingService) Close() {
|
||||
if pps.stopped.CompareAndSwap(false, true) {
|
||||
close(pps.stopCh)
|
||||
}
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) processEvent(e raw.Event) error {
|
||||
pps.log.Debug().Str("Type", e.Type).Str("ID", e.ID).Msg("processing event received")
|
||||
|
||||
var (
|
||||
next any
|
||||
pp *postprocessing.Postprocessing
|
||||
err error
|
||||
)
|
||||
|
||||
ctx := e.GetTraceContext(pps.ctx)
|
||||
ctx, span := pps.tp.Tracer("postprocessing").Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
ackEvent := true
|
||||
defer func() {
|
||||
if ackEvent {
|
||||
if err := e.Ack(); err != nil {
|
||||
pps.log.Error().Err(err).Msg("unable to ack event")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
switch ev := e.Event.Event.(type) {
|
||||
case events.BytesReceived:
|
||||
pp = &postprocessing.Postprocessing{
|
||||
ID: ev.UploadID,
|
||||
URL: ev.URL,
|
||||
User: ev.ExecutingUser,
|
||||
Filename: ev.Filename,
|
||||
Filesize: ev.Filesize,
|
||||
ResourceID: ev.ResourceID,
|
||||
Steps: pps.steps,
|
||||
InitiatorID: e.InitiatorID,
|
||||
ImpersonatingUser: ev.ImpersonatingUser,
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
next = pp.Init(ev)
|
||||
case events.PostprocessingStepFinished:
|
||||
if ev.UploadID == "" {
|
||||
// no current upload - this was an on demand scan
|
||||
return nil
|
||||
}
|
||||
pp, err = pps.getPP(pps.store, ev.UploadID)
|
||||
if err != nil {
|
||||
pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
|
||||
return fmt.Errorf("%w: cannot get upload", ErrEvent)
|
||||
}
|
||||
next = pp.NextStep(ev)
|
||||
|
||||
switch pp.Status.Outcome {
|
||||
case events.PPOutcomeRetry:
|
||||
// schedule retry
|
||||
backoff := pp.BackoffDuration()
|
||||
go func() {
|
||||
time.Sleep(backoff)
|
||||
retryEvent := events.StartPostprocessingStep{
|
||||
UploadID: pp.ID,
|
||||
URL: pp.URL,
|
||||
ExecutingUser: pp.User,
|
||||
Filename: pp.Filename,
|
||||
Filesize: pp.Filesize,
|
||||
ResourceID: pp.ResourceID,
|
||||
StepToStart: pp.Status.CurrentStep,
|
||||
ImpersonatingUser: pp.ImpersonatingUser,
|
||||
}
|
||||
err := events.Publish(ctx, pps.pub, retryEvent)
|
||||
if err != nil {
|
||||
pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot publish RestartPostprocessing event")
|
||||
}
|
||||
}()
|
||||
}
|
||||
case events.StartPostprocessingStep:
|
||||
if ev.StepToStart != events.PPStepDelay {
|
||||
return nil
|
||||
}
|
||||
pp, err = pps.getPP(pps.store, ev.UploadID)
|
||||
if err != nil {
|
||||
pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
|
||||
return fmt.Errorf("%w: cannot get upload", ErrEvent)
|
||||
}
|
||||
pp.Delay(func(next any) {
|
||||
if err := events.Publish(ctx, pps.pub, next); err != nil {
|
||||
pps.log.Error().Err(err).Msg("cannot publish event")
|
||||
}
|
||||
})
|
||||
case events.UploadReady:
|
||||
// the upload failed - let's keep it around for a while - but mark it as finished
|
||||
pp, err = pps.getPP(pps.store, ev.UploadID)
|
||||
if err != nil {
|
||||
pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
|
||||
return fmt.Errorf("%w: cannot get upload", ErrEvent)
|
||||
}
|
||||
|
||||
if ev.Failed {
|
||||
pps.metrics.Finished.WithLabelValues("failed").Inc()
|
||||
if !pp.StartTime.IsZero() {
|
||||
pps.metrics.Duration.WithLabelValues("failed").Observe(time.Since(pp.StartTime).Seconds())
|
||||
}
|
||||
pp.Finished = true
|
||||
return storePP(pps.store, pp)
|
||||
}
|
||||
|
||||
pps.metrics.Finished.WithLabelValues("succeeded").Inc()
|
||||
if !pp.StartTime.IsZero() {
|
||||
pps.metrics.Duration.WithLabelValues("succeeded").Observe(time.Since(pp.StartTime).Seconds())
|
||||
}
|
||||
// the storage provider thinks the upload is done - so no need to keep it any more
|
||||
if err := pps.store.Delete(ev.UploadID); err != nil {
|
||||
pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot delete upload")
|
||||
return fmt.Errorf("%w: cannot delete upload", ErrEvent)
|
||||
}
|
||||
case events.ResumePostprocessing:
|
||||
return pps.handleResumePPEvent(ctx, ev)
|
||||
}
|
||||
|
||||
if pp != nil {
|
||||
ctx = ctxpkg.ContextSetInitiator(ctx, pp.InitiatorID)
|
||||
|
||||
if err := storePP(pps.store, pp); err != nil {
|
||||
ackEvent = false
|
||||
pps.log.Error().Str("uploadID", pp.ID).Err(err).Msg("cannot store upload")
|
||||
return fmt.Errorf("%w: cannot store upload", ErrEvent)
|
||||
}
|
||||
}
|
||||
|
||||
if next != nil {
|
||||
if err := events.Publish(ctx, pps.pub, next); err != nil {
|
||||
pps.log.Error().Err(err).Msg("unable to publish event")
|
||||
return fmt.Errorf("%w: unable to publish event", ErrFatal) // we can't publish -> we are screwed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) {
|
||||
recs, err := sto.Read(uploadID)
|
||||
if err != nil {
|
||||
if err == store.ErrNotFound {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(recs) == 0 {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
if len(recs) > 1 {
|
||||
return nil, fmt.Errorf("expected only one result for '%s', got %d", uploadID, len(recs))
|
||||
}
|
||||
|
||||
pp := postprocessing.New(pps.c)
|
||||
err = json.Unmarshal(recs[0].Value, pp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pp, nil
|
||||
}
|
||||
|
||||
func getSteps(c config.Postprocessing) []events.Postprocessingstep {
|
||||
// NOTE: improved version only allows configuring order of postprocessing steps
|
||||
// But we aim for a system where postprocessing steps can be configured per space, ideally by the spaceadmin itself
|
||||
// We need to iterate over configuring PP service when we see fit
|
||||
steps := make([]events.Postprocessingstep, 0, len(c.Steps))
|
||||
for _, s := range c.Steps {
|
||||
steps = append(steps, events.Postprocessingstep(s))
|
||||
}
|
||||
|
||||
return steps
|
||||
}
|
||||
|
||||
func storePP(sto store.Store, pp *postprocessing.Postprocessing) error {
|
||||
b, err := json.Marshal(pp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sto.Write(&store.Record{
|
||||
Key: pp.ID,
|
||||
Value: b,
|
||||
})
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) handleResumePPEvent(ctx context.Context, ev events.ResumePostprocessing) error {
|
||||
ids := []string{ev.UploadID}
|
||||
if ev.Step != "" {
|
||||
ids = pps.findUploadsByStep(ev.Step)
|
||||
}
|
||||
|
||||
for _, id := range ids {
|
||||
if err := pps.resumePP(ctx, id); err != nil {
|
||||
pps.log.Error().Str("uploadID", id).Err(err).Msg("cannot resume upload")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) resumePP(ctx context.Context, uploadID string) error {
|
||||
pp, err := pps.getPP(pps.store, uploadID)
|
||||
if err != nil {
|
||||
if err == ErrNotFound {
|
||||
if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{
|
||||
UploadID: uploadID,
|
||||
Timestamp: utils.TSNow(),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot get upload: %w", err)
|
||||
}
|
||||
|
||||
if pp.Finished {
|
||||
// dont retry finished uploads
|
||||
return nil
|
||||
}
|
||||
|
||||
return events.Publish(ctx, pps.pub, pp.CurrentStep())
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) findUploadsByStep(step events.Postprocessingstep) []string {
|
||||
var ids []string
|
||||
|
||||
keys, err := pps.store.List()
|
||||
if err != nil {
|
||||
pps.log.Error().Err(err).Msg("cannot list uploads")
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
rec, err := pps.store.Read(k)
|
||||
if err != nil {
|
||||
pps.log.Error().Err(err).Msg("cannot read upload")
|
||||
continue
|
||||
}
|
||||
|
||||
if len(rec) != 1 {
|
||||
pps.log.Error().Err(err).Msg("expected only one result")
|
||||
continue
|
||||
}
|
||||
|
||||
pp := &postprocessing.Postprocessing{}
|
||||
err = json.Unmarshal(rec[0].Value, pp)
|
||||
if err != nil {
|
||||
pps.log.Error().Err(err).Msg("cannot unmarshal upload")
|
||||
continue
|
||||
}
|
||||
|
||||
if pp.Status.CurrentStep == step {
|
||||
ids = append(ids, pp.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metrics.Metrics, logger log.Logger) {
|
||||
consumer, err := stream.JetStream().Consumer(ctx, name)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("failed to get consumer")
|
||||
}
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
go func() {
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("failed to get consumer")
|
||||
continue
|
||||
}
|
||||
|
||||
m.EventsOutstandingAcks.Set(float64(info.NumAckPending))
|
||||
m.EventsUnprocessed.Set(float64(info.NumPending))
|
||||
m.EventsRedelivered.Set(float64(info.NumRedelivered))
|
||||
logger.Trace().Msg("updated postprocessing event metrics")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
Reference in New Issue
Block a user