Initial QSfera import

Server/services/postprocessing/Makefile

@@ -0,0 +1,11 @@
+SHELL := bash
+NAME := postprocessing
+
+ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
+include ../../.bingo/Variables.mk
+endif
+
+include ../../.make/default.mk
+include ../../.make/go.mk
+include ../../.make/release.mk
+include ../../.make/docs.mk

Server/services/postprocessing/README.md

@@ -0,0 +1,135 @@
+# Postprocessing
+
+The `postprocessing` service handles the coordination of asynchronous postprocessing steps.
+
+## General Prerequisites
+
+To use the postprocessing service, an event system needs to be configured for all services. By default, `КуСфера` ships with a preconfigured `nats` service.
+
+## Postprocessing Functionality
+
+The storageprovider service (`storage-users`) can be configured to initiate asynchronous postprocessing by setting the `OC_ASYNC_UPLOADS` environment variable to `true`. If this is the case, postprocessing will get initiated *after* uploading a file and all bytes have been received.
+
+The `postprocessing` service will then coordinate configured postprocessing steps like scanning the file for viruses. During postprocessing, the file will be in a `processing state` where only a limited set of actions are available. Note that this processing state excludes file accessibility by users.
+
+When all postprocessing steps have completed successfully, the file will be made accessible for users.
+
+## Storing Postprocessing Data
+
+The `postprocessing` service needs to store some metadata about uploads to be able to orchestrate post-processing. When running in single binary mode, the default in-memory implementation will be just fine. In distributed deployments it is recommended to use a persistent store, see below for more details.
+
+The `postprocessing` service stores its metadata via the configured store in `POSTPROCESSING_STORE`. Possible stores are:
+  -   `memory`: Basic in-memory store and the default.
+  -   `redis-sentinel`: Stores data in a configured Redis Sentinel cluster.
+  -   `nats-js-kv`: Stores data using key-value-store feature of [nats jetstream](https://docs.nats.io/nats-concepts/jetstream/key-value-store)
+  -   `noop`: Stores nothing. Useful for testing. Not recommended in production environments.
+
+Other store types may work but are not supported currently.
+
+Note: The service can only be scaled if not using `memory` store and the stores are configured identically over all instances!
+
+Note that if you have used one of the deprecated stores, you should reconfigure to one of the supported ones as the deprecated stores will be removed in a later version.
+
+Store specific notes:
+  -   When using `redis-sentinel`, the Redis master to use is configured via e.g. `OC_CACHE_STORE_NODES` in the form of `<sentinel-host>:<sentinel-port>/<redis-master>` like `10.10.0.200:26379/mymaster`.
+  -   When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
+  -   When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
+
+## Additional Prerequisites for the Postprocessing Service
+
+When postprocessing has been enabled, configuring any postprocessing step will require the requested services to be enabled and pre-configured. For example, to use the `virusscan` step, one needs to have an enabled and configured `antivirus` service.
+
+## Postprocessing Steps
+
+The postporcessing service is individually configurable. This is achieved by allowing a list of postprocessing steps that are processed in order of their appearance in the `POSTPROCESSING_STEPS` envvar. This envvar expects a comma separated list of steps that will be executed. Currently known steps to the system are `virusscan` and `delay`. Custom steps can be added but need an existing target for processing.
+
+### Virus Scanning
+
+To enable virus scanning as a postprocessing step after uploading a file, the environment variable `POSTPROCESSING_STEPS` needs to contain the word `virusscan` at one location in the list of steps. As a result, each uploaded file gets virus scanned as part of the postprocessing steps. Note that the `antivirus` service is required to be enabled and configured for this to work.
+
+### Delay
+
+Though this is for development purposes only and NOT RECOMMENDED on production systems, setting the environment variable `POSTPROCESSING_DELAY` to a duration not equal to zero will add a delay step with the configured amount of time. КуСфера will continue postprocessing the file after the configured delay. Use the environment variable `POSTPROCESSING_STEPS` and the keyword `delay` if you have multiple postprocessing steps and want to define their order. If `POSTPROCESSING_DELAY` is set but the keyword `delay` is not contained in `POSTPROCESSING_STEPS`, it will be processed as last postprocessing step without being listed there. In this case, a log entry will be written on service startup to notify the admin about that situation. That log entry can be avoided by adding the keyword `delay` to `POSTPROCESSING_STEPS`.
+
+### Custom Postprocessing Steps
+By using the envvar `POSTPROCESSING_STEPS`, custom postprocessing steps can be added. Any word can be used as step name but be careful not to conflict with exising keywords like `virusscan` and `delay`. In addition, if a keyword is misspelled or the corresponding service does either not exist or does not follow the necessary event communication, the postprocessing service will wait forever getting the required response to proceed and does not continue any other processing.
+
+#### Prerequisites
+For using custom postprocessing steps you need a custom service listening to the configured event system (see `General Prerequisites`)
+
+#### Workflow
+When defining a custom postprocessing step (eg. `"customstep"`), the postprocessing service will eventually send an event during postprocessing. The event will be of type `StartPostprocessingStep` with its field `StepToStart` set to `"customstep"`. When the service defined as custom step receives this event, it can safely execute its actions. The postprocessing service will wait until it has finished its work. The event contains further information (filename, executing user, size, ...) and also requires tokens and URLs to download the file in case byte inspection is necessary.
+
+Once the service defined as custom step has finished its work, it should send an event of type `PostprocessingFinished` via the configured events system back to the postprocessing service. This event needs to contain a `FinishedStep` field set to `"customstep"`. It also must contain the outcome of the step, which can be one of the following:
+
+-   `delete`: Abort postprocessing, delete the file.
+-   `abort`: Abort postprocessing, keep the file.
+-   `retry`: There was a problem that was most likely temporary and may be solved by trying again after some backoff duration. Retry runs automatically and is defined by the backoff behavior as described below.
+-   `continue`: Continue postprocessing, this is the success case.
+
+The backoff behavior as mentioned in the `retry` outcome can be configured using the `POSTPROCESSING_RETRY_BACKOFF_DURATION` and `POSTPROCESSING_MAX_RETRIES` environment variables. The backoff duration is calculated using the following formula after each failure: `backoff_duration = POSTPROCESSING_RETRY_BACKOFF_DURATION * 2^(number of failures - 1)`. This means that the time between the next round grows exponentially limited by the number of retries. Steps that still don't succeed after the maximum number of retries will be automatically moved to the `abort` state.
+
+See the [cs3 org](https://github.com/cs3org/reva/blob/edge/pkg/events/postprocessing.go) for up-to-date information of reserved step names and event definitions.
+
+## CLI Commands
+
+### Resume Postprocessing
+
+**IMPORTANT**
+> If not noted otherwise, commands with the `restart` option can also use the `resume` option. This changes behaviour slightly.
+>
+> * `restart`\
+> When restarting an upload, all steps for open items will be restarted, except if otherwise defined.
+> * `resume`\
+> When resuming an upload, processing will continue unfinished items from their last completed step.
+
+If post-processing fails in one step due to an unforeseen error, current uploads will not be resumed automatically. A system administrator can instead run CLI commands to resume the failed upload manually which is at minimum a two step process.
+
+For details on the `storage-users` command see the **Manage Unfinished Uploads** documentation in the `storage-users` service documentation.
+
+Depending if you want to restart/resume all or defined failed uploads, different commands are used.
+
+-   First, list ongoing upload sessions to identify possibly failed ones.\
+    Note that there never can be a clear identification of a failed upload session due to various reasons causing them. You need to apply more critera like free space on disk, a failed service like antivirus etc. to declare an upload as failed.
+
+    ```bash
+    qsfera storage-users uploads sessions
+    ```
+
+-   **All failed uploads**\
+    If you want to restart/resume all failed uploads, just rerun the command with the relevant flag. Note that this is the preferred command to handle failed processing steps:
+    ```bash
+    qsfera storage-users uploads sessions --resume
+    ```
+
+-   **Particular failed uploads**\
+    Use the `postprocessing` command to resume defined failed uploads. For postprocessing steps, the default is to resume . Note that at the moment, `resume` is an alias for `restart` to keep old functionality. `restart` is subject of change and will most likely be removed in a later version.
+
+    - **Defined by ID**\
+      If you want to resume only a specific upload, use the postprocessing resume command with the ID selected:
+      ```bash
+      qsfera postprocessing resume -u <uploadID>
+      ```
+
+    - **Defined by step**\
+      Alternatively, instead of restarting one specific upload, a system admin can also resume all uploads that are currently in a specific step.\
+      Examples:\
+      ```bash
+      qsfera postprocessing resume                # Resumes all uploads where postprocessing is finished, but upload is not finished
+      qsfera postprocessing resume -s "finished"  # Equivalent to the above
+      qsfera postprocessing resume -s "virusscan" # Resume all uploads currently in virusscan step
+      ```
+
+## Metrics
+
+The postprocessing service exposes the following prometheus metrics at `<debug_endpoint>/metrics` (as configured using the `POSTPROCESSING_DEBUG_ADDR` env var):
+
+| Metric Name | Type | Description | Labels |
+| --- | --- | --- | --- |
+| `qsfera_postprocessing_build_info` | Gauge | Build information | `version` |
+| `qsfera_postprocessing_events_outstanding_acks` | Gauge | Number of outstanding acks for events | |
+| `qsfera_postprocessing_events_unprocessed` | Gauge | Number of unprocessed events | |
+| `qsfera_postprocessing_events_redelivered` | Gauge | Number of redelivered events | |
+| `qsfera_postprocessing_in_progress` | Gauge | Number of postprocessing events in progress | |
+| `qsfera_postprocessing_finished` | Counter | Number of finished postprocessing events | `status` |
+| `qsfera_postprocessing_duration_seconds` | Histogram | Duration of postprocessing operations in seconds | `status` |

Server/services/postprocessing/pkg/command/health.go

@@ -0,0 +1,18 @@
+package command
+
+import (
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+// Health is the entrypoint for the health command.
+func Health(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "health",
+		Short: "Check health status",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// Not implemented
+			return nil
+		},
+	}
+}

Server/services/postprocessing/pkg/command/postprocessing.go

@@ -0,0 +1,88 @@
+package command
+
+import (
+	"context"
+
+	"github.com/qsfera/server/pkg/config/configlog"
+	"github.com/qsfera/server/pkg/generators"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/qsfera/server/services/postprocessing/pkg/config/parser"
+	"github.com/opencloud-eu/reva/v2/pkg/events"
+	"github.com/opencloud-eu/reva/v2/pkg/events/stream"
+	"github.com/opencloud-eu/reva/v2/pkg/utils"
+
+	"github.com/spf13/cobra"
+)
+
+// RestartPostprocessing cli command to restart postprocessing
+func RestartPostprocessing(cfg *config.Config) *cobra.Command {
+	restartPostprocessingCmd := &cobra.Command{
+		Use:     "resume",
+		Aliases: []string{"restart"},
+		Short:   "resume postprocessing for an uploadID",
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
+			stream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
+				Endpoint:             cfg.Postprocessing.Events.Endpoint,
+				Cluster:              cfg.Postprocessing.Events.Cluster,
+				EnableTLS:            cfg.Postprocessing.Events.EnableTLS,
+				TLSInsecure:          cfg.Postprocessing.Events.TLSInsecure,
+				TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
+				AuthUsername:         cfg.Postprocessing.Events.AuthUsername,
+				AuthPassword:         cfg.Postprocessing.Events.AuthPassword,
+			})
+			if err != nil {
+				return err
+			}
+
+			uid, _ := cmd.Flags().GetString("upload-id")
+			step := ""
+			if uid == "" {
+				step, _ = cmd.Flags().GetString("step")
+			}
+
+			restart, _ := cmd.Flags().GetBool("restart")
+			var ev events.Unmarshaller
+			switch {
+			case restart:
+				ev = events.RestartPostprocessing{
+					UploadID:  uid,
+					Timestamp: utils.TSNow(),
+				}
+			default:
+				ev = events.ResumePostprocessing{
+					UploadID:  uid,
+					Step:      events.Postprocessingstep(step),
+					Timestamp: utils.TSNow(),
+				}
+			}
+
+			return events.Publish(context.Background(), stream, ev)
+		},
+	}
+
+	restartPostprocessingCmd.Flags().StringP(
+		"upload-id",
+		"u",
+		"",
+		"the uploadid to resume. Ignored if unset.",
+	)
+	restartPostprocessingCmd.Flags().StringP(
+		"step",
+		"s",
+		"finished",
+		"resume all uploads in the given postprocessing step. Ignored if upload-id is set.",
+	)
+	restartPostprocessingCmd.Flags().BoolP(
+		"restart",
+		"r",
+		false,
+		"restart postprocessing for the given uploadID. Ignores the step flag.",
+	)
+
+	return restartPostprocessingCmd
+}

Server/services/postprocessing/pkg/command/root.go

@@ -0,0 +1,37 @@
+package command
+
+import (
+	"os"
+
+	"github.com/qsfera/server/pkg/clihelper"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+
+	"github.com/spf13/cobra"
+)
+
+// GetCommands provides all commands for this service
+func GetCommands(cfg *config.Config) []*cobra.Command {
+	return []*cobra.Command{
+		// start this service
+		Server(cfg),
+
+		// interaction with this service
+		RestartPostprocessing(cfg),
+
+		// infos about this service
+		Health(cfg),
+		Version(cfg),
+	}
+}
+
+// Execute is the entry point for the postprocessing command.
+func Execute(cfg *config.Config) error {
+	app := clihelper.DefaultApp(&cobra.Command{
+		Use:   "postprocessing",
+		Short: "starts postprocessing service",
+	})
+	app.AddCommand(GetCommands(cfg)...)
+	app.SetArgs(os.Args[1:])
+
+	return app.ExecuteContext(cfg.Context)
+}

Server/services/postprocessing/pkg/command/server.go

@@ -0,0 +1,98 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/pkg/runner"
+	"github.com/qsfera/server/pkg/tracing"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/qsfera/server/services/postprocessing/pkg/config/parser"
+	"github.com/qsfera/server/services/postprocessing/pkg/server/debug"
+	"github.com/qsfera/server/services/postprocessing/pkg/service"
+	"github.com/opencloud-eu/reva/v2/pkg/store"
+
+	"github.com/spf13/cobra"
+	microstore "go-micro.dev/v4/store"
+)
+
+// Server is the entrypoint for the server command.
+func Server(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "server",
+		Short: fmt.Sprintf("start %s service without runtime (unsupervised mode)", cfg.Service.Name),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			err := parser.ParseConfig(cfg)
+			if err != nil {
+				fmt.Printf("%v", err)
+				os.Exit(1)
+			}
+			return err
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
+
+			var cancel context.CancelFunc
+			if cfg.Context == nil {
+				cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
+				defer cancel()
+			}
+			ctx := cfg.Context
+
+			traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
+			if err != nil {
+				return err
+			}
+
+			gr := runner.NewGroup()
+			{
+				st := store.Create(
+					store.Store(cfg.Store.Store),
+					store.TTL(cfg.Store.TTL),
+					microstore.Nodes(cfg.Store.Nodes...),
+					microstore.Database(cfg.Store.Database),
+					microstore.Table(cfg.Store.Table),
+					store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
+				)
+
+				svc, err := service.NewPostprocessingService(ctx, logger, st, traceProvider, cfg)
+				if err != nil {
+					return err
+				}
+
+				gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
+					return svc.Run()
+				}, func() {
+					svc.Close()
+				}))
+			}
+
+			{
+				debugServer, err := debug.Server(
+					debug.Logger(logger),
+					debug.Context(ctx),
+					debug.Config(cfg),
+				)
+				if err != nil {
+					logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server")
+					return err
+				}
+
+				gr.Add(runner.NewGolangHttpServerRunner("postprocessing_debug", debugServer))
+			}
+
+			grResults := gr.Run(ctx)
+
+			// return the first non-nil error found in the results
+			for _, grResult := range grResults {
+				if grResult.RunnerError != nil {
+					return grResult.RunnerError
+				}
+			}
+			return nil
+		},
+	}
+}

Server/services/postprocessing/pkg/command/version.go

@@ -0,0 +1,18 @@
+package command
+
+import (
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+// Version prints the service versions of all running instances.
+func Version(cfg *config.Config) *cobra.Command {
+	return &cobra.Command{
+		Use:   "version",
+		Short: "print the version of this binary and the running extension instances",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// not implemented
+			return nil
+		},
+	}
+}

Server/services/postprocessing/pkg/config/config.go

@@ -0,0 +1,69 @@
+package config
+
+import (
+	"context"
+	"time"
+
+	"github.com/qsfera/server/pkg/shared"
+)
+
+// Config combines all available configuration parts.
+type Config struct {
+	Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
+
+	Service Service `yaml:"-"`
+
+	LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;POSTPROCESSING_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
+	Debug    Debug  `yaml:"debug"`
+
+	Store          Store          `yaml:"store"`
+	Postprocessing Postprocessing `yaml:"postprocessing"`
+
+	Context context.Context `yaml:"-"`
+}
+
+// Postprocessing defines the config options for the postprocessing service.
+type Postprocessing struct {
+	Events  Events `yaml:"events"`
+	Workers int    `yaml:"workers" env:"POSTPROCESSING_WORKERS" desc:"The number of concurrent go routines that fetch events from the event queue." introductionVersion:"1.0.0"`
+
+	Steps           []string      `yaml:"steps" env:"POSTPROCESSING_STEPS" desc:"A list of postprocessing steps processed in order of their appearance. Currently supported values by the system are: 'virusscan', 'policies' and 'delay'. Custom steps are allowed. See the documentation for instructions. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
+	Delayprocessing time.Duration `yaml:"delayprocessing" env:"POSTPROCESSING_DELAY" desc:"After uploading a file but before making it available for download, a delay step can be added. Intended for developing purposes only. If a duration is set but the keyword 'delay' is not explicitely added to 'POSTPROCESSING_STEPS', the delay step will be processed as last step. In such a case, a log entry will be written on service startup to remind the admin about that situation. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
+
+	RetryBackoffDuration time.Duration `yaml:"retry_backoff_duration" env:"POSTPROCESSING_RETRY_BACKOFF_DURATION" desc:"The base for the exponential backoff duration before retrying a failed postprocessing step. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
+	MaxRetries           int           `yaml:"max_retries" env:"POSTPROCESSING_MAX_RETRIES" desc:"The maximum number of retries for a failed postprocessing step." introductionVersion:"1.0.0"`
+}
+
+// Events combines the configuration options for the event bus.
+type Events struct {
+	Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POSTPROCESSING_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
+	Cluster  string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POSTPROCESSING_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
+
+	TLSInsecure          bool   `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POSTPROCESSING_EVENTS_TLS_INSECURE" desc:"Whether the КуСфера server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
+	TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;POSTPROCESSING_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided POSTPROCESSING_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"`
+	EnableTLS            bool   `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;POSTPROCESSING_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
+	AuthUsername         string `yaml:"username" env:"OC_EVENTS_AUTH_USERNAME;POSTPROCESSING_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
+	AuthPassword         string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;POSTPROCESSING_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
+
+	MaxAckPending int           `yaml:"max_ack_pending" env:"SEARCH_EVENTS_MAX_ACK_PENDING" desc:"The maximum number of unacknowledged messages. This is used to limit the number of messages that can be in flight at the same time." introductionVersion:"4.0.0"`
+	AckWait       time.Duration `yaml:"ack_wait" env:"SEARCH_EVENTS_ACK_WAIT" desc:"The time to wait for an ack before the message is redelivered. This is used to ensure that messages are not lost if the consumer crashes." introductionVersion:"4.0.0"`
+}
+
+// Debug defines the available debug configuration.
+type Debug struct {
+	Addr   string `yaml:"addr" env:"POSTPROCESSING_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
+	Token  string `yaml:"token" env:"POSTPROCESSING_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
+	Pprof  bool   `yaml:"pprof" env:"POSTPROCESSING_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
+	Zpages bool   `yaml:"zpages" env:"POSTPROCESSING_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
+}
+
+// Store configures the store to use
+type Store struct {
+	Store        string        `yaml:"store" env:"OC_PERSISTENT_STORE;POSTPROCESSING_STORE" desc:"The type of the store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
+	Nodes        []string      `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;POSTPROCESSING_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
+	Database     string        `yaml:"database" env:"POSTPROCESSING_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
+	Table        string        `yaml:"table" env:"POSTPROCESSING_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
+	TTL          time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;POSTPROCESSING_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
+	AuthUsername string        `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;POSTPROCESSING_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
+	AuthPassword string        `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;POSTPROCESSING_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
+}

Server/services/postprocessing/pkg/config/defaults/defaultconfig.go

@@ -0,0 +1,59 @@
+package defaults
+
+import (
+	"time"
+
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+)
+
+// FullDefaultConfig returns a full sanitized config
+func FullDefaultConfig() *config.Config {
+	cfg := DefaultConfig()
+	EnsureDefaults(cfg)
+	Sanitize(cfg)
+	return cfg
+}
+
+// DefaultConfig is the default configuration
+func DefaultConfig() *config.Config {
+	return &config.Config{
+		Debug: config.Debug{
+			Addr:   "127.0.0.1:9255",
+			Token:  "",
+			Pprof:  false,
+			Zpages: false,
+		},
+		Service: config.Service{
+			Name: "postprocessing",
+		},
+		Postprocessing: config.Postprocessing{
+			Events: config.Events{
+				Endpoint:      "127.0.0.1:9233",
+				Cluster:       "qsfera-cluster",
+				MaxAckPending: 10_000,
+				AckWait:       1 * time.Minute,
+			},
+			Workers:              3,
+			RetryBackoffDuration: 5 * time.Second,
+			MaxRetries:           14,
+		},
+		Store: config.Store{
+			Store:    "nats-js-kv",
+			Nodes:    []string{"127.0.0.1:9233"},
+			Database: "postprocessing",
+			Table:    "",
+			TTL:      7 * 24 * time.Hour,
+		},
+	}
+}
+
+// EnsureDefaults ensures defaults on a config
+func EnsureDefaults(cfg *config.Config) {
+	if cfg.LogLevel == "" {
+		cfg.LogLevel = "error"
+	}
+}
+
+// Sanitize does nothing atm
+func Sanitize(cfg *config.Config) {
+}

Server/services/postprocessing/pkg/config/parser/parse.go

@@ -0,0 +1,60 @@
+package parser
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	occfg "github.com/qsfera/server/pkg/config"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/qsfera/server/services/postprocessing/pkg/config/defaults"
+	"github.com/opencloud-eu/reva/v2/pkg/events"
+
+	"github.com/qsfera/server/pkg/config/envdecode"
+)
+
+// ParseConfig loads configuration from known paths.
+func ParseConfig(cfg *config.Config) error {
+	err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
+	if err != nil {
+		return err
+	}
+
+	defaults.EnsureDefaults(cfg)
+
+	// load all env variables relevant to the config in the current context.
+	if err := envdecode.Decode(cfg); err != nil {
+		// no environment variable set for this config is an expected "error"
+		if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
+			return err
+		}
+	}
+
+	defaults.Sanitize(cfg)
+
+	return Validate(cfg)
+}
+
+// Validate validates the config
+func Validate(cfg *config.Config) error {
+	if cfg.Postprocessing.Delayprocessing != 0 {
+		if !contains(cfg.Postprocessing.Steps, events.PPStepDelay) {
+			if len(cfg.Postprocessing.Steps) > 0 {
+				s := strings.Join(append(cfg.Postprocessing.Steps, string(events.PPStepDelay)), ",")
+				fmt.Printf("Added delay step to the list of postprocessing steps. NOTE: Use envvar `POSTPROCESSING_STEPS=%s` to suppress this message and choose the order of postprocessing steps.\n", s)
+			}
+
+			cfg.Postprocessing.Steps = append(cfg.Postprocessing.Steps, string(events.PPStepDelay))
+		}
+	}
+	return nil
+}
+
+func contains(all []string, candidate events.Postprocessingstep) bool {
+	for _, s := range all {
+		if s == string(candidate) {
+			return true
+		}
+	}
+	return false
+}

Server/services/postprocessing/pkg/config/service.go

@@ -0,0 +1,6 @@
+package config
+
+// Service defines the available service configuration.
+type Service struct {
+	Name string `yaml:"-"`
+}

Server/services/postprocessing/pkg/metrics/metrics.go

@@ -0,0 +1,77 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// Namespace defines the namespace for the defines metrics.
+	Namespace = "qsfera"
+
+	// Subsystem defines the subsystem for the defines metrics.
+	Subsystem = "postprocessing"
+
+	buildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "build_info",
+		Help:      "Build information",
+	}, []string{"version"})
+	eventsOutstandingAcks = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "events_outstanding_acks",
+		Help:      "Number of outstanding acks for events",
+	})
+	eventsUnprocessed = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "events_unprocessed",
+		Help:      "Number of unprocessed events",
+	})
+	eventsRedelivered = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "events_redelivered",
+		Help:      "Number of redelivered events",
+	})
+	finished = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "finished",
+		Help:      "Number of finished postprocessing events",
+	}, []string{"status"})
+	duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: Namespace,
+		Subsystem: Subsystem,
+		Name:      "duration_seconds",
+		Help:      "Duration of postprocessing operations in seconds",
+		Buckets:   []float64{0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1200},
+	}, []string{"status"})
+)
+
+// Metrics defines the available metrics of this service.
+type Metrics struct {
+	// Counter  *prometheus.CounterVec
+	BuildInfo             *prometheus.GaugeVec
+	EventsOutstandingAcks prometheus.Gauge
+	EventsUnprocessed     prometheus.Gauge
+	EventsRedelivered     prometheus.Gauge
+	Finished              *prometheus.CounterVec
+	Duration              *prometheus.HistogramVec
+}
+
+// New initializes the available metrics.
+func New() *Metrics {
+	m := &Metrics{
+		BuildInfo:             buildInfo,
+		EventsOutstandingAcks: eventsOutstandingAcks,
+		EventsUnprocessed:     eventsUnprocessed,
+		EventsRedelivered:     eventsRedelivered,
+		Finished:              finished,
+		Duration:              duration,
+	}
+
+	return m
+}

Server/services/postprocessing/pkg/postprocessing/postprocessing.go

@@ -0,0 +1,137 @@
+package postprocessing
+
+import (
+	"math"
+	"time"
+
+	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/opencloud-eu/reva/v2/pkg/events"
+)
+
+// Postprocessing handles postprocessing of a file
+type Postprocessing struct {
+	ID                string
+	URL               string
+	User              *user.User
+	ImpersonatingUser *user.User
+	Filename          string
+	Filesize          uint64
+	ResourceID        *provider.ResourceId
+	Steps             []events.Postprocessingstep
+	Status            Status
+	Failures          int
+	InitiatorID       string
+	Finished          bool
+	StartTime         time.Time
+
+	config config.Postprocessing
+}
+
+// Status is helper struct to show current postprocessing status
+type Status struct {
+	CurrentStep events.Postprocessingstep
+	Outcome     events.PostprocessingOutcome
+}
+
+// New returns a new postprocessing instance
+func New(config config.Postprocessing) *Postprocessing {
+	return &Postprocessing{
+		config: config,
+	}
+}
+
+// Init is the first step of the postprocessing
+func (pp *Postprocessing) Init(_ events.BytesReceived) any {
+	if len(pp.Steps) == 0 {
+		return pp.finished(events.PPOutcomeContinue)
+	}
+
+	return pp.step(pp.Steps[0])
+}
+
+// NextStep returns the next postprocessing step
+func (pp *Postprocessing) NextStep(ev events.PostprocessingStepFinished) any {
+	switch ev.Outcome {
+	case events.PPOutcomeContinue:
+		return pp.next(ev.FinishedStep)
+	case events.PPOutcomeRetry:
+		pp.Failures++
+		if pp.Failures > pp.config.MaxRetries {
+			return pp.finished(events.PPOutcomeAbort)
+		}
+		return pp.retry()
+	default:
+		return pp.finished(ev.Outcome)
+	}
+}
+
+// CurrentStep returns the current postprocessing step
+func (pp *Postprocessing) CurrentStep() any {
+	if pp.Status.CurrentStep == events.PPStepFinished {
+		return pp.finished(pp.Status.Outcome)
+	}
+	return pp.step(pp.Status.CurrentStep)
+}
+
+// Delay will sleep the configured time then continue
+func (pp *Postprocessing) Delay(f func(next any)) {
+	next := pp.next(events.PPStepDelay)
+	go func() {
+		time.Sleep(pp.config.Delayprocessing)
+		f(next)
+	}()
+}
+
+// BackoffDuration calculates the duration for exponential backoff based on the number of failures.
+func (pp *Postprocessing) BackoffDuration() time.Duration {
+	return pp.config.RetryBackoffDuration * time.Duration(math.Pow(2, float64(pp.Failures-1)))
+}
+
+func (pp *Postprocessing) next(current events.Postprocessingstep) any {
+	l := len(pp.Steps)
+	for i, s := range pp.Steps {
+		if s == current && i+1 < l {
+			return pp.step(pp.Steps[i+1])
+		}
+	}
+	return pp.finished(events.PPOutcomeContinue)
+}
+
+func (pp *Postprocessing) step(next events.Postprocessingstep) events.StartPostprocessingStep {
+	pp.Status.CurrentStep = next
+	return events.StartPostprocessingStep{
+		UploadID:          pp.ID,
+		URL:               pp.URL,
+		ExecutingUser:     pp.User,
+		Filename:          pp.Filename,
+		Filesize:          pp.Filesize,
+		ResourceID:        pp.ResourceID,
+		StepToStart:       next,
+		ImpersonatingUser: pp.ImpersonatingUser,
+	}
+}
+
+func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events.PostprocessingFinished {
+	pp.Status.CurrentStep = events.PPStepFinished
+	pp.Status.Outcome = outcome
+	return events.PostprocessingFinished{
+		UploadID:          pp.ID,
+		ExecutingUser:     pp.User,
+		Filename:          pp.Filename,
+		Outcome:           outcome,
+		ImpersonatingUser: pp.ImpersonatingUser,
+	}
+}
+
+func (pp *Postprocessing) retry() events.PostprocessingRetry {
+	pp.Status.Outcome = events.PPOutcomeRetry
+	return events.PostprocessingRetry{
+		UploadID:        pp.ID,
+		ExecutingUser:   pp.User,
+		Filename:        pp.Filename,
+		Failures:        pp.Failures,
+		BackoffDuration: pp.BackoffDuration(),
+	}
+}

Server/services/postprocessing/pkg/server/debug/option.go

@@ -0,0 +1,50 @@
+package debug
+
+import (
+	"context"
+
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+)
+
+// Option defines a single option function.
+type Option func(o *Options)
+
+// Options defines the available options for this package.
+type Options struct {
+	Logger  log.Logger
+	Context context.Context
+	Config  *config.Config
+}
+
+// newOptions initializes the available default options.
+func newOptions(opts ...Option) Options {
+	opt := Options{}
+
+	for _, o := range opts {
+		o(&opt)
+	}
+
+	return opt
+}
+
+// Logger provides a function to set the logger option.
+func Logger(val log.Logger) Option {
+	return func(o *Options) {
+		o.Logger = val
+	}
+}
+
+// Context provides a function to set the context option.
+func Context(val context.Context) Option {
+	return func(o *Options) {
+		o.Context = val
+	}
+}
+
+// Config provides a function to set the config option.
+func Config(val *config.Config) Option {
+	return func(o *Options) {
+		o.Config = val
+	}
+}

Server/services/postprocessing/pkg/server/debug/server.go

@@ -0,0 +1,30 @@
+package debug
+
+import (
+	"net/http"
+
+	"github.com/qsfera/server/pkg/checks"
+	"github.com/qsfera/server/pkg/handlers"
+	"github.com/qsfera/server/pkg/service/debug"
+	"github.com/qsfera/server/pkg/version"
+)
+
+// Server initializes the debug service and server.
+func Server(opts ...Option) (*http.Server, error) {
+	options := newOptions(opts...)
+
+	readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
+		WithLogger(options.Logger).
+		WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Postprocessing.Events.Endpoint))
+
+	return debug.NewService(
+		debug.Logger(options.Logger),
+		debug.Name(options.Config.Service.Name),
+		debug.Version(version.GetString()),
+		debug.Address(options.Config.Debug.Addr),
+		debug.Token(options.Config.Debug.Token),
+		debug.Pprof(options.Config.Debug.Pprof),
+		debug.Zpages(options.Config.Debug.Zpages),
+		debug.Ready(handlers.NewCheckHandler(readyHandlerConfiguration)),
+	), nil
+}

Server/services/postprocessing/pkg/service/service.go

@@ -0,0 +1,452 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/qsfera/server/pkg/generators"
+	"github.com/qsfera/server/pkg/log"
+	"github.com/qsfera/server/pkg/version"
+	"github.com/qsfera/server/services/postprocessing/pkg/config"
+	"github.com/qsfera/server/services/postprocessing/pkg/metrics"
+	"github.com/qsfera/server/services/postprocessing/pkg/postprocessing"
+	ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
+	"github.com/opencloud-eu/reva/v2/pkg/events"
+	"github.com/opencloud-eu/reva/v2/pkg/events/raw"
+	"github.com/opencloud-eu/reva/v2/pkg/events/stream"
+	"github.com/opencloud-eu/reva/v2/pkg/utils"
+	"go-micro.dev/v4/store"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// PostprocessingService is an instance of the service handling postprocessing of files
+type PostprocessingService struct {
+	ctx     context.Context
+	log     log.Logger
+	events  <-chan raw.Event
+	pub     events.Publisher
+	steps   []events.Postprocessingstep
+	store   store.Store
+	c       config.Postprocessing
+	tp      trace.TracerProvider
+	metrics *metrics.Metrics
+	stopCh  chan struct{}
+	stopped atomic.Bool
+}
+
+var (
+	// ErrFatal is returned when a fatal error occurs and we want to exit.
+	ErrFatal = errors.New("fatal error")
+	// ErrEvent is returned when something went wrong with a specific event.
+	ErrEvent = errors.New("event error")
+	// ErrNotFound is returned when a postprocessing is not found in the store.
+	ErrNotFound = errors.New("postprocessing not found")
+)
+
+// NewPostprocessingService returns a new instance of a postprocessing service
+func NewPostprocessingService(ctx context.Context, logger log.Logger, sto store.Store, tp trace.TracerProvider, cfg *config.Config) (*PostprocessingService, error) {
+	connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
+	pub, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
+		Endpoint:             cfg.Postprocessing.Events.Endpoint,
+		Cluster:              cfg.Postprocessing.Events.Cluster,
+		EnableTLS:            cfg.Postprocessing.Events.EnableTLS,
+		TLSInsecure:          cfg.Postprocessing.Events.TLSInsecure,
+		TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
+		AuthUsername:         cfg.Postprocessing.Events.AuthUsername,
+		AuthPassword:         cfg.Postprocessing.Events.AuthPassword,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := raw.FromConfig(ctx, connName, raw.Config{
+		Endpoint:             cfg.Postprocessing.Events.Endpoint,
+		Cluster:              cfg.Postprocessing.Events.Cluster,
+		EnableTLS:            cfg.Postprocessing.Events.EnableTLS,
+		TLSInsecure:          cfg.Postprocessing.Events.TLSInsecure,
+		TLSRootCACertificate: cfg.Postprocessing.Events.TLSRootCACertificate,
+		AuthUsername:         cfg.Postprocessing.Events.AuthUsername,
+		AuthPassword:         cfg.Postprocessing.Events.AuthPassword,
+		MaxAckPending:        cfg.Postprocessing.Events.MaxAckPending,
+		AckWait:              cfg.Postprocessing.Events.AckWait,
+	})
+
+	evs, err := raw.Consume("postprocessing-pull",
+		events.BytesReceived{},
+		events.StartPostprocessingStep{},
+		events.UploadReady{},
+		events.PostprocessingStepFinished{},
+		events.ResumePostprocessing{})
+	if err != nil {
+		return nil, err
+	}
+
+	m := metrics.New()
+	m.BuildInfo.WithLabelValues(version.GetString()).Set(1)
+	monitorMetrics(ctx, raw, "postprocessing-pull", m, logger)
+
+	return &PostprocessingService{
+		ctx:     ctx,
+		log:     logger,
+		events:  evs,
+		pub:     pub,
+		steps:   getSteps(cfg.Postprocessing),
+		store:   sto,
+		c:       cfg.Postprocessing,
+		tp:      tp,
+		metrics: m,
+		stopCh:  make(chan struct{}, 1),
+	}, nil
+}
+
+// Run to fulfil Runner interface
+func (pps *PostprocessingService) Run() error {
+	wg := sync.WaitGroup{}
+
+	for range pps.c.Workers {
+		wg.Go(func() {
+
+		EventLoop:
+			for {
+				select {
+				case <-pps.stopCh:
+					// stop requested
+					// TODO: we might need a way to unsubscribe from the event channel, otherwise
+					// we'll be leaking a goroutine in reva that will be stuck waiting for
+					// someone to read from the event channel.
+					// Note: redis implementation seems to have a timeout, so the goroutine
+					// will exit if there is nobody processing the events and the timeout
+					// is reached. The behavior is unclear with natsjs
+					break EventLoop
+				case e, ok := <-pps.events:
+					if !ok {
+						// event channel is closed, so nothing more to do
+						break EventLoop
+					}
+
+					err := pps.processEvent(e)
+					if err != nil {
+						switch {
+						case errors.Is(err, ErrFatal):
+							pps.log.Fatal().Err(err).Msg("fatal error - exiting")
+						case errors.Is(err, ErrEvent):
+							pps.log.Error().Err(err).Msg("continuing")
+						default:
+							pps.log.Fatal().Err(err).Msg("unknown error - exiting")
+						}
+					}
+
+					if pps.stopped.Load() {
+						// if stopped, don't process any more events
+						break EventLoop
+					}
+				}
+			}
+		})
+	}
+
+	wg.Wait()
+
+	return nil
+}
+
+// Close will make the postprocessing service to stop processing, so the `Run`
+// method can finish.
+// TODO: Underlying services can't be stopped. This means that some goroutines
+// will get stuck trying to push events through a channel nobody is reading
+// from, so resources won't be freed and there will be memory leaks. For now,
+// if the service is stopped, you should close the app soon after.
+func (pps *PostprocessingService) Close() {
+	if pps.stopped.CompareAndSwap(false, true) {
+		close(pps.stopCh)
+	}
+}
+
+func (pps *PostprocessingService) processEvent(e raw.Event) error {
+	pps.log.Debug().Str("Type", e.Type).Str("ID", e.ID).Msg("processing event received")
+
+	var (
+		next any
+		pp   *postprocessing.Postprocessing
+		err  error
+	)
+
+	ctx := e.GetTraceContext(pps.ctx)
+	ctx, span := pps.tp.Tracer("postprocessing").Start(ctx, "processEvent")
+	defer span.End()
+
+	ackEvent := true
+	defer func() {
+		if ackEvent {
+			if err := e.Ack(); err != nil {
+				pps.log.Error().Err(err).Msg("unable to ack event")
+			}
+		}
+	}()
+
+	switch ev := e.Event.Event.(type) {
+	case events.BytesReceived:
+		pp = &postprocessing.Postprocessing{
+			ID:                ev.UploadID,
+			URL:               ev.URL,
+			User:              ev.ExecutingUser,
+			Filename:          ev.Filename,
+			Filesize:          ev.Filesize,
+			ResourceID:        ev.ResourceID,
+			Steps:             pps.steps,
+			InitiatorID:       e.InitiatorID,
+			ImpersonatingUser: ev.ImpersonatingUser,
+			StartTime:         time.Now(),
+		}
+		next = pp.Init(ev)
+	case events.PostprocessingStepFinished:
+		if ev.UploadID == "" {
+			// no current upload - this was an on demand scan
+			return nil
+		}
+		pp, err = pps.getPP(pps.store, ev.UploadID)
+		if err != nil {
+			pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
+			return fmt.Errorf("%w: cannot get upload", ErrEvent)
+		}
+		next = pp.NextStep(ev)
+
+		switch pp.Status.Outcome {
+		case events.PPOutcomeRetry:
+			// schedule retry
+			backoff := pp.BackoffDuration()
+			go func() {
+				time.Sleep(backoff)
+				retryEvent := events.StartPostprocessingStep{
+					UploadID:          pp.ID,
+					URL:               pp.URL,
+					ExecutingUser:     pp.User,
+					Filename:          pp.Filename,
+					Filesize:          pp.Filesize,
+					ResourceID:        pp.ResourceID,
+					StepToStart:       pp.Status.CurrentStep,
+					ImpersonatingUser: pp.ImpersonatingUser,
+				}
+				err := events.Publish(ctx, pps.pub, retryEvent)
+				if err != nil {
+					pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot publish RestartPostprocessing event")
+				}
+			}()
+		}
+	case events.StartPostprocessingStep:
+		if ev.StepToStart != events.PPStepDelay {
+			return nil
+		}
+		pp, err = pps.getPP(pps.store, ev.UploadID)
+		if err != nil {
+			pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
+			return fmt.Errorf("%w: cannot get upload", ErrEvent)
+		}
+		pp.Delay(func(next any) {
+			if err := events.Publish(ctx, pps.pub, next); err != nil {
+				pps.log.Error().Err(err).Msg("cannot publish event")
+			}
+		})
+	case events.UploadReady:
+		// the upload failed - let's keep it around for a while - but mark it as finished
+		pp, err = pps.getPP(pps.store, ev.UploadID)
+		if err != nil {
+			pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload")
+			return fmt.Errorf("%w: cannot get upload", ErrEvent)
+		}
+
+		if ev.Failed {
+			pps.metrics.Finished.WithLabelValues("failed").Inc()
+			if !pp.StartTime.IsZero() {
+				pps.metrics.Duration.WithLabelValues("failed").Observe(time.Since(pp.StartTime).Seconds())
+			}
+			pp.Finished = true
+			return storePP(pps.store, pp)
+		}
+
+		pps.metrics.Finished.WithLabelValues("succeeded").Inc()
+		if !pp.StartTime.IsZero() {
+			pps.metrics.Duration.WithLabelValues("succeeded").Observe(time.Since(pp.StartTime).Seconds())
+		}
+		// the storage provider thinks the upload is done - so no need to keep it any more
+		if err := pps.store.Delete(ev.UploadID); err != nil {
+			pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot delete upload")
+			return fmt.Errorf("%w: cannot delete upload", ErrEvent)
+		}
+	case events.ResumePostprocessing:
+		return pps.handleResumePPEvent(ctx, ev)
+	}
+
+	if pp != nil {
+		ctx = ctxpkg.ContextSetInitiator(ctx, pp.InitiatorID)
+
+		if err := storePP(pps.store, pp); err != nil {
+			ackEvent = false
+			pps.log.Error().Str("uploadID", pp.ID).Err(err).Msg("cannot store upload")
+			return fmt.Errorf("%w: cannot store upload", ErrEvent)
+		}
+	}
+
+	if next != nil {
+		if err := events.Publish(ctx, pps.pub, next); err != nil {
+			pps.log.Error().Err(err).Msg("unable to publish event")
+			return fmt.Errorf("%w: unable to publish event", ErrFatal) // we can't publish -> we are screwed
+		}
+	}
+	return nil
+}
+
+func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) {
+	recs, err := sto.Read(uploadID)
+	if err != nil {
+		if err == store.ErrNotFound {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	if len(recs) == 0 {
+		return nil, ErrNotFound
+	}
+
+	if len(recs) > 1 {
+		return nil, fmt.Errorf("expected only one result for '%s', got %d", uploadID, len(recs))
+	}
+
+	pp := postprocessing.New(pps.c)
+	err = json.Unmarshal(recs[0].Value, pp)
+	if err != nil {
+		return nil, err
+	}
+
+	return pp, nil
+}
+
+func getSteps(c config.Postprocessing) []events.Postprocessingstep {
+	// NOTE: improved version only allows configuring order of postprocessing steps
+	// But we aim for a system where postprocessing steps can be configured per space, ideally by the spaceadmin itself
+	// We need to iterate over configuring PP service when we see fit
+	steps := make([]events.Postprocessingstep, 0, len(c.Steps))
+	for _, s := range c.Steps {
+		steps = append(steps, events.Postprocessingstep(s))
+	}
+
+	return steps
+}
+
+func storePP(sto store.Store, pp *postprocessing.Postprocessing) error {
+	b, err := json.Marshal(pp)
+	if err != nil {
+		return err
+	}
+
+	return sto.Write(&store.Record{
+		Key:   pp.ID,
+		Value: b,
+	})
+}
+
+func (pps *PostprocessingService) handleResumePPEvent(ctx context.Context, ev events.ResumePostprocessing) error {
+	ids := []string{ev.UploadID}
+	if ev.Step != "" {
+		ids = pps.findUploadsByStep(ev.Step)
+	}
+
+	for _, id := range ids {
+		if err := pps.resumePP(ctx, id); err != nil {
+			pps.log.Error().Str("uploadID", id).Err(err).Msg("cannot resume upload")
+		}
+	}
+	return nil
+}
+
+func (pps *PostprocessingService) resumePP(ctx context.Context, uploadID string) error {
+	pp, err := pps.getPP(pps.store, uploadID)
+	if err != nil {
+		if err == ErrNotFound {
+			if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{
+				UploadID:  uploadID,
+				Timestamp: utils.TSNow(),
+			}); err != nil {
+				return err
+			}
+			return nil
+		}
+		return fmt.Errorf("cannot get upload: %w", err)
+	}
+
+	if pp.Finished {
+		// dont retry finished uploads
+		return nil
+	}
+
+	return events.Publish(ctx, pps.pub, pp.CurrentStep())
+}
+
+func (pps *PostprocessingService) findUploadsByStep(step events.Postprocessingstep) []string {
+	var ids []string
+
+	keys, err := pps.store.List()
+	if err != nil {
+		pps.log.Error().Err(err).Msg("cannot list uploads")
+	}
+
+	for _, k := range keys {
+		rec, err := pps.store.Read(k)
+		if err != nil {
+			pps.log.Error().Err(err).Msg("cannot read upload")
+			continue
+		}
+
+		if len(rec) != 1 {
+			pps.log.Error().Err(err).Msg("expected only one result")
+			continue
+		}
+
+		pp := &postprocessing.Postprocessing{}
+		err = json.Unmarshal(rec[0].Value, pp)
+		if err != nil {
+			pps.log.Error().Err(err).Msg("cannot unmarshal upload")
+			continue
+		}
+
+		if pp.Status.CurrentStep == step {
+			ids = append(ids, pp.ID)
+		}
+	}
+
+	return ids
+}
+
+func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metrics.Metrics, logger log.Logger) {
+	consumer, err := stream.JetStream().Consumer(ctx, name)
+	if err != nil {
+		logger.Error().Err(err).Msg("failed to get consumer")
+	}
+	ticker := time.NewTicker(5 * time.Second)
+	go func() {
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				info, err := consumer.Info(ctx)
+				if err != nil {
+					logger.Error().Err(err).Msg("failed to get consumer")
+					continue
+				}
+
+				m.EventsOutstandingAcks.Set(float64(info.NumAckPending))
+				m.EventsUnprocessed.Set(float64(info.NumPending))
+				m.EventsRedelivered.Set(float64(info.NumRedelivered))
+				logger.Trace().Msg("updated postprocessing event metrics")
+			}
+		}
+	}()
+}