From f4ba4e0f64dc906fdb9ac0bc828af509204d6800 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 15 Mar 2023 15:21:45 +0100 Subject: [PATCH 1/8] backport antivirus from experimental Signed-off-by: jkoberg --- Makefile | 1 + ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/command/antivirus.go | 30 +++ services/antivirus/Makefile | 37 ++++ services/antivirus/README.md | 0 services/antivirus/cmd/antivirus/main.go | 14 ++ services/antivirus/pkg/command/health.go | 61 ++++++ services/antivirus/pkg/command/root.go | 54 +++++ services/antivirus/pkg/command/server.go | 107 +++++++++ services/antivirus/pkg/command/version.go | 26 +++ services/antivirus/pkg/config/config.go | 72 ++++++ .../pkg/config/defaults/defaultconfig.go | 54 +++++ services/antivirus/pkg/config/parser/parse.go | 38 ++++ services/antivirus/pkg/scanners/clamav.go | 35 +++ services/antivirus/pkg/scanners/icap.go | 68 ++++++ services/antivirus/pkg/scanners/scanners.go | 34 +++ services/antivirus/pkg/service/service.go | 205 ++++++++++++++++++ .../policies/pkg/service/event/service.go | 3 +- .../postprocessing/pkg/service/service.go | 1 - 20 files changed, 842 insertions(+), 2 deletions(-) create mode 100644 ocis/pkg/command/antivirus.go create mode 100644 services/antivirus/Makefile create mode 100644 services/antivirus/README.md create mode 100644 services/antivirus/cmd/antivirus/main.go create mode 100644 services/antivirus/pkg/command/health.go create mode 100644 services/antivirus/pkg/command/root.go create mode 100644 services/antivirus/pkg/command/server.go create mode 100644 services/antivirus/pkg/command/version.go create mode 100644 services/antivirus/pkg/config/config.go create mode 100644 services/antivirus/pkg/config/defaults/defaultconfig.go create mode 100644 services/antivirus/pkg/config/parser/parse.go create mode 100644 services/antivirus/pkg/scanners/clamav.go create mode 100644 services/antivirus/pkg/scanners/icap.go create mode 100644 services/antivirus/pkg/scanners/scanners.go create mode 100644 services/antivirus/pkg/service/service.go diff --git a/Makefile b/Makefile index 63eed8599..056c7c57a 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ L10N_MODULES := \ # if you add a module here please also add it to the .drone.star file OCIS_MODULES = \ + services/antivirus \ services/app-provider \ services/app-registry \ services/audit \ diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 3d368f832..1b10a1e74 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "github.com/owncloud/ocis/v2/ocis-pkg/shared" + antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/config" appRegistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/config" audit "github.com/owncloud/ocis/v2/services/audit/pkg/config" @@ -72,6 +73,7 @@ type Config struct { AdminUserID string `yaml:"admin_user_id" env:"OCIS_ADMIN_USER_ID" desc:"ID of a user, that should receive admin privileges. Consider that the UUID can be encoded in some LDAP deployment configurations like in .ldif files. These need to be decoded beforehand."` Runtime Runtime `yaml:"runtime"` + Antivirus *antivirus.Config `yaml:"antivirus"` AppProvider *appProvider.Config `yaml:"app_provider"` AppRegistry *appRegistry.Config `yaml:"app_registry"` Audit *audit.Config `yaml:"audit"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index d59571d11..89579f7d8 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -1,6 +1,7 @@ package config import ( + antivirus "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults" appProvider "github.com/owncloud/ocis/v2/services/app-provider/pkg/config/defaults" appRegistry "github.com/owncloud/ocis/v2/services/app-registry/pkg/config/defaults" audit "github.com/owncloud/ocis/v2/services/audit/pkg/config/defaults" @@ -45,6 +46,7 @@ func DefaultConfig() *Config { Host: "localhost", }, + Antivirus: antivirus.DefaultConfig(), AppProvider: appProvider.DefaultConfig(), AppRegistry: appRegistry.DefaultConfig(), Audit: audit.DefaultConfig(), diff --git a/ocis/pkg/command/antivirus.go b/ocis/pkg/command/antivirus.go new file mode 100644 index 000000000..5671930a9 --- /dev/null +++ b/ocis/pkg/command/antivirus.go @@ -0,0 +1,30 @@ +package command + +import ( + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" + "github.com/owncloud/ocis/v2/ocis/pkg/command/helper" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/command" + "github.com/urfave/cli/v2" +) + +// AntivirusCommand is the entrypoint for the antivirus command. +func AntivirusCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: cfg.Antivirus.Service.Name, + Usage: helper.SubcommandDescription(cfg.Antivirus.Service.Name), + Category: "services", + Before: func(c *cli.Context) error { + configlog.Error(parser.ParseConfig(cfg, true)) + //cfg.Antivirus.Commons = cfg.Commons + return nil + }, + Subcommands: command.GetCommands(cfg.Antivirus), + } +} + +func init() { + register.AddCommand(AntivirusCommand) +} diff --git a/services/antivirus/Makefile b/services/antivirus/Makefile new file mode 100644 index 000000000..2220cb7c1 --- /dev/null +++ b/services/antivirus/Makefile @@ -0,0 +1,37 @@ +SHELL := bash +NAME := antivirus + +include ../../.make/recursion.mk + +############ tooling ############ +ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI +include ../../.bingo/Variables.mk +endif + +############ go tooling ############ +include ../../.make/go.mk + +############ release ############ +include ../../.make/release.mk + +############ docs generate ############ +include ../../.make/docs.mk + +.PHONY: docs-generate +docs-generate: config-docs-generate + +############ generate ############ +include ../../.make/generate.mk + +.PHONY: ci-go-generate +ci-go-generate: # CI runs ci-node-generate automatically before this target + +.PHONY: ci-node-generate +ci-node-generate: + +############ licenses ############ +.PHONY: ci-node-check-licenses +ci-node-check-licenses: + +.PHONY: ci-node-save-licenses +ci-node-save-licenses: diff --git a/services/antivirus/README.md b/services/antivirus/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/services/antivirus/cmd/antivirus/main.go b/services/antivirus/cmd/antivirus/main.go new file mode 100644 index 000000000..b37a1ca3e --- /dev/null +++ b/services/antivirus/cmd/antivirus/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "os" + + "github.com/owncloud/ocis/v2/services/antivirus/pkg/command" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults" +) + +func main() { + if err := command.Execute(defaults.DefaultConfig()); err != nil { + os.Exit(1) + } +} diff --git a/services/antivirus/pkg/command/health.go b/services/antivirus/pkg/command/health.go new file mode 100644 index 000000000..d348184ca --- /dev/null +++ b/services/antivirus/pkg/command/health.go @@ -0,0 +1,61 @@ +package command + +import ( + "fmt" + "net/http" + + "github.com/owncloud/ocis/v2/ocis-pkg/log" + + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/parser" + "github.com/urfave/cli/v2" +) + +// Health is the entrypoint for the health command. +func Health(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "health", + Usage: "check health status", + Category: "info", + Before: func(c *cli.Context) error { + return configlog.ReturnError(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + logger := log.NewLogger( + log.Name(cfg.Service.Name), + log.Level(cfg.Log.Level), + log.Pretty(cfg.Log.Pretty), + log.Color(cfg.Log.Color), + log.File(cfg.Log.File), + ) + + resp, err := http.Get( + fmt.Sprintf( + "http://%s/healthz", + cfg.Debug.Addr, + ), + ) + + if err != nil { + logger.Fatal(). + Err(err). + Msg("Failed to request health check") + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Fatal(). + Int("code", resp.StatusCode). + Msg("Health seems to be in bad state") + } + + logger.Debug(). + Int("code", resp.StatusCode). + Msg("Health got a good state") + + return nil + }, + } +} diff --git a/services/antivirus/pkg/command/root.go b/services/antivirus/pkg/command/root.go new file mode 100644 index 000000000..5f35285c7 --- /dev/null +++ b/services/antivirus/pkg/command/root.go @@ -0,0 +1,54 @@ +package command + +import ( + "context" + "os" + + "github.com/owncloud/ocis/v2/ocis-pkg/clihelper" + ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/thejerf/suture/v4" + "github.com/urfave/cli/v2" +) + +// GetCommands provides all commands for this service +func GetCommands(cfg *config.Config) cli.Commands { + return []*cli.Command{ + Server(cfg), + Health(cfg), + Version(cfg), + } +} + +// Execute is the entry point for the antivirus command. +func Execute(cfg *config.Config) error { + app := clihelper.DefaultApp(&cli.App{ + Name: "antivirus", + Usage: "Serve ownCloud antivirus for oCIS", + Commands: GetCommands(cfg), + }) + + return app.Run(os.Args) +} + +// SutureService allows for the web command to be embedded and supervised by a suture supervisor tree. +type SutureService struct { + cfg *config.Config +} + +// NewSutureService creates a new web.SutureService +func NewSutureService(cfg *ociscfg.Config) suture.Service { + cfg.Policies.Commons = cfg.Commons + return SutureService{ + cfg: cfg.Antivirus, + } +} + +func (s SutureService) Serve(ctx context.Context) error { + s.cfg.Context = ctx + if err := Execute(s.cfg); err != nil { + return err + } + + return nil +} diff --git a/services/antivirus/pkg/command/server.go b/services/antivirus/pkg/command/server.go new file mode 100644 index 000000000..d1de27fbd --- /dev/null +++ b/services/antivirus/pkg/command/server.go @@ -0,0 +1,107 @@ +package command + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/oklog/run" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/ocis-pkg/service/debug" + "github.com/owncloud/ocis/v2/ocis-pkg/version" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/parser" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/service" + "github.com/urfave/cli/v2" +) + +// Server is the entrypoint for the server command. +func Server(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "server", + Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", "authz"), + Category: "server", + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + var ( + gr = run.Group{} + ctx, cancel = func() (context.Context, context.CancelFunc) { + if cfg.Context == nil { + return context.WithCancel(context.Background()) + } + return context.WithCancel(cfg.Context) + }() + logger = log.NewLogger( + log.Name(cfg.Service.Name), + log.Level(cfg.Log.Level), + log.Pretty(cfg.Log.Pretty), + log.Color(cfg.Log.Color), + log.File(cfg.Log.File), + ) + ) + defer cancel() + + { + svc, err := service.NewAntivirus(cfg, logger) + if err != nil { + return err + } + + gr.Add(svc.Run, func(_ error) { + cancel() + }) + } + + { + server := debug.NewService( + debug.Logger(logger), + debug.Name(cfg.Service.Name), + debug.Version(version.GetString()), + debug.Address(cfg.Debug.Addr), + debug.Token(cfg.Debug.Token), + debug.Pprof(cfg.Debug.Pprof), + debug.Zpages(cfg.Debug.Zpages), + debug.Health( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } + }, + ), + debug.Ready( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } + }, + ), + ) + + gr.Add(server.ListenAndServe, func(_ error) { + _ = server.Shutdown(ctx) + cancel() + }) + } + + return gr.Run() + }, + } +} diff --git a/services/antivirus/pkg/command/version.go b/services/antivirus/pkg/command/version.go new file mode 100644 index 000000000..3286c932c --- /dev/null +++ b/services/antivirus/pkg/command/version.go @@ -0,0 +1,26 @@ +package command + +import ( + "fmt" + + "github.com/owncloud/ocis/v2/ocis-pkg/version" + + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/urfave/cli/v2" +) + +// Version prints the service versions of all running instances. +func Version(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "version", + Usage: "print the version of this binary and the running service instances", + Category: "info", + Action: func(c *cli.Context) error { + fmt.Println("Version: " + version.GetString()) + fmt.Printf("Compiled: %s\n", version.Compiled()) + fmt.Println("") + + return nil + }, + } +} diff --git a/services/antivirus/pkg/config/config.go b/services/antivirus/pkg/config/config.go new file mode 100644 index 000000000..f351027eb --- /dev/null +++ b/services/antivirus/pkg/config/config.go @@ -0,0 +1,72 @@ +package config + +import ( + "context" +) + +// Config combines all available configuration parts. +type Config struct { + File string + Log *Log + + Debug Debug `mask:"struct" yaml:"debug"` + + Service Service `yaml:"-"` + + InfectedFileHandling string `yaml:"infected-file-handling" env:"ANTIVIRUS_INFECTED_FILE_HANDLING" desc:"Defines the behaviour when a virus has been found. Options are: 'delete', 'continue' and 'abort '. Delete will delete the file. Continue will mark the file as infected but continues further processing. Abort will keep the file in the upload folder for further admin inspection and will not move it to its target space."` + Events Events + Scanner Scanner + MaxScanSize string `yaml:"max-scan-size" env:"ANTIVIRUS_MAX_SCAN_SIZE" desc:"The maximum scan size the virusscanner can handle. Only that much bytes of a file will be scanned. 0 means unlimited and is the default. Usable common abbreviations: [KB, KiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB."` + + Context context.Context `yaml:"-" json:"-"` +} + +// Service defines the available service configuration. +type Service struct { + Name string `yaml:"-"` +} + +// Log defines the available log configuration. +type Log struct { + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;POLICIES_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;POLICIES_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;POLICIES_LOG_COLOR" desc:"Activates colorized log output."` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;POLICIES_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` +} + +// Debug defines the available debug configuration. +type Debug struct { + Addr string `yaml:"addr" env:"POLICIES_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` + Token string `yaml:"token" env:"POLICIES_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."` + Pprof bool `yaml:"pprof" env:"POLICIES_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."` + Zpages bool `yaml:"zpages" env:"POLICIES_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` +} + +// Events combines the configuration options for the event bus. +type Events struct { + Endpoint string `yaml:"endpoint" env:"USERLOG_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` + Cluster string `yaml:"cluster" env:"USERLOG_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` + TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;USERLOG_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."` + TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"USERLOG_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."` + EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;USERLOG_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."` +} + +// Scanner provides configuration options for the antivirusscanner +type Scanner struct { + Type string `yaml:"type" env:"ANTIVIRUS_SCANNER_TYPE" desc:"The scanner to use. Must be one of: clamav, icap"` + + ClamAV ClamAV // only if Type == clamav + ICAP ICAP // only if Type == icap +} + +// ClamAV provides configuration option for clamav +type ClamAV struct { + Socket string `yaml:"socket" env:"ANTIVIRUS_CLAMAV_SOCKET" desc:"The socket clamav is running on. Note the default value is an example which needs adaption according your OS."` +} + +// ICAP provides configuration option for ICAP +type ICAP struct { + Timeout int64 `yaml:"timeout" env:"ANTIVIRUS_ICAP_TIMEOUT" desc:"Timeout for the ICAP client."` + URL string `yaml:"url" env:"ANTIVIRUS_ICAP_URL" desc:"URL of the ICAP server."` + Service string `yaml:"service" env:"ANTIVIRUS_ICAP_SERVICE" desc:"Name of the ICAP server."` +} diff --git a/services/antivirus/pkg/config/defaults/defaultconfig.go b/services/antivirus/pkg/config/defaults/defaultconfig.go new file mode 100644 index 000000000..90bc649a3 --- /dev/null +++ b/services/antivirus/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,54 @@ +package defaults + +import ( + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" +) + +// FullDefaultConfig returns a fully initialized default configuration which is needed for doc generation. +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + EnsureDefaults(cfg) + Sanitize(cfg) + return cfg +} + +// DefaultConfig returns the services default config +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9277", + Token: "", + }, + Service: config.Service{ + Name: "antivirus", + }, + Events: config.Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "ocis-cluster", + }, + InfectedFileHandling: "delete", + Scanner: config.Scanner{ + Type: "clamav", + ClamAV: config.ClamAV{ + Socket: "/run/clamav/clamd.ctl", + }, + ICAP: config.ICAP{ + URL: "icap://127.0.0.1:1344", + Service: "avscan", + Timeout: 300, + }, + }, + } +} + +// EnsureDefaults adds default values to the configuration if they are not set yet +func EnsureDefaults(cfg *config.Config) { + if cfg.Log == nil { + cfg.Log = &config.Log{} + } +} + +// Sanitize sanitizes the configuration +func Sanitize(cfg *config.Config) { + +} diff --git a/services/antivirus/pkg/config/parser/parse.go b/services/antivirus/pkg/config/parser/parse.go new file mode 100644 index 000000000..a8ab37073 --- /dev/null +++ b/services/antivirus/pkg/config/parser/parse.go @@ -0,0 +1,38 @@ +package parser + +import ( + "errors" + + ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config/defaults" + + "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" +) + +// ParseConfig loads configuration from known paths. +func ParseConfig(cfg *config.Config) error { + _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) + if err != nil { + return err + } + + defaults.EnsureDefaults(cfg) + + // load all env variables relevant to the config in the current context. + if err := envdecode.Decode(cfg); err != nil { + // no environment variable set for this config is an expected "error" + if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + return err + } + } + + defaults.Sanitize(cfg) + + return Validate(cfg) +} + +// Validate validates our little config +func Validate(cfg *config.Config) error { + return nil +} diff --git a/services/antivirus/pkg/scanners/clamav.go b/services/antivirus/pkg/scanners/clamav.go new file mode 100644 index 000000000..4f5b8bddb --- /dev/null +++ b/services/antivirus/pkg/scanners/clamav.go @@ -0,0 +1,35 @@ +package scanners + +import ( + "io" + "time" + + "github.com/dutchcoders/go-clamd" +) + +// NewClamAV returns an Scanner talking to clamAV via socket +func NewClamAV(socket string) *ClamAV { + return &ClamAV{ + clamd: clamd.NewClamd(socket), + } +} + +// ClamAV is a Scanner based on clamav +type ClamAV struct { + clamd *clamd.Clamd +} + +// Scan to fulfill Scanner interface +func (s ClamAV) Scan(file io.Reader) (ScanResult, error) { + ch, err := s.clamd.ScanStream(file, make(chan bool)) + if err != nil { + return ScanResult{}, err + } + + r := <-ch + return ScanResult{ + Infected: r.Status == clamd.RES_FOUND, + Description: r.Description, + Scantime: time.Now(), + }, nil +} diff --git a/services/antivirus/pkg/scanners/icap.go b/services/antivirus/pkg/scanners/icap.go new file mode 100644 index 000000000..7b426c032 --- /dev/null +++ b/services/antivirus/pkg/scanners/icap.go @@ -0,0 +1,68 @@ +package scanners + +import ( + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "time" + + ic "github.com/egirna/icap-client" +) + +// NewICAP returns a Scanner talking to an ICAP server +func NewICAP(icapURL string, icapService string, timeout time.Duration) (ICAP, error) { + endpoint, err := url.Parse(icapURL) + if err != nil { + return ICAP{}, err + } + + endpoint.Scheme = "icap" + endpoint.Path = icapService + + return ICAP{ + client: &ic.Client{ + Timeout: timeout, + }, + endpoint: endpoint.String(), + }, nil +} + +// ICAP is a Scanner talking to an ICAP server +type ICAP struct { + client *ic.Client + endpoint string +} + +// Scan to fulfill Scanner interface +func (s ICAP) Scan(file io.Reader) (ScanResult, error) { + sr := ScanResult{} + + httpReq, err := http.NewRequest(http.MethodGet, "http://localhost", file) + if err != nil { + return sr, err + } + + req, err := ic.NewRequest(ic.MethodREQMOD, s.endpoint, httpReq, nil) + if err != nil { + return sr, err + } + + resp, err := s.client.Do(req) + if err != nil { + return sr, err + } + + if data, infected := resp.Header["X-Infection-Found"]; infected { + sr.Infected = infected + re := regexp.MustCompile(`Threat=(.*);`) + match := re.FindStringSubmatch(fmt.Sprint(data)) + + if len(match) > 1 { + sr.Description = match[1] + } + } + + return sr, nil +} diff --git a/services/antivirus/pkg/scanners/scanners.go b/services/antivirus/pkg/scanners/scanners.go new file mode 100644 index 000000000..ef2fe4648 --- /dev/null +++ b/services/antivirus/pkg/scanners/scanners.go @@ -0,0 +1,34 @@ +package scanners + +import ( + "fmt" + "io" + "time" + + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" +) + +// ScanResult is the common scan result to all scanners +type ScanResult struct { + Infected bool + Scantime time.Time + Description string +} + +// Scanner is an abstraction for the actual virus scan +type Scanner interface { + Scan(file io.Reader) (ScanResult, error) +} + +// New returns a new scanner from config +func New(c config.Scanner) (Scanner, error) { + switch c.Type { + default: + return nil, fmt.Errorf("unknown av scanner: '%s'", c.Type) + case "clamav": + return NewClamAV(c.ClamAV.Socket), nil + case "icap": + return NewICAP(c.ICAP.URL, c.ICAP.Service, time.Duration(c.ICAP.Timeout)*time.Second) + } + +} diff --git a/services/antivirus/pkg/service/service.go b/services/antivirus/pkg/service/service.go new file mode 100644 index 000000000..a56793c18 --- /dev/null +++ b/services/antivirus/pkg/service/service.go @@ -0,0 +1,205 @@ +package service + +import ( + "bytes" + "context" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cs3org/reva/v2/pkg/bytesize" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/stream" + "github.com/cs3org/reva/v2/pkg/rhttp" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/config" + "github.com/owncloud/ocis/v2/services/antivirus/pkg/scanners" +) + +// Scanner is an abstraction for the actual virus scan +type Scanner interface { + Scan(file io.Reader) (scanners.ScanResult, error) +} + +// NewAntivirus returns a service implementation for Service. +func NewAntivirus(c *config.Config, l log.Logger) (Antivirus, error) { + av := Antivirus{c: c, l: l, client: rhttp.GetHTTPClient(rhttp.Insecure(true))} + + var err error + av.s, err = scanners.New(c.Scanner) + if err != nil { + return av, err + } + + switch o := events.PostprocessingOutcome(c.InfectedFileHandling); o { + case events.PPOutcomeContinue, events.PPOutcomeAbort, events.PPOutcomeDelete: + av.o = o + default: + return av, fmt.Errorf("unknown infected file handling '%s'", o) + } + + if c.MaxScanSize != "" { + b, err := bytesize.Parse(c.MaxScanSize) + if err != nil { + return av, err + } + + av.m = b.Bytes() + } + + return av, nil +} + +// Antivirus defines implements the business logic for Service. +type Antivirus struct { + c *config.Config + l log.Logger + s Scanner + o events.PostprocessingOutcome + m uint64 + + client *http.Client +} + +// Run runs the service +func (av Antivirus) Run() error { + evtsCfg := av.c.Events + + var rootCAPool *x509.CertPool + if evtsCfg.TLSRootCACertificate != "" { + rootCrtFile, err := os.Open(evtsCfg.TLSRootCACertificate) + if err != nil { + return err + } + + var certBytes bytes.Buffer + if _, err := io.Copy(&certBytes, rootCrtFile); err != nil { + return err + } + + rootCAPool = x509.NewCertPool() + rootCAPool.AppendCertsFromPEM(certBytes.Bytes()) + evtsCfg.TLSInsecure = false + } + + stream, err := stream.NatsFromConfig(stream.NatsConfig(av.c.Events)) + if err != nil { + return err + } + + ch, err := events.Consume(stream, "antivirus", events.StartPostprocessingStep{}) + if err != nil { + return err + } + + for e := range ch { + ev := e.Event.(events.StartPostprocessingStep) + if ev.StepToStart != events.PPStepAntivirus { + continue + } + + var errmsg string + res, err := av.process(ev) + if err != nil { + errmsg = err.Error() + } + + outcome := events.PPOutcomeContinue + if res.Infected { + outcome = av.o + } + + av.l.Info().Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Str("virus", res.Description).Str("outcome", string(outcome)).Str("filename", ev.Filename).Str("user", ev.ExecutingUser.GetId().GetOpaqueId()).Bool("infected", res.Infected).Msg("File scanned") + if err := events.Publish(stream, events.PostprocessingStepFinished{ + Outcome: outcome, + UploadID: ev.UploadID, + ExecutingUser: ev.ExecutingUser, + Filename: ev.Filename, + Result: events.VirusscanResult{ + Infected: res.Infected, + Description: res.Description, + Scandate: time.Now(), + ResourceID: ev.ResourceID, + ErrorMsg: errmsg, + }, + }); err != nil { + av.l.Fatal().Err(err).Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Msg("cannot publish events - exiting") + return err + } + } + + return nil +} + +// process the scan +func (av Antivirus) process(ev events.StartPostprocessingStep) (scanners.ScanResult, error) { + if ev.Filesize == 0 || (0 < av.m && av.m < ev.Filesize) { + return scanners.ScanResult{ + Scantime: time.Now(), + }, nil + } + + var err error + var rrc io.ReadCloser + + switch ev.UploadID { + default: + rrc, err = av.downloadViaToken(ev.URL) + case "": + rrc, err = av.downloadViaReva(ev.URL, ev.Token, ev.RevaToken) + } + if err != nil { + av.l.Error().Err(err).Str("uploadid", ev.UploadID).Msg("error downloading file") + return scanners.ScanResult{}, err + } + defer rrc.Close() + + res, err := av.s.Scan(rrc) + if err != nil { + av.l.Error().Err(err).Str("uploadid", ev.UploadID).Msg("error scanning file") + } + + return res, err + +} + +// download will download the file +func (av Antivirus) downloadViaToken(url string) (io.ReadCloser, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + return av.doDownload(req) +} + +// download will download the file +func (av Antivirus) downloadViaReva(url string, dltoken string, revatoken string) (io.ReadCloser, error) { + ctx := ctxpkg.ContextSetToken(context.Background(), revatoken) + + req, err := rhttp.NewRequest(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Reva-Transfer", dltoken) + + return av.doDownload(req) +} + +func (av Antivirus) doDownload(req *http.Request) (io.ReadCloser, error) { + res, err := av.client.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + res.Body.Close() + return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode) + } + + return res.Body, nil +} diff --git a/services/policies/pkg/service/event/service.go b/services/policies/pkg/service/event/service.go index 054a53979..40d455e8b 100644 --- a/services/policies/pkg/service/event/service.go +++ b/services/policies/pkg/service/event/service.go @@ -2,6 +2,7 @@ package eventSVC import ( "context" + "github.com/cs3org/reva/v2/pkg/events" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/policies/pkg/engine" @@ -37,7 +38,7 @@ func (s Service) Run() error { for e := range ch { switch ev := e.Event.(type) { case events.StartPostprocessingStep: - if ev.StepToStart != "policies" { + if ev.StepToStart != events.PPStepPolicies { continue } diff --git a/services/postprocessing/pkg/service/service.go b/services/postprocessing/pkg/service/service.go index f8a8af0f0..020d45441 100644 --- a/services/postprocessing/pkg/service/service.go +++ b/services/postprocessing/pkg/service/service.go @@ -21,7 +21,6 @@ func NewPostprocessingService(stream events.Stream, logger log.Logger, c config. evs, err := events.Consume(stream, "postprocessing", events.BytesReceived{}, events.StartPostprocessingStep{}, - events.VirusscanFinished{}, events.UploadReady{}, events.PostprocessingStepFinished{}, ) From c550390d82169da0566a2ff98c89375a73f070fe Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 16 Mar 2023 15:52:42 +0100 Subject: [PATCH 2/8] service checklist Signed-off-by: jkoberg --- .drone.star | 1 + docs/services/antivirus/_index.md | 17 +++++++++++ docs/services/antivirus/configuration.md | 15 ++++++++++ services/antivirus/README.md | 36 ++++++++++++++++++++++++ services/antivirus/pkg/scanners/icap.go | 1 + 5 files changed, 70 insertions(+) create mode 100644 docs/services/antivirus/_index.md create mode 100644 docs/services/antivirus/configuration.md diff --git a/.drone.star b/.drone.star index 5be5c9f62..43bae8104 100644 --- a/.drone.star +++ b/.drone.star @@ -54,6 +54,7 @@ dirs = { config = { "modules": [ # if you add a module here please also add it to the root level Makefile + "services/antivirus", "services/app-provider", "services/app-registry", "services/audit", diff --git a/docs/services/antivirus/_index.md b/docs/services/antivirus/_index.md new file mode 100644 index 000000000..764dbd11a --- /dev/null +++ b/docs/services/antivirus/_index.md @@ -0,0 +1,17 @@ +--- +title: Antivirus +date: 2023-03-16:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/antivirus +geekdocFilePath: _index.md +geekdocCollapseSection: true +--- + +## Abstract + +wating for readme to be approved + +## Table of Contents + +{{< toc-tree >}} diff --git a/docs/services/antivirus/configuration.md b/docs/services/antivirus/configuration.md new file mode 100644 index 000000000..056336e97 --- /dev/null +++ b/docs/services/antivirus/configuration.md @@ -0,0 +1,15 @@ +--- +title: Service Configuration +date: 2023-03-16T00:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/antivirus +geekdocFilePath: configuration.md +geekdocCollapseSection: true +--- + +## Example YAML Config + +{{< include file="services/_includes/antivirus.yaml" language="yaml" >}} + +{{< include file="services/_includes/antivirus_configvars.md" >}} diff --git a/services/antivirus/README.md b/services/antivirus/README.md index e69de29bb..d8f2e4139 100644 --- a/services/antivirus/README.md +++ b/services/antivirus/README.md @@ -0,0 +1,36 @@ +# Antivirus Service + +The `antivirus` service is responsible for scanning files for viruses + +## Configuration + +### Antivirus Scanner Type + +The antivirus service currently supports `icap` and `clamav` as antivirus scanners. Use `ANTIVIRUS_SCANNER_TYPE` to configure this. +Note that configuration depends heavily on chosen antivirus scanner. See Enviroment Variable descriptions for details. + +### Maximum Scan size + +Since several factors might make need necessary to limit the maximum filesize the `antivirus` service has an option to set a max scan size. +Use `ANTIVIRUS_MAX_SCAN_SIZE` to scan only that amount of bytes of a file. Obviously it is recommended to set this as high as possible, but several factors (scanner type and version, bandwith and performance issues, ...) might force to set this to a certain filesize. + +### Infected File Handling + +The `antivirus` service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` envvar: + - `delete` (default): Infected files will be deleted immediately. Further postprocessing is cancelled. + - `abort`: Infected files will be kept. Further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. (Advanced option) + - `continue`: Infected files will be marked as infected but postprocessing continues normally. Note: Infected Files are not prevented from download. Risk of spreading viruses. (Obviously not recommended) + +## Operation Modes + +The `antivirus` service can scan files during postprocessing. `on demand` scanning will be added in the future. + +### Postprocessing + +Note: Needs to be configured via the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) + +The `antivirus` service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"` + +### On Demand + +On demand scanning is currently not supported diff --git a/services/antivirus/pkg/scanners/icap.go b/services/antivirus/pkg/scanners/icap.go index 7b426c032..c0b2a8745 100644 --- a/services/antivirus/pkg/scanners/icap.go +++ b/services/antivirus/pkg/scanners/icap.go @@ -54,6 +54,7 @@ func (s ICAP) Scan(file io.Reader) (ScanResult, error) { return sr, err } + // TODO: make header configurable. See oc10 documentation: https://doc.owncloud.com/server/10.12/admin_manual/configuration/server/virus-scanner-support.html if data, infected := resp.Header["X-Infection-Found"]; infected { sr.Infected = infected re := regexp.MustCompile(`Threat=(.*);`) From fc4ba499b16a3f0d834714ab0b86bd7c08fe997f Mon Sep 17 00:00:00 2001 From: kobergj Date: Fri, 17 Mar 2023 12:08:18 +0100 Subject: [PATCH 3/8] improve antivirus documentation Co-authored-by: Martin --- docs/services/antivirus/_index.md | 41 ++++++++++++++++++++++-- docs/services/antivirus/configuration.md | 3 -- services/antivirus/README.md | 36 +++++++++++---------- services/antivirus/pkg/config/config.go | 6 ++-- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/docs/services/antivirus/_index.md b/docs/services/antivirus/_index.md index 764dbd11a..55e3cc4de 100644 --- a/docs/services/antivirus/_index.md +++ b/docs/services/antivirus/_index.md @@ -10,8 +10,45 @@ geekdocCollapseSection: true ## Abstract -wating for readme to be approved - ## Table of Contents {{< toc-tree >}} + +## Antivirus Service + +The `antivirus` service is responsible for scanning files for viruses. + +### Configuration + +#### Antivirus Scanner Type + +The antivirus service currently supports [icap](https://tools.ietf.org/html/rfc3507) and [clamav](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. + + - For `icap`, only scanners using the `X-Infection-Found` header are currently supported. + - For `clamav` only local sockets can currently be configured. + +#### Maximum Scan size + +Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. + +#### Infected File Handling + +The antivirus service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` environment variable: + + - `delete`: (default): Infected files will be deleted immediately, further postprocessing is cancelled. + - `abort`: (advanced option): Infected files will be kept, further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. To identify the file for further investigation, the antivirus service logs the abort/infected state including the file ID. The file is located in the `storage/users/uploads` folder of the ocis data directory and persists until it is manually deleted by the admin via the [Manage Unfinished Uploads](https://doc.owncloud.com/ocis/next/deployment/services/s-list/storage-users.html#manage-unfinished-uploads) command. + - `continue`: (obviously not recommended): Infected files will be marked via metadata as infected but postprocessing continues normally. Note: Infected Files are moved to their final destination and therefore not prevented from download which includes the risk of spreading viruses. + +In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent. + +#### Scanner Inaccessability + +In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessability of the scanner used. + +### Operation Modes + +The antivirus service can scan files during `postprocessing`. `on demand` scanning is currently not available and might be added in a future release. + +#### Postprocessing + +The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. diff --git a/docs/services/antivirus/configuration.md b/docs/services/antivirus/configuration.md index 056336e97..c7159a755 100644 --- a/docs/services/antivirus/configuration.md +++ b/docs/services/antivirus/configuration.md @@ -10,6 +10,3 @@ geekdocCollapseSection: true ## Example YAML Config -{{< include file="services/_includes/antivirus.yaml" language="yaml" >}} - -{{< include file="services/_includes/antivirus_configvars.md" >}} diff --git a/services/antivirus/README.md b/services/antivirus/README.md index d8f2e4139..3e9599f32 100644 --- a/services/antivirus/README.md +++ b/services/antivirus/README.md @@ -1,36 +1,38 @@ # Antivirus Service -The `antivirus` service is responsible for scanning files for viruses +The `antivirus` service is responsible for scanning files for viruses. ## Configuration ### Antivirus Scanner Type -The antivirus service currently supports `icap` and `clamav` as antivirus scanners. Use `ANTIVIRUS_SCANNER_TYPE` to configure this. -Note that configuration depends heavily on chosen antivirus scanner. See Enviroment Variable descriptions for details. +The antivirus service currently supports [icap](https://tools.ietf.org/html/rfc3507) and [clamav](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. + + - For `icap`, only scanners using the `X-Infection-Found` header are currently supported. + - For `clamav` only local sockets can currently be configured. ### Maximum Scan size -Since several factors might make need necessary to limit the maximum filesize the `antivirus` service has an option to set a max scan size. -Use `ANTIVIRUS_MAX_SCAN_SIZE` to scan only that amount of bytes of a file. Obviously it is recommended to set this as high as possible, but several factors (scanner type and version, bandwith and performance issues, ...) might force to set this to a certain filesize. +Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. ### Infected File Handling -The `antivirus` service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` envvar: - - `delete` (default): Infected files will be deleted immediately. Further postprocessing is cancelled. - - `abort`: Infected files will be kept. Further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. (Advanced option) - - `continue`: Infected files will be marked as infected but postprocessing continues normally. Note: Infected Files are not prevented from download. Risk of spreading viruses. (Obviously not recommended) +The antivirus service allows three different ways of handling infected files. Those can be set via the `ANTIVIRUS_INFECTED_FILE_HANDLING` environment variable: + + - `delete`: (default): Infected files will be deleted immediately, further postprocessing is cancelled. + - `abort`: (advanced option): Infected files will be kept, further postprocessing is cancelled. Files can be manually retrieved and inspected by an admin. To identify the file for further investigation, the antivirus service logs the abort/infected state including the file ID. The file is located in the `storage/users/uploads` folder of the ocis data directory and persists until it is manually deleted by the admin via the [Manage Unfinished Uploads](https://doc.owncloud.com/ocis/next/deployment/services/s-list/storage-users.html#manage-unfinished-uploads) command. + - `continue`: (obviously not recommended): Infected files will be marked via metadata as infected but postprocessing continues normally. Note: Infected Files are moved to their final destination and therefore not prevented from download which includes the risk of spreading viruses. + +In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent. + +### Scanner Inaccessability + +In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessability of the scanner used. ## Operation Modes -The `antivirus` service can scan files during postprocessing. `on demand` scanning will be added in the future. +The antivirus service can scan files during `postprocessing`. `on demand` scanning is currently not available and might be added in a future release. ### Postprocessing -Note: Needs to be configured via the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) - -The `antivirus` service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"` - -### On Demand - -On demand scanning is currently not supported +The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. diff --git a/services/antivirus/pkg/config/config.go b/services/antivirus/pkg/config/config.go index f351027eb..58c67273b 100644 --- a/services/antivirus/pkg/config/config.go +++ b/services/antivirus/pkg/config/config.go @@ -13,7 +13,7 @@ type Config struct { Service Service `yaml:"-"` - InfectedFileHandling string `yaml:"infected-file-handling" env:"ANTIVIRUS_INFECTED_FILE_HANDLING" desc:"Defines the behaviour when a virus has been found. Options are: 'delete', 'continue' and 'abort '. Delete will delete the file. Continue will mark the file as infected but continues further processing. Abort will keep the file in the upload folder for further admin inspection and will not move it to its target space."` + InfectedFileHandling string `yaml:"infected-file-handling" env:"ANTIVIRUS_INFECTED_FILE_HANDLING" desc:"Defines the behaviour when a virus has been found. Supported options are: 'delete', 'continue' and 'abort '. Delete will delete the file. Continue will mark the file as infected but continues further processing. Abort will keep the file in the uploads folder for further admin inspection and will not move it to its final destination."` Events Events Scanner Scanner MaxScanSize string `yaml:"max-scan-size" env:"ANTIVIRUS_MAX_SCAN_SIZE" desc:"The maximum scan size the virusscanner can handle. Only that much bytes of a file will be scanned. 0 means unlimited and is the default. Usable common abbreviations: [KB, KiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB."` @@ -53,7 +53,7 @@ type Events struct { // Scanner provides configuration options for the antivirusscanner type Scanner struct { - Type string `yaml:"type" env:"ANTIVIRUS_SCANNER_TYPE" desc:"The scanner to use. Must be one of: clamav, icap"` + Type string `yaml:"type" env:"ANTIVIRUS_SCANNER_TYPE" desc:"The antivirus scanner to use. Supported values are 'clamav' and 'icap'."` ClamAV ClamAV // only if Type == clamav ICAP ICAP // only if Type == icap @@ -68,5 +68,5 @@ type ClamAV struct { type ICAP struct { Timeout int64 `yaml:"timeout" env:"ANTIVIRUS_ICAP_TIMEOUT" desc:"Timeout for the ICAP client."` URL string `yaml:"url" env:"ANTIVIRUS_ICAP_URL" desc:"URL of the ICAP server."` - Service string `yaml:"service" env:"ANTIVIRUS_ICAP_SERVICE" desc:"Name of the ICAP server."` + Service string `yaml:"service" env:"ANTIVIRUS_ICAP_SERVICE" desc:"The name of the ICAP service."` } From 642d8f0028c5ff765a95b4a2796c0f9db656f656 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 17 Mar 2023 12:45:10 +0100 Subject: [PATCH 4/8] notification for virusscan Signed-off-by: jkoberg --- services/userlog/pkg/command/server.go | 3 ++ services/userlog/pkg/service/conversion.go | 36 ++++++++++++++++++++-- services/userlog/pkg/service/service.go | 12 ++++++++ services/userlog/pkg/service/templates.go | 5 +++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/services/userlog/pkg/command/server.go b/services/userlog/pkg/command/server.go index 6f09a5371..5501bcb79 100644 --- a/services/userlog/pkg/command/server.go +++ b/services/userlog/pkg/command/server.go @@ -25,6 +25,9 @@ import ( // all events we care about var _registeredEvents = []events.Unmarshaller{ + // file related + events.PostprocessingStepFinished{}, + // space related events.SpaceDisabled{}, events.SpaceDeleted{}, diff --git a/services/userlog/pkg/service/conversion.go b/services/userlog/pkg/service/conversion.go index 16788143e..12a7b264f 100644 --- a/services/userlog/pkg/service/conversion.go +++ b/services/userlog/pkg/service/conversion.go @@ -25,8 +25,9 @@ import ( var _translationFS embed.FS var ( - _resourceTypeSpace = "storagespace" - _resourceTypeShare = "share" + _resourceTypeResource = "resource" + _resourceTypeSpace = "storagespace" + _resourceTypeShare = "share" _domain = "userlog" ) @@ -96,6 +97,13 @@ func (c *Converter) ConvertEvent(event *ehmsg.Event) (OC10Notification, error) { switch ev := einterface.(type) { default: return OC10Notification{}, errors.New("unknown event type") + // file related + case events.PostprocessingStepFinished: + if ev.FinishedStep != events.PPStepAntivirus { + return OC10Notification{}, errors.New("unknown event type") + } + res := ev.Result.(events.VirusscanResult) + return c.virusMessage(event.Id, VirusFound, ev.ExecutingUser, res.ResourceID, ev.Filename, res.Description, res.Scandate) // space related case events.SpaceDisabled: return c.spaceMessage(event.Id, SpaceDisabled, ev.Executant, ev.ID.GetOpaqueId(), ev.Timestamp) @@ -227,6 +235,30 @@ func (c *Converter) shareMessage(eventid string, nt NotificationTemplate, execut }, nil } +func (c *Converter) virusMessage(eventid string, nt NotificationTemplate, executant *user.User, rid *storageprovider.ResourceId, filename string, virus string, ts time.Time) (OC10Notification, error) { + subj, subjraw, msg, msgraw, err := composeMessage(nt, c.locale, c.translationPath, map[string]interface{}{ + "resourcename": filename, + "virusdescription": virus, + }) + if err != nil { + return OC10Notification{}, err + } + + return OC10Notification{ + EventID: eventid, + Service: c.serviceName, + UserName: executant.GetUsername(), + Timestamp: ts.Format(time.RFC3339Nano), + ResourceID: storagespace.FormatResourceID(*rid), + ResourceType: _resourceTypeResource, + Subject: subj, + SubjectRaw: subjraw, + Message: msg, + MessageRaw: msgraw, + MessageDetails: generateDetails(nil, nil, nil, nil), + }, nil +} + func (c *Converter) authenticate(usr *user.User) (context.Context, error) { if ctx, ok := c.contexts[usr.GetId().GetOpaqueId()]; ok { return ctx, nil diff --git a/services/userlog/pkg/service/service.go b/services/userlog/pkg/service/service.go index 2c6ee2837..9c731080f 100644 --- a/services/userlog/pkg/service/service.go +++ b/services/userlog/pkg/service/service.go @@ -90,6 +90,18 @@ func (ul *UserlogService) MemorizeEvents(ch <-chan events.Event) { switch e := event.Event.(type) { default: err = errors.New("unhandled event") + // file related + case events.PostprocessingStepFinished: + if e.FinishedStep != events.PPStepAntivirus { + continue + } + result := e.Result.(events.VirusscanResult) + if !result.Infected { + continue + } + + // TODO: should space mangers also be informed? + users = append(users, e.ExecutingUser.GetId().GetOpaqueId()) // space related // TODO: how to find spaceadmins? case events.SpaceDisabled: users, err = ul.findSpaceMembers(ul.impersonate(e.Executant), e.ID.GetOpaqueId(), viewer) diff --git a/services/userlog/pkg/service/templates.go b/services/userlog/pkg/service/templates.go index 412b6ee4e..b80d59159 100644 --- a/services/userlog/pkg/service/templates.go +++ b/services/userlog/pkg/service/templates.go @@ -5,6 +5,10 @@ func Template(s string) string { return s } // the available templates var ( + VirusFound = NotificationTemplate{ + Subject: Template("Virus found"), + Message: Template("Virus found in {resource}. Upload not possible. Virus: {virus}"), + } SpaceShared = NotificationTemplate{ Subject: Template("Space shared"), Message: Template("{user} added you to Space {space}"), @@ -51,6 +55,7 @@ var _placeholders = map[string]string{ "{user}": "{{ .username }}", "{space}": "{{ .spacename }}", "{resource}": "{{ .resourcename }}", + "{virus}": "{{ .virusdescription }}", } // NotificationTemplate is the data structure for the notifications From d8e31feccf5c3b7c2c6d1b7e21076c25045d3a93 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 16 Mar 2023 16:06:42 +0100 Subject: [PATCH 5/8] bump reva Signed-off-by: jkoberg --- go.mod | 5 ++++- go.sum | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bf663590e..70d1f698a 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,10 @@ require ( github.com/blevesearch/bleve/v2 v2.3.6 github.com/coreos/go-oidc/v3 v3.4.0 github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 - github.com/cs3org/reva/v2 v2.12.1-0.20230316154706-3c11349102b7 + github.com/cs3org/reva/v2 v2.12.1-0.20230321085342-13f8b522b0d8 github.com/disintegration/imaging v1.6.2 + github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e + github.com/egirna/icap-client v0.1.1 github.com/gabriel-vasile/mimetype v1.4.1 github.com/ggwhite/go-masker v1.0.9 github.com/go-chi/chi/v5 v5.0.8 @@ -163,6 +165,7 @@ require ( github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/egirna/icap v0.0.0-20181108071049-d5ee18bd70bc // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/emvi/iso-639-1 v1.0.1 // indirect github.com/evanphx/json-patch/v5 v5.5.0 // indirect diff --git a/go.sum b/go.sum index e5043845a..45eef5868 100644 --- a/go.sum +++ b/go.sum @@ -623,8 +623,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.10 h1:Rjs6x4s/aQFXiaPjw3uhB4VdxRqoxHXOJrrj4BsMn9o= github.com/crewjam/saml v0.4.10/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4= -github.com/cs3org/reva/v2 v2.12.1-0.20230316154706-3c11349102b7 h1:5VPGJ4gTxSvbSmbI5H+n+X6PD8JsoOxVQynW0ddZ9so= -github.com/cs3org/reva/v2 v2.12.1-0.20230316154706-3c11349102b7/go.mod h1:FNAYs5H3xs8v0OFmNgZtiMAzIMXd/6TJmO0uZuNn8pQ= +github.com/cs3org/reva/v2 v2.12.1-0.20230321085342-13f8b522b0d8 h1:1J0z1BZTppSILfrVveamCIdMm8AZMKUcY5gQHKke16Y= +github.com/cs3org/reva/v2 v2.12.1-0.20230321085342-13f8b522b0d8/go.mod h1:FNAYs5H3xs8v0OFmNgZtiMAzIMXd/6TJmO0uZuNn8pQ= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -658,9 +658,15 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e h1:rcHHSQqzCgvlwP0I/fQ8rQMn/MpHE5gWSLdtpxtP6KQ= +github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e/go.mod h1:Byz7q8MSzSPkouskHJhX0er2mZY/m0Vj5bMeMCkkyY4= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/egirna/icap v0.0.0-20181108071049-d5ee18bd70bc h1:6IxmRbXV8WXVkcYcTzkU219A3UZeNMX/e6X2sve1wXA= +github.com/egirna/icap v0.0.0-20181108071049-d5ee18bd70bc/go.mod h1:FdVN2WHg7zOHhJ7kZQdDorfFhIfqZaHttjAzDDvAXHE= +github.com/egirna/icap-client v0.1.1 h1:UURZRA7+36bBmMgJZHB+W4d3hfp20pDUA/QL794C0ck= +github.com/egirna/icap-client v0.1.1/go.mod h1:6yHhnak1cKRyhDoRxnzlRKJbOTXPgh9Oe3tOs7Sq3vw= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= From 6c00708dbccedab111de637abd5a1a31ffe372ea Mon Sep 17 00:00:00 2001 From: kobergj Date: Wed, 22 Mar 2023 13:45:53 +0100 Subject: [PATCH 6/8] improve antivirus documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörn Friedrich Dreyer --- docs/services/antivirus/_index.md | 10 ++++----- ocis/pkg/command/antivirus.go | 1 - services/antivirus/README.md | 10 ++++----- services/antivirus/pkg/command/server.go | 2 +- services/antivirus/pkg/config/config.go | 28 ++++++++++++------------ 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/docs/services/antivirus/_index.md b/docs/services/antivirus/_index.md index 55e3cc4de..e29d0be6e 100644 --- a/docs/services/antivirus/_index.md +++ b/docs/services/antivirus/_index.md @@ -22,14 +22,14 @@ The `antivirus` service is responsible for scanning files for viruses. #### Antivirus Scanner Type -The antivirus service currently supports [icap](https://tools.ietf.org/html/rfc3507) and [clamav](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. +The antivirus service currently supports [ICAP](https://tools.ietf.org/html/rfc3507) and [ClamAV](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. - For `icap`, only scanners using the `X-Infection-Found` header are currently supported. - For `clamav` only local sockets can currently be configured. #### Maximum Scan size -Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. +Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously, it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. #### Infected File Handling @@ -41,9 +41,9 @@ The antivirus service allows three different ways of handling infected files. Th In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent. -#### Scanner Inaccessability +#### Scanner Inaccessibility -In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessability of the scanner used. +In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessibility of the scanner used. ### Operation Modes @@ -51,4 +51,4 @@ The antivirus service can scan files during `postprocessing`. `on demand` scanni #### Postprocessing -The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. +The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `virusscan`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. diff --git a/ocis/pkg/command/antivirus.go b/ocis/pkg/command/antivirus.go index 5671930a9..24cd36efb 100644 --- a/ocis/pkg/command/antivirus.go +++ b/ocis/pkg/command/antivirus.go @@ -18,7 +18,6 @@ func AntivirusCommand(cfg *config.Config) *cli.Command { Category: "services", Before: func(c *cli.Context) error { configlog.Error(parser.ParseConfig(cfg, true)) - //cfg.Antivirus.Commons = cfg.Commons return nil }, Subcommands: command.GetCommands(cfg.Antivirus), diff --git a/services/antivirus/README.md b/services/antivirus/README.md index 3e9599f32..2ed95fc63 100644 --- a/services/antivirus/README.md +++ b/services/antivirus/README.md @@ -6,14 +6,14 @@ The `antivirus` service is responsible for scanning files for viruses. ### Antivirus Scanner Type -The antivirus service currently supports [icap](https://tools.ietf.org/html/rfc3507) and [clamav](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. +The antivirus service currently supports [ICAP](https://tools.ietf.org/html/rfc3507) and [ClamAV](http://www.clamav.net/index.html) as antivirus scanners. The `ANTIVIRUS_SCANNER_TYPE` environment variable is used to select the scanner. The detailed configuration for each scanner heavily depends on the scanner type selected. See the environment variables for more details. - For `icap`, only scanners using the `X-Infection-Found` header are currently supported. - For `clamav` only local sockets can currently be configured. ### Maximum Scan size -Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. +Several factors can make it necessary to limit the maximum filesize the antivirus service will use for scanning. Use the `ANTIVIRUS_MAX_SCAN_SIZE` environment variable to scan only a given amount of bytes. Obviously, it is recommended to scan the whole file, but several factors like scanner type and version, bandwith, performance issues, etc. might make a limit necessary. ### Infected File Handling @@ -25,9 +25,9 @@ The antivirus service allows three different ways of handling infected files. Th In all cases, a log entry is added declaring the infection and handling method and a notification via the `userlog` service sent. -### Scanner Inaccessability +### Scanner Inaccessibility -In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessability of the scanner used. +In case a scanner is not accessible by the antivirus service like a network outage, service outage or hardware outage, the antivirus service uses the `abort` case for further processing, independent of the actual setting made. In any case, an error is logged noting the inaccessibility of the scanner used. ## Operation Modes @@ -35,4 +35,4 @@ The antivirus service can scan files during `postprocessing`. `on demand` scanni ### Postprocessing -The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `"virusscan"`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. +The antivirus service will scan files during postprocessing. It listens for a postprocessing step called `virusscan`. This step can be added in the environment variable `POSTPROCESSING_STEPS`. Read the documentation of the [postprocessing service](https://github.com/owncloud/ocis/tree/master/services/postprocessing) for more details. diff --git a/services/antivirus/pkg/command/server.go b/services/antivirus/pkg/command/server.go index d1de27fbd..c256ddb7d 100644 --- a/services/antivirus/pkg/command/server.go +++ b/services/antivirus/pkg/command/server.go @@ -21,7 +21,7 @@ import ( func Server(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "server", - Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", "authz"), + Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name), Category: "server", Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) diff --git a/services/antivirus/pkg/config/config.go b/services/antivirus/pkg/config/config.go index 58c67273b..acf4cc19c 100644 --- a/services/antivirus/pkg/config/config.go +++ b/services/antivirus/pkg/config/config.go @@ -16,7 +16,7 @@ type Config struct { InfectedFileHandling string `yaml:"infected-file-handling" env:"ANTIVIRUS_INFECTED_FILE_HANDLING" desc:"Defines the behaviour when a virus has been found. Supported options are: 'delete', 'continue' and 'abort '. Delete will delete the file. Continue will mark the file as infected but continues further processing. Abort will keep the file in the uploads folder for further admin inspection and will not move it to its final destination."` Events Events Scanner Scanner - MaxScanSize string `yaml:"max-scan-size" env:"ANTIVIRUS_MAX_SCAN_SIZE" desc:"The maximum scan size the virusscanner can handle. Only that much bytes of a file will be scanned. 0 means unlimited and is the default. Usable common abbreviations: [KB, KiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB."` + MaxScanSize string `yaml:"max-scan-size" env:"ANTIVIRUS_MAX_SCAN_SIZE" desc:"The maximum scan size the virusscanner can handle. Only this many bytes of a file will be scanned. 0 means unlimited and is the default. Usable common abbreviations: [KB, KiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB."` Context context.Context `yaml:"-" json:"-"` } @@ -28,27 +28,27 @@ type Service struct { // Log defines the available log configuration. type Log struct { - Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;POLICIES_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."` - Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;POLICIES_LOG_PRETTY" desc:"Activates pretty log output."` - Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;POLICIES_LOG_COLOR" desc:"Activates colorized log output."` - File string `mapstructure:"file" env:"OCIS_LOG_FILE;POLICIES_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;ANTIVIRUS_LOG_LEVEL" desc:"The log level. Valid values are: \"panic\", \"fatal\", \"error\", \"warn\", \"info\", \"debug\", \"trace\"."` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;ANTIVIRUS_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;ANTIVIRUS_LOG_COLOR" desc:"Activates colorized log output."` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;ANTIVIRUS_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` } // Debug defines the available debug configuration. type Debug struct { - Addr string `yaml:"addr" env:"POLICIES_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` - Token string `yaml:"token" env:"POLICIES_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."` - Pprof bool `yaml:"pprof" env:"POLICIES_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."` - Zpages bool `yaml:"zpages" env:"POLICIES_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` + Addr string `yaml:"addr" env:"ANTIVIRUS_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` + Token string `yaml:"token" env:"ANTIVIRUS_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."` + Pprof bool `yaml:"pprof" env:"ANTIVIRUS_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."` + Zpages bool `yaml:"zpages" env:"ANTIVIRUS_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` } // Events combines the configuration options for the event bus. type Events struct { - Endpoint string `yaml:"endpoint" env:"USERLOG_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` - Cluster string `yaml:"cluster" env:"USERLOG_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` - TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;USERLOG_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."` - TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"USERLOG_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."` - EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;USERLOG_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."` + Endpoint string `yaml:"endpoint" env:"ANTIVIRUS_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` + Cluster string `yaml:"cluster" env:"ANTIVIRUS_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` + TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;ANTIVIRUS_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."` + TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"ANTIVIRUS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided ANTIVIRUS_EVENTS_TLS_INSECURE will be seen as false."` + EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;ANTIVIRUS_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services."` } // Scanner provides configuration options for the antivirusscanner From 86980441fea223c667fb3df5ed0b9f0dfb8e8268 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 22 Mar 2023 13:59:54 +0100 Subject: [PATCH 7/8] move debughandlers to their own package Signed-off-by: jkoberg --- ocis-pkg/handlers/debughandlers.go | 34 ++++++++++++++++++++++ services/antivirus/pkg/command/server.go | 33 ++------------------- services/userlog/pkg/service/conversion.go | 5 ++-- 3 files changed, 40 insertions(+), 32 deletions(-) create mode 100644 ocis-pkg/handlers/debughandlers.go diff --git a/ocis-pkg/handlers/debughandlers.go b/ocis-pkg/handlers/debughandlers.go new file mode 100644 index 000000000..0ac3fed39 --- /dev/null +++ b/ocis-pkg/handlers/debughandlers.go @@ -0,0 +1,34 @@ +package handlers + +import ( + "io" + "net/http" +) + +// Health can be used for a health endpoint +func Health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } +} + +// Ready can be used as a ready endpoint +func Ready(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } +} diff --git a/services/antivirus/pkg/command/server.go b/services/antivirus/pkg/command/server.go index c256ddb7d..257ab6c2b 100644 --- a/services/antivirus/pkg/command/server.go +++ b/services/antivirus/pkg/command/server.go @@ -3,11 +3,10 @@ package command import ( "context" "fmt" - "io" - "net/http" "github.com/oklog/run" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/handlers" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/service/debug" "github.com/owncloud/ocis/v2/ocis-pkg/version" @@ -65,34 +64,8 @@ func Server(cfg *config.Config) *cli.Command { debug.Token(cfg.Debug.Token), debug.Pprof(cfg.Debug.Pprof), debug.Zpages(cfg.Debug.Zpages), - debug.Health( - func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - - // TODO: check if services are up and running - - _, err := io.WriteString(w, http.StatusText(http.StatusOK)) - // io.WriteString should not fail but if it does we want to know. - if err != nil { - panic(err) - } - }, - ), - debug.Ready( - func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - - // TODO: check if services are up and running - - _, err := io.WriteString(w, http.StatusText(http.StatusOK)) - // io.WriteString should not fail but if it does we want to know. - if err != nil { - panic(err) - } - }, - ), + debug.Health(handlers.Health), + debug.Ready(handlers.Ready), ) gr.Add(server.ListenAndServe, func(_ error) { diff --git a/services/userlog/pkg/service/conversion.go b/services/userlog/pkg/service/conversion.go index 12a7b264f..90e9579f5 100644 --- a/services/userlog/pkg/service/conversion.go +++ b/services/userlog/pkg/service/conversion.go @@ -5,6 +5,7 @@ import ( "context" "embed" "errors" + "fmt" "io/fs" "strings" "text/template" @@ -96,11 +97,11 @@ func (c *Converter) ConvertEvent(event *ehmsg.Event) (OC10Notification, error) { switch ev := einterface.(type) { default: - return OC10Notification{}, errors.New("unknown event type") + return OC10Notification{}, fmt.Errorf("unknown event type: %T", ev) // file related case events.PostprocessingStepFinished: if ev.FinishedStep != events.PPStepAntivirus { - return OC10Notification{}, errors.New("unknown event type") + return OC10Notification{}, fmt.Errorf("unknown event type: %T", ev) } res := ev.Result.(events.VirusscanResult) return c.virusMessage(event.Id, VirusFound, ev.ExecutingUser, res.ResourceID, ev.Filename, res.Description, res.Scandate) From 81d15217512ede888c41e0cd1e8924d94808b747 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 22 Mar 2023 17:34:24 +0100 Subject: [PATCH 8/8] abort processing when virus scan errord Signed-off-by: jkoberg --- services/antivirus/pkg/service/service.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/antivirus/pkg/service/service.go b/services/antivirus/pkg/service/service.go index a56793c18..961273802 100644 --- a/services/antivirus/pkg/service/service.go +++ b/services/antivirus/pkg/service/service.go @@ -108,9 +108,14 @@ func (av Antivirus) Run() error { errmsg = err.Error() } - outcome := events.PPOutcomeContinue - if res.Infected { + var outcome events.PostprocessingOutcome + switch { + case res.Infected: outcome = av.o + case !res.Infected && err == nil: + outcome = events.PPOutcomeContinue + default: + outcome = events.PPOutcomeAbort } av.l.Info().Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Str("virus", res.Description).Str("outcome", string(outcome)).Str("filename", ev.Filename).Str("user", ev.ExecutingUser.GetId().GetOpaqueId()).Bool("infected", res.Infected).Msg("File scanned")