Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,54 @@
package command
import (
"fmt"
"net/http"
"github.com/qsfera/server/pkg/log"
"github.com/spf13/cobra"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/services/antivirus/pkg/config"
"github.com/qsfera/server/services/antivirus/pkg/config/parser"
)
// Health is the entrypoint for the health command.
func Health(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "health",
Short: "check health status",
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnError(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
resp, err := http.Get(
fmt.Sprintf(
"http://%s/healthz",
cfg.Debug.Addr,
),
)
if err != nil {
logger.Fatal().
Err(err).
Msg("Failed to request health check")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
logger.Fatal().
Int("code", resp.StatusCode).
Msg("Health seems to be in bad state")
}
logger.Debug().
Int("code", resp.StatusCode).
Msg("Health got a good state")
return nil
},
}
}
@@ -0,0 +1,29 @@
package command
import (
"os"
"github.com/qsfera/server/pkg/clihelper"
"github.com/qsfera/server/services/antivirus/pkg/config"
"github.com/spf13/cobra"
)
// GetCommands provides all commands for this service
func GetCommands(cfg *config.Config) []*cobra.Command {
return []*cobra.Command{
Server(cfg),
Health(cfg),
Version(cfg),
}
}
// Execute is the entry point for the antivirus command.
func Execute(cfg *config.Config) error {
app := clihelper.DefaultApp(&cobra.Command{
Use: "antivirus",
Short: "Antivirus service for КуСфера",
})
app.AddCommand(GetCommands(cfg)...)
app.SetArgs(os.Args[1:])
return app.ExecuteContext(cfg.Context)
}
@@ -0,0 +1,82 @@
package command
import (
"context"
"fmt"
"os"
"os/signal"
"github.com/qsfera/server/pkg/config/configlog"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/pkg/runner"
"github.com/qsfera/server/pkg/tracing"
"github.com/qsfera/server/services/antivirus/pkg/config"
"github.com/qsfera/server/services/antivirus/pkg/config/parser"
"github.com/qsfera/server/services/antivirus/pkg/server/debug"
"github.com/qsfera/server/services/antivirus/pkg/service"
"github.com/spf13/cobra"
)
// Server is the entrypoint for the server command.
func Server(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "server",
Short: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name),
PreRunE: func(cmd *cobra.Command, args []string) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
RunE: func(cmd *cobra.Command, args []string) error {
var cancel context.CancelFunc
if cfg.Context == nil {
cfg.Context, cancel = signal.NotifyContext(context.Background(), runner.StopSignals...)
defer cancel()
}
ctx := cfg.Context
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
traceProvider, err := tracing.GetTraceProvider(cmd.Context(), cfg.Commons.TracesExporter, cfg.Service.Name)
if err != nil {
return err
}
gr := runner.NewGroup()
{
svc, err := service.NewAntivirus(cfg, logger, traceProvider)
if err != nil {
fmt.Errorf("failed to initialize antivirus service: %v", err)
os.Exit(1)
}
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
return svc.Run()
}, func() {
svc.Close()
}))
}
{
debugServer, err := debug.Server(
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
)
if err != nil {
logger.Info().Err(err).Str("server", "debug").Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGolangHttpServerRunner(cfg.Service.Name+".debug", debugServer))
}
grResults := gr.Run(ctx)
// return the first non-nil error found in the results
for _, grResult := range grResults {
if grResult.RunnerError != nil {
return grResult.RunnerError
}
}
return nil
},
}
}
@@ -0,0 +1,25 @@
package command
import (
"fmt"
"github.com/qsfera/server/pkg/version"
"github.com/spf13/cobra"
"github.com/qsfera/server/services/antivirus/pkg/config"
)
// Version prints the service versions of all running instances.
func Version(cfg *config.Config) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "print the version of this binary and the running service instances",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Compiled: %s\n", version.Compiled())
fmt.Println("")
return nil
},
}
}
@@ -0,0 +1,95 @@
package config
import (
"context"
"time"
"github.com/qsfera/server/pkg/shared"
)
// ScannerType gives info which scanner is used
type ScannerType string
const (
// ScannerTypeClamAV defines that clamav is used
ScannerTypeClamAV ScannerType = "clamav"
// ScannerTypeICap defines that icap is used
ScannerTypeICap ScannerType = "icap"
)
// MaxScanSizeMode defines the mode of handling files that exceed the maximum scan size
type MaxScanSizeMode string
const (
// MaxScanSizeModeSkip defines that files that are bigger than the max scan size will be skipped
MaxScanSizeModeSkip MaxScanSizeMode = "skip"
// MaxScanSizeModePartial defines that only the file up to the max size will be used
MaxScanSizeModePartial MaxScanSizeMode = "partial"
)
// Config combines all available configuration parts.
type Config struct {
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
File string
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;ANTIVIRUS_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
Debug Debug `yaml:"debug" mask:"struct"`
Service Service `yaml:"-"`
InfectedFileHandling string `yaml:"infected-file-handling" env:"ANTIVIRUS_INFECTED_FILE_HANDLING" desc:"Defines the behaviour when a virus has been found. Supported options are: 'delete', 'continue' and 'abort '. Delete will delete the file. Continue will mark the file as infected but continues further processing. Abort will keep the file in the uploads folder for further admin inspection and will not move it to its final destination." introductionVersion:"1.0.0"`
Events Events
Workers int `yaml:"workers" env:"ANTIVIRUS_WORKERS" desc:"The number of concurrent go routines that fetch events from the event queue." introductionVersion:"1.0.0"`
Scanner Scanner
MaxScanSize string `yaml:"max-scan-size" env:"ANTIVIRUS_MAX_SCAN_SIZE" desc:"The maximum scan size the virus scanner can handle.0 means unlimited. Usable common abbreviations: [KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB, EB, EiB], example: 2GB." introductionVersion:"1.0.0"`
MaxScanSizeMode MaxScanSizeMode `yaml:"max-scan-size-mode" env:"ANTIVIRUS_MAX_SCAN_SIZE_MODE" desc:"Defines the mode of handling files that exceed the maximum scan size. Supported options are: 'skip', which skips files that are bigger than the max scan size, and 'truncate' (default), which only uses the file up to the max size." introductionVersion:"2.1.0"`
Context context.Context `json:"-" yaml:"-"`
DebugScanOutcome string `yaml:"-" env:"ANTIVIRUS_DEBUG_SCAN_OUTCOME" desc:"A predefined outcome for virus scanning, FOR DEBUG PURPOSES ONLY! (example values: 'found,infected')" introductionVersion:"1.0.0"`
}
// Service defines the available service configuration.
type Service struct {
Name string `yaml:"-"`
}
// Debug defines the available debug configuration.
type Debug struct {
Addr string `yaml:"addr" env:"ANTIVIRUS_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
Token string `yaml:"token" env:"ANTIVIRUS_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
Pprof bool `yaml:"pprof" env:"ANTIVIRUS_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
Zpages bool `yaml:"zpages" env:"ANTIVIRUS_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
}
// Events combines the configuration options for the event bus.
type Events struct {
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;ANTIVIRUS_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;ANTIVIRUS_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;ANTIVIRUS_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;ANTIVIRUS_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided ANTIVIRUS_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"`
EnableTLS bool `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;ANTIVIRUS_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
AuthUsername string `yaml:"username" env:"OC_EVENTS_AUTH_USERNAME;ANTIVIRUS_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
AuthPassword string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;ANTIVIRUS_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
}
// Scanner provides configuration options for the virus scanner
type Scanner struct {
Type ScannerType `yaml:"type" env:"ANTIVIRUS_SCANNER_TYPE" desc:"The antivirus scanner to use. Supported values are 'clamav' and 'icap'." introductionVersion:"1.0.0"`
ClamAV ClamAV // only if Type == clamav
ICAP ICAP // only if Type == icap
}
// ClamAV provides configuration option for clamav
type ClamAV struct {
Socket string `yaml:"socket" env:"ANTIVIRUS_CLAMAV_SOCKET" desc:"The socket clamav is running on. Note the default value is an example which needs adaption according your OS." introductionVersion:"1.0.0"`
Timeout time.Duration `yaml:"scan_timeout" env:"ANTIVIRUS_CLAMAV_SCAN_TIMEOUT" desc:"Scan timeout for the ClamAV client. Defaults to '5m' (5 minutes). See the Environment Variable Types description for more details." introductionVersion:"2.1.0"`
}
// ICAP provides configuration options for icap
type ICAP struct {
Timeout time.Duration `yaml:"scan_timeout" env:"ANTIVIRUS_ICAP_SCAN_TIMEOUT" desc:"Scan timeout for the ICAP client. Defaults to '5m' (5 minutes). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
URL string `yaml:"url" env:"ANTIVIRUS_ICAP_URL" desc:"URL of the ICAP server." introductionVersion:"1.0.0"`
Service string `yaml:"service" env:"ANTIVIRUS_ICAP_SERVICE" desc:"The name of the ICAP service." introductionVersion:"1.0.0"`
}
@@ -0,0 +1,66 @@
package defaults
import (
"time"
"github.com/qsfera/server/services/antivirus/pkg/config"
)
// FullDefaultConfig returns a fully initialized default configuration which is needed for doc generation.
func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()
EnsureDefaults(cfg)
Sanitize(cfg)
return cfg
}
// DefaultConfig returns the services default config
func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9277",
Token: "",
},
Service: config.Service{
Name: "antivirus",
},
Events: config.Events{
Endpoint: "127.0.0.1:9233",
Cluster: "qsfera-cluster",
},
Workers: 10,
InfectedFileHandling: "delete",
// defaults from clamav sample conf: MaxScanSize=400M, MaxFileSize=100M, StreamMaxLength=100M
// https://github.com/Cisco-Talos/clamav/blob/main/etc/clamd.conf.sample
MaxScanSize: "100MB",
MaxScanSizeMode: config.MaxScanSizeModePartial,
Scanner: config.Scanner{
Type: config.ScannerTypeClamAV,
ClamAV: config.ClamAV{
Socket: "/run/clamav/clamd.ctl",
Timeout: 5 * time.Minute,
},
ICAP: config.ICAP{
URL: "icap://127.0.0.1:1344",
Service: "avscan",
Timeout: 5 * time.Minute,
},
},
}
}
// EnsureDefaults adds default values to the configuration if they are not set yet
func EnsureDefaults(cfg *config.Config) {
if cfg.LogLevel == "" {
cfg.LogLevel = "error"
}
}
// Sanitize sanitizes the configuration
func Sanitize(cfg *config.Config) {
defaultConfig := DefaultConfig()
if cfg.MaxScanSize == "" {
cfg.MaxScanSize = defaultConfig.MaxScanSize
}
}
@@ -0,0 +1,38 @@
package parser
import (
"errors"
occfg "github.com/qsfera/server/pkg/config"
"github.com/qsfera/server/services/antivirus/pkg/config"
"github.com/qsfera/server/services/antivirus/pkg/config/defaults"
"github.com/qsfera/server/pkg/config/envdecode"
)
// ParseConfig loads configuration from known paths.
func ParseConfig(cfg *config.Config) error {
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
if err != nil {
return err
}
defaults.EnsureDefaults(cfg)
// load all env variables relevant to the config in the current context.
if err := envdecode.Decode(cfg); err != nil {
// no environment variable set for this config is an expected "error"
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
return err
}
}
defaults.Sanitize(cfg)
return Validate(cfg)
}
// Validate validates our little config
func Validate(cfg *config.Config) error {
return nil
}
@@ -0,0 +1,51 @@
package scanners
import (
"fmt"
"time"
"github.com/dutchcoders/go-clamd"
)
// NewClamAV returns a Scanner talking to clamAV via socket
func NewClamAV(socket string, timeout time.Duration) (*ClamAV, error) {
c := clamd.NewClamd(socket)
if err := c.Ping(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrScannerNotReachable, err)
}
return &ClamAV{
clamd: clamd.NewClamd(socket),
timeout: timeout,
}, nil
}
// ClamAV is a Scanner based on clamav
type ClamAV struct {
clamd *clamd.Clamd
timeout time.Duration
}
// Scan to fulfill Scanner interface
func (s ClamAV) Scan(in Input) (Result, error) {
abort := make(chan bool, 1)
defer close(abort)
ch, err := s.clamd.ScanStream(in.Body, abort)
if err != nil {
return Result{}, err
}
select {
case <-time.After(s.timeout):
abort <- true
return Result{}, fmt.Errorf("%w: %s", ErrScanTimeout, in.Url)
case s := <-ch:
return Result{
Infected: s.Status == clamd.RES_FOUND,
Description: s.Description,
ScanTime: time.Now(),
}, nil
}
}
@@ -0,0 +1,120 @@
package scanners_test
import (
"context"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/qsfera/server/services/antivirus/pkg/scanners"
)
func newUnixListener(t testing.TB, lc net.ListenConfig, v ...string) net.Listener {
d, err := os.MkdirTemp("", "")
assert.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, os.RemoveAll(d))
})
nl, err := lc.Listen(context.Background(), "unix", filepath.Join(d, "sock"))
require.NoError(t, err)
go func() {
i := 0
for {
if len(v) == i {
break
}
conn, err := nl.Accept()
require.NoError(t, err)
time.Sleep(100 * time.Millisecond)
_, err = conn.Write([]byte(v[i]))
require.NoError(t, err)
require.NoError(t, conn.Close())
i++
}
}()
return nl
}
func TestNewClamAV(t *testing.T) {
t.Run("returns a scanner", func(t *testing.T) {
ul := newUnixListener(t, net.ListenConfig{}, "PONG\n")
defer func() {
assert.NoError(t, ul.Close())
}()
done := make(chan bool, 1)
go func() {
_, err := scanners.NewClamAV(ul.Addr().String(), 10*time.Second)
assert.NoError(t, err)
done <- true
}()
assert.True(t, <-done)
})
t.Run("fails if scanner is not pingable", func(t *testing.T) {
_, err := scanners.NewClamAV("", 0)
assert.ErrorIs(t, err, scanners.ErrScannerNotReachable)
})
}
func TestNewClamAV_Scan(t *testing.T) {
t.Run("returns a result", func(t *testing.T) {
ul := newUnixListener(t, net.ListenConfig{}, "PONG\n", "stream: Win.Test.EICAR_HDB-1 FOUND\n")
defer func() {
assert.NoError(t, ul.Close())
}()
done := make(chan bool, 1)
go func() {
scanner, err := scanners.NewClamAV(ul.Addr().String(), 10*time.Second)
assert.NoError(t, err)
result, err := scanner.Scan(scanners.Input{Body: strings.NewReader("DATA")})
assert.NoError(t, err)
assert.Equal(t, result.Description, "Win.Test.EICAR_HDB-1")
assert.True(t, result.Infected)
done <- true
}()
assert.True(t, <-done)
})
t.Run("aborts after a certain time", func(t *testing.T) {
ul := newUnixListener(t, net.ListenConfig{}, "PONG\n", "stream: Win.Test.EICAR_HDB-1 FOUND\n")
defer func() {
assert.NoError(t, ul.Close())
}()
done := make(chan bool, 1)
go func() {
scanner, err := scanners.NewClamAV(ul.Addr().String(), 10*time.Second)
assert.NoError(t, err)
result, err := scanner.Scan(scanners.Input{Body: strings.NewReader("DATA")})
assert.NoError(t, err)
assert.Equal(t, result.Description, "Win.Test.EICAR_HDB-1")
assert.True(t, result.Infected)
done <- true
}()
assert.True(t, <-done)
})
}
@@ -0,0 +1,112 @@
package scanners
import (
"context"
"fmt"
"net/http"
"net/url"
"path"
"regexp"
"time"
"github.com/opencloud-eu/reva/v2/pkg/mime"
ic "github.com/opencloud-eu/icap-client"
)
// Scanner is the interface that wraps the basic Do method
type Scanner interface {
Do(req ic.Request) (ic.Response, error)
}
// NewICAP returns a Scanner talking to an ICAP server
func NewICAP(icapURL string, icapService string, timeout time.Duration) (ICAP, error) {
endpoint, err := url.Parse(icapURL)
if err != nil {
return ICAP{}, err
}
endpoint.Scheme = "icap"
endpoint.Path = icapService
client, err := ic.NewClient(
ic.WithICAPConnectionTimeout(timeout),
)
if err != nil {
return ICAP{}, err
}
return ICAP{Client: &client, URL: endpoint.String()}, nil
}
// ICAP is responsible for scanning files using an ICAP server
type ICAP struct {
Client Scanner
URL string
}
// Scan scans a file using the ICAP server
func (s ICAP) Scan(in Input) (Result, error) {
ctx := context.TODO()
result := Result{}
optReq, err := ic.NewRequest(ctx, ic.MethodOPTIONS, s.URL, nil, nil)
if err != nil {
return result, err
}
optRes, err := s.Client.Do(optReq)
if err != nil {
return result, err
}
httpReq, err := http.NewRequest(http.MethodPost, in.Url, in.Body)
if err != nil {
return result, err
}
httpReq.ContentLength = in.Size
if mt := mime.Detect(path.Ext(in.Name) == "", in.Name); mt != "" {
httpReq.Header.Set("Content-Type", mt)
}
req, err := ic.NewRequest(ctx, ic.MethodREQMOD, s.URL, httpReq, nil)
if err != nil {
return result, err
}
if optRes.PreviewBytes > 0 {
err = req.SetPreview(optRes.PreviewBytes)
if err != nil {
return result, err
}
}
res, err := s.Client.Do(req)
if err != nil {
return result, err
}
result.ScanTime = time.Now()
// TODO: make header configurable
if data, infected := res.Header["X-Infection-Found"]; infected {
result.Infected = infected
match := regexp.MustCompile(`Threat=(.*);`).FindStringSubmatch(fmt.Sprint(data))
if len(match) > 1 {
result.Description = match[1]
}
}
if result.Infected || res.ContentResponse == nil {
return result, nil
}
// mcafee forwards the scan result as HTML in the content response;
// status 403 indicates that the file is infected
result.Infected = res.ContentResponse.StatusCode == http.StatusForbidden
result.Description = res.ContentResponse.Status
return result, nil
}
@@ -0,0 +1,187 @@
package scanners_test
import (
"bytes"
"errors"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
ic "github.com/opencloud-eu/icap-client"
"github.com/qsfera/server/services/antivirus/pkg/scanners"
"github.com/qsfera/server/services/antivirus/pkg/scanners/mocks"
)
func TestICAP_Scan(t *testing.T) {
var (
earlyExitErr = errors.New("stop here")
testUrl = "icap://test"
client = mocks.NewScanner(t)
scanner = &scanners.ICAP{Client: client, URL: testUrl}
)
t.Run("it sends a OPTIONS request to determine details", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, ic.MethodOPTIONS, request.Method)
assert.Equal(t, testUrl, request.URL.String())
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{})
assert.ErrorIs(t, earlyExitErr, err) // we can exit early, just in case check the error to be identical to the early exit error
})
t.Run("it sends a REQMOD request with all the details", func(t *testing.T) {
t.Run("request with ContentLength", func(t *testing.T) {
t.Run("with size", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, ic.MethodREQMOD, request.Method)
assert.Equal(t, testUrl, request.URL.String())
assert.EqualValues(t, 999, request.HTTPRequest.ContentLength)
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Size: 999})
assert.ErrorIs(t, earlyExitErr, err)
})
t.Run("without size", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, ic.MethodREQMOD, request.Method)
assert.Equal(t, testUrl, request.URL.String())
assert.EqualValues(t, 0, request.HTTPRequest.ContentLength)
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{})
assert.ErrorIs(t, earlyExitErr, err)
})
})
t.Run("request with Content-Type header", func(t *testing.T) {
t.Run("name contains known extension", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, "application/pdf", request.HTTPRequest.Header.Get("Content-Type"))
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Name: "report.pdf"})
assert.ErrorIs(t, earlyExitErr, err)
})
t.Run("name with unknown extension", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, "application/octet-stream", request.HTTPRequest.Header.Get("Content-Type"))
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Name: "report.unknown"})
assert.ErrorIs(t, earlyExitErr, err)
})
t.Run("name without extension", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, "httpd/unix-directory", request.HTTPRequest.Header.Get("Content-Type"))
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Name: "report"})
assert.ErrorIs(t, earlyExitErr, err)
})
})
t.Run("request with the OPTIONS response preview size ", func(t *testing.T) {
t.Run("with PreviewBytes set", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{PreviewBytes: 444}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, 444, request.PreviewBytes)
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Body: bytes.NewReader(make([]byte, 888))})
assert.ErrorIs(t, earlyExitErr, err)
})
t.Run("without PreviewBytes set", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, 0, request.PreviewBytes)
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Body: bytes.NewReader(make([]byte, 888))})
assert.ErrorIs(t, earlyExitErr, err)
})
})
})
t.Run("request with the OPTIONS response preview size ", func(t *testing.T) {
t.Run("with PreviewBytes set", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{PreviewBytes: 444}, nil).Once()
client.EXPECT().Do(mock.Anything).RunAndReturn(func(request ic.Request) (ic.Response, error) {
assert.Equal(t, 444, request.PreviewBytes)
return ic.Response{}, earlyExitErr
}).Once()
_, err := scanner.Scan(scanners.Input{Body: bytes.NewReader(make([]byte, 888))})
assert.ErrorIs(t, earlyExitErr, err)
})
t.Run("it handles virus scan results", func(t *testing.T) {
t.Run("no virus", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
result, err := scanner.Scan(scanners.Input{})
assert.Nil(t, err)
assert.False(t, result.Infected)
})
// clamav returns an X-Infection-Found header with the threat description
t.Run("X-Infection-Found header ", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).Return(ic.Response{Header: http.Header{"X-Infection-Found": []string{"Threat=bad threat;"}}}, nil).Once()
result, err := scanner.Scan(scanners.Input{})
assert.Nil(t, err)
assert.True(t, result.Infected)
assert.Equal(t, "bad threat", result.Description)
})
// skyhigh returns the information via the content response
t.Run("X-Infection-Found header", func(t *testing.T) {
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).Return(ic.Response{ContentResponse: &http.Response{StatusCode: http.StatusForbidden, Status: "some status"}}, nil).Once()
result, err := scanner.Scan(scanners.Input{})
assert.Nil(t, err)
assert.True(t, result.Infected)
assert.Equal(t, "some status", result.Description)
client.EXPECT().Do(mock.Anything).Return(ic.Response{}, nil).Once()
client.EXPECT().Do(mock.Anything).Return(ic.Response{ContentResponse: &http.Response{StatusCode: http.StatusOK}}, nil).Once()
result, err = scanner.Scan(scanners.Input{})
assert.Nil(t, err)
assert.False(t, result.Infected)
})
})
})
}
@@ -0,0 +1,97 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
"github.com/opencloud-eu/icap-client"
mock "github.com/stretchr/testify/mock"
)
// NewScanner creates a new instance of Scanner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewScanner(t interface {
mock.TestingT
Cleanup(func())
}) *Scanner {
mock := &Scanner{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Scanner is an autogenerated mock type for the Scanner type
type Scanner struct {
mock.Mock
}
type Scanner_Expecter struct {
mock *mock.Mock
}
func (_m *Scanner) EXPECT() *Scanner_Expecter {
return &Scanner_Expecter{mock: &_m.Mock}
}
// Do provides a mock function for the type Scanner
func (_mock *Scanner) Do(req icapclient.Request) (icapclient.Response, error) {
ret := _mock.Called(req)
if len(ret) == 0 {
panic("no return value specified for Do")
}
var r0 icapclient.Response
var r1 error
if returnFunc, ok := ret.Get(0).(func(icapclient.Request) (icapclient.Response, error)); ok {
return returnFunc(req)
}
if returnFunc, ok := ret.Get(0).(func(icapclient.Request) icapclient.Response); ok {
r0 = returnFunc(req)
} else {
r0 = ret.Get(0).(icapclient.Response)
}
if returnFunc, ok := ret.Get(1).(func(icapclient.Request) error); ok {
r1 = returnFunc(req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Scanner_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do'
type Scanner_Do_Call struct {
*mock.Call
}
// Do is a helper method to define mock.On call
// - req icapclient.Request
func (_e *Scanner_Expecter) Do(req interface{}) *Scanner_Do_Call {
return &Scanner_Do_Call{Call: _e.mock.On("Do", req)}
}
func (_c *Scanner_Do_Call) Run(run func(req icapclient.Request)) *Scanner_Do_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 icapclient.Request
if args[0] != nil {
arg0 = args[0].(icapclient.Request)
}
run(
arg0,
)
})
return _c
}
func (_c *Scanner_Do_Call) Return(response icapclient.Response, err error) *Scanner_Do_Call {
_c.Call.Return(response, err)
return _c
}
func (_c *Scanner_Do_Call) RunAndReturn(run func(req icapclient.Request) (icapclient.Response, error)) *Scanner_Do_Call {
_c.Call.Return(run)
return _c
}
@@ -0,0 +1,31 @@
package scanners
import (
"errors"
"io"
"time"
)
var (
// ErrScanTimeout is returned when a scan times out
ErrScanTimeout = errors.New("time out waiting for clamav to respond while scanning")
// ErrScannerNotReachable is returned when the scanner is not reachable
ErrScannerNotReachable = errors.New("failed to reach the scanner")
)
type (
// The Result is the common scan result to all scanners
Result struct {
Infected bool
ScanTime time.Time
Description string
}
// The Input is the common input to all scanners
Input struct {
Body io.Reader
Size int64
Url string
Name string
}
)
@@ -0,0 +1,50 @@
package debug
import (
"context"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/antivirus/pkg/config"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Context context.Context
Config *config.Config
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// Context provides a function to set the context option.
func Context(val context.Context) Option {
return func(o *Options) {
o.Context = val
}
}
// Config provides a function to set the config option.
func Config(val *config.Config) Option {
return func(o *Options) {
o.Config = val
}
}
@@ -0,0 +1,50 @@
package debug
import (
"context"
"errors"
"net/http"
"net/url"
"github.com/dutchcoders/go-clamd"
"github.com/qsfera/server/pkg/checks"
"github.com/qsfera/server/pkg/handlers"
"github.com/qsfera/server/pkg/service/debug"
"github.com/qsfera/server/pkg/version"
)
// Server initializes the debug service and server.
func Server(opts ...Option) (*http.Server, error) {
options := newOptions(opts...)
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
WithLogger(options.Logger).
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint)).
WithCheck("antivirus reachability", func(ctx context.Context) error {
cfg := options.Config
switch cfg.Scanner.Type {
default:
return errors.New("no antivirus configured")
case "clamav":
return clamd.NewClamd(cfg.Scanner.ClamAV.Socket).Ping()
case "icap":
u, err := url.Parse(cfg.Scanner.ICAP.URL)
if err != nil {
return err
}
return checks.NewTCPCheck(u.Host)(ctx)
}
})
return debug.NewService(
debug.Logger(options.Logger),
debug.Name(options.Config.Service.Name),
debug.Version(version.GetString()),
debug.Address(options.Config.Debug.Addr),
debug.Token(options.Config.Debug.Token),
debug.Pprof(options.Config.Debug.Pprof),
debug.Zpages(options.Config.Debug.Zpages),
debug.Ready(handlers.NewCheckHandler(readyHandlerConfiguration)),
), nil
}
@@ -0,0 +1,348 @@
package service
import (
"bytes"
"context"
"crypto/x509"
"errors"
"fmt"
"io"
"net/http"
"os"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/opencloud-eu/reva/v2/pkg/bytesize"
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/rhttp"
"go.opentelemetry.io/otel/trace"
"github.com/qsfera/server/pkg/generators"
"github.com/qsfera/server/pkg/log"
"github.com/qsfera/server/services/antivirus/pkg/config"
"github.com/qsfera/server/services/antivirus/pkg/scanners"
)
var (
// ErrFatal is returned when a fatal error occurs, and we want to exit.
ErrFatal = errors.New("fatal error")
// ErrEvent is returned when something went wrong with a specific event.
ErrEvent = errors.New("event error")
)
// Scanner is an abstraction for the actual virus scan
type Scanner interface {
Scan(body scanners.Input) (scanners.Result, error)
}
// NewAntivirus returns a service implementation for Service.
func NewAntivirus(cfg *config.Config, logger log.Logger, tracerProvider trace.TracerProvider) (Antivirus, error) {
var scanner Scanner
var err error
switch cfg.Scanner.Type {
default:
return Antivirus{}, fmt.Errorf("unknown av scanner: '%s'", cfg.Scanner.Type)
case config.ScannerTypeClamAV:
scanner, err = scanners.NewClamAV(cfg.Scanner.ClamAV.Socket, cfg.Scanner.ClamAV.Timeout)
case config.ScannerTypeICap:
scanner, err = scanners.NewICAP(cfg.Scanner.ICAP.URL, cfg.Scanner.ICAP.Service, cfg.Scanner.ICAP.Timeout)
}
if err != nil {
return Antivirus{}, err
}
av := Antivirus{
config: cfg,
log: logger,
tracerProvider: tracerProvider,
scanner: scanner,
client: rhttp.GetHTTPClient(rhttp.Insecure(true)),
stopCh: make(chan struct{}, 1),
stopped: new(atomic.Bool),
}
switch mode := cfg.MaxScanSizeMode; mode {
case config.MaxScanSizeModeSkip, config.MaxScanSizeModePartial:
break
default:
return av, fmt.Errorf("unknown max scan size mode '%s'", cfg.MaxScanSizeMode)
}
switch outcome := events.PostprocessingOutcome(cfg.InfectedFileHandling); outcome {
case events.PPOutcomeContinue, events.PPOutcomeAbort, events.PPOutcomeDelete:
av.outcome = outcome
default:
return av, fmt.Errorf("unknown infected file handling '%s'", outcome)
}
if cfg.MaxScanSize != "" {
b, err := bytesize.Parse(cfg.MaxScanSize)
if err != nil {
return av, err
}
av.maxScanSize = b.Bytes()
}
return av, nil
}
// Antivirus defines implements the business logic for Service.
type Antivirus struct {
config *config.Config
log log.Logger
scanner Scanner
outcome events.PostprocessingOutcome
maxScanSize uint64
tracerProvider trace.TracerProvider
client *http.Client
stopCh chan struct{}
stopped *atomic.Bool
}
// Run runs the service
func (av Antivirus) Run() error {
eventsCfg := av.config.Events
var rootCAPool *x509.CertPool
if av.config.Events.TLSRootCACertificate != "" {
rootCrtFile, err := os.Open(eventsCfg.TLSRootCACertificate)
if err != nil {
return err
}
var certBytes bytes.Buffer
if _, err := io.Copy(&certBytes, rootCrtFile); err != nil {
return err
}
rootCAPool = x509.NewCertPool()
rootCAPool.AppendCertsFromPEM(certBytes.Bytes())
av.config.Events.TLSInsecure = false
}
connName := generators.GenerateConnectionName(av.config.Service.Name, generators.NTypeBus)
natsStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(av.config.Events))
if err != nil {
return err
}
ch, err := events.Consume(natsStream, "antivirus", events.StartPostprocessingStep{})
if err != nil {
return err
}
wg := sync.WaitGroup{}
for range av.config.Workers {
wg.Go(func() {
EventLoop:
for {
select {
case e, ok := <-ch:
if !ok {
break EventLoop
}
err := av.processEvent(e, natsStream)
if err != nil {
switch {
case errors.Is(err, ErrFatal):
av.log.Fatal().Err(err).Msg("fatal error - exiting")
case errors.Is(err, ErrEvent):
av.log.Error().Err(err).Msg("continuing")
default:
av.log.Fatal().Err(err).Msg("unknown error - exiting")
}
}
if av.stopped.Load() {
break EventLoop
}
case <-av.stopCh:
break EventLoop
}
}
})
}
wg.Wait()
return nil
}
func (av Antivirus) Close() {
if av.stopped.CompareAndSwap(false, true) {
close(av.stopCh)
}
}
func (av Antivirus) processEvent(e events.Event, s events.Publisher) error {
ctx, span := av.tracerProvider.Tracer("antivirus").Start(e.GetTraceContext(context.Background()), "processEvent")
defer span.End()
av.log.Info().Str("traceID", span.SpanContext().TraceID().String()).Msg("TraceID")
ev := e.Event.(events.StartPostprocessingStep)
if ev.StepToStart != events.PPStepAntivirus {
return nil
}
if av.config.DebugScanOutcome != "" {
av.log.Warn().Str("antivir, clamav", ">>>>>>> ANTIVIRUS_DEBUG_SCAN_OUTCOME IS SET NO ACTUAL VIRUS SCAN IS PERFORMED!").Send()
if err := events.Publish(ctx, s, events.PostprocessingStepFinished{
FinishedStep: events.PPStepAntivirus,
Outcome: events.PostprocessingOutcome(av.config.DebugScanOutcome),
UploadID: ev.UploadID,
ExecutingUser: ev.ExecutingUser,
Filename: ev.Filename,
Result: events.VirusscanResult{
Infected: true,
Description: "DEBUG: forced outcome",
Scandate: time.Now(),
ResourceID: ev.ResourceID,
},
}); err != nil {
av.log.Fatal().Err(err).Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Msg("cannot publish events - exiting")
return fmt.Errorf("%w: cannot publish events", ErrFatal)
}
return fmt.Errorf("%w: no actual virus scan performed", ErrEvent)
}
av.log.Debug().Str("uploadid", ev.UploadID).Str("filename", ev.Filename).Msg("Starting virus scan.")
var errmsg string
start := time.Now()
res, err := av.process(ev)
if err != nil {
errmsg = err.Error()
}
duration := time.Since(start)
var outcome events.PostprocessingOutcome
switch {
case res.Infected:
outcome = av.outcome
case !res.Infected && err == nil:
outcome = events.PPOutcomeContinue
case err != nil:
outcome = events.PPOutcomeRetry
default:
// Not sure what this is about. Abort.
outcome = events.PPOutcomeAbort
}
av.log.Info().Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Str("virus", res.Description).Str("outcome", string(outcome)).Str("filename", ev.Filename).Str("user", ev.ExecutingUser.GetId().GetOpaqueId()).Bool("infected", res.Infected).Dur("duration", duration).Msg("File scanned")
if err := events.Publish(ctx, s, events.PostprocessingStepFinished{
FinishedStep: events.PPStepAntivirus,
Outcome: outcome,
UploadID: ev.UploadID,
ExecutingUser: ev.ExecutingUser,
Filename: ev.Filename,
Result: events.VirusscanResult{
Infected: res.Infected,
Description: res.Description,
Scandate: time.Now(),
ResourceID: ev.ResourceID,
ErrorMsg: errmsg,
},
}); err != nil {
av.log.Fatal().Err(err).Str("uploadid", ev.UploadID).Interface("resourceID", ev.ResourceID).Msg("cannot publish events - exiting")
return fmt.Errorf("%w: %s", ErrFatal, err)
}
return nil
}
// process the scan
func (av Antivirus) process(ev events.StartPostprocessingStep) (scanners.Result, error) {
if ev.Filesize == 0 {
av.log.Info().Str("uploadid", ev.UploadID).Msg("Skipping file to be virus scanned, file size is 0.")
return scanners.Result{ScanTime: time.Now()}, nil
}
filesize := ev.Filesize
headers := make(map[string]string)
switch {
case av.maxScanSize == 0:
// there is no size limit
break
case av.config.MaxScanSizeMode == config.MaxScanSizeModeSkip && filesize > av.maxScanSize:
// skip the file if it is bigger than the max scan size
av.log.Info().Str("uploadid", ev.UploadID).Uint64("filesize", filesize).
Msg("Skipping file to be virus scanned, file size is bigger than max scan size.")
return scanners.Result{ScanTime: time.Now()}, nil
case av.config.MaxScanSizeMode == config.MaxScanSizeModePartial && filesize > av.maxScanSize:
// set the range header to only download the first maxScanSize bytes
headers["Range"] = fmt.Sprintf("bytes=0-%d", av.maxScanSize-1)
filesize = av.maxScanSize // inform the scanner that we are only scanning part of the file
}
var err error
var rrc io.ReadCloser
switch ev.UploadID {
default:
rrc, err = av.downloadViaToken(ev.URL, headers)
case "":
rrc, err = av.downloadViaReva(ev.URL, ev.Token, ev.RevaToken, headers)
}
if err != nil {
av.log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("error downloading file")
return scanners.Result{}, err
}
defer func() {
_ = rrc.Close()
}()
av.log.Debug().Str("uploadid", ev.UploadID).Msg("Downloaded file successfully, starting virusscan")
res, err := av.scanner.Scan(scanners.Input{Body: rrc, Size: int64(filesize), Url: ev.URL, Name: ev.Filename})
if err != nil {
av.log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("error scanning file")
}
return res, err
}
// download will download the file
func (av Antivirus) downloadViaToken(url string, headers map[string]string) (io.ReadCloser, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
return av.doDownload(req, headers)
}
// download will download the file
func (av Antivirus) downloadViaReva(url string, dltoken string, revatoken string, headers map[string]string) (io.ReadCloser, error) {
req, err := rhttp.NewRequest(ctxpkg.ContextSetToken(context.Background(), revatoken), http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("X-Reva-Transfer", dltoken)
return av.doDownload(req, headers)
}
func (av Antivirus) doDownload(req *http.Request, headers map[string]string) (io.ReadCloser, error) {
for k, v := range headers {
req.Header.Add(k, v)
}
res, err := av.client.Do(req)
if err != nil {
return nil, err
}
if !slices.Contains([]int{http.StatusOK, http.StatusPartialContent}, res.StatusCode) {
_ = res.Body.Close()
return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
}
return res.Body, nil
}