Initial QSfera import
This commit is contained in:
Курнат Андрей
@@ -0,0 +1,41 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"go-micro.dev/v4/client"
|
||||
)
|
||||
|
||||
// Config combines all available configuration parts.
|
||||
type Config struct {
|
||||
Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service
|
||||
|
||||
Service Service `yaml:"-"`
|
||||
|
||||
LogLevel string `yaml:"loglevel" env:"OC_LOG_LEVEL;SEARCH_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'." introductionVersion:"1.0.0"`
|
||||
Debug Debug `yaml:"debug"`
|
||||
|
||||
GRPC GRPCConfig `yaml:"grpc"`
|
||||
GrpcClient client.Client `yaml:"-"`
|
||||
|
||||
TokenManager *TokenManager `yaml:"token_manager"`
|
||||
|
||||
Reva *shared.Reva `yaml:"reva"`
|
||||
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
|
||||
Events Events `yaml:"events"`
|
||||
Engine Engine `yaml:"engine"`
|
||||
Extractor Extractor `yaml:"extractor"`
|
||||
ContentExtractionSizeLimit uint64 `yaml:"content_extraction_size_limit" env:"SEARCH_CONTENT_EXTRACTION_SIZE_LIMIT" desc:"Maximum file size in bytes that is allowed for content extraction." introductionVersion:"1.0.0"`
|
||||
BatchSize int `yaml:"batch_size" env:"SEARCH_BATCH_SIZE" desc:"The number of documents to process in a single batch. Defaults to 500." introductionVersion:"1.0.0"`
|
||||
|
||||
ServiceAccount ServiceAccount `yaml:"service_account"`
|
||||
|
||||
Context context.Context `yaml:"-"`
|
||||
}
|
||||
|
||||
// ServiceAccount is the configuration for the used service account
|
||||
type ServiceAccount struct {
|
||||
ServiceAccountID string `yaml:"service_account_id" env:"OC_SERVICE_ACCOUNT_ID;SEARCH_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details." introductionVersion:"1.0.0"`
|
||||
ServiceAccountSecret string `yaml:"service_account_secret" env:"OC_SERVICE_ACCOUNT_SECRET;SEARCH_SERVICE_ACCOUNT_SECRET" desc:"The service account secret." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package config
|
||||
|
||||
// Extractor defines which extractor to use
|
||||
type Extractor struct {
|
||||
Type string `yaml:"type" env:"SEARCH_EXTRACTOR_TYPE" desc:"Defines the content extraction engine. Defaults to 'basic'. Supported values are: 'basic' and 'tika'." introductionVersion:"1.0.0"`
|
||||
CS3AllowInsecure bool `yaml:"cs3_allow_insecure" env:"OC_INSECURE;SEARCH_EXTRACTOR_CS3SOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the CS3 source." introductionVersion:"1.0.0"`
|
||||
Tika ExtractorTika `yaml:"tika"`
|
||||
}
|
||||
|
||||
// ExtractorTika configures the Tika extractor
|
||||
type ExtractorTika struct {
|
||||
TikaURL string `yaml:"tika_url" env:"SEARCH_EXTRACTOR_TIKA_TIKA_URL" desc:"URL of the tika server." introductionVersion:"1.0.0"`
|
||||
CleanStopWords bool `yaml:"clean_stop_words" env:"SEARCH_EXTRACTOR_TIKA_CLEAN_STOP_WORDS" desc:"Defines if stop words should be cleaned or not. See the documentation for more details." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package config
|
||||
|
||||
// Debug defines the available debug configuration.
|
||||
type Debug struct {
|
||||
Addr string `yaml:"addr" env:"SEARCH_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed." introductionVersion:"1.0.0"`
|
||||
Token string `yaml:"token" env:"SEARCH_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint." introductionVersion:"1.0.0"`
|
||||
Pprof bool `yaml:"pprof" env:"SEARCH_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling." introductionVersion:"1.0.0"`
|
||||
Zpages bool `yaml:"zpages" env:"SEARCH_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package defaults
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/defaults"
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"github.com/qsfera/server/pkg/structs"
|
||||
"github.com/qsfera/server/services/search/pkg/config"
|
||||
)
|
||||
|
||||
// FullDefaultConfig returns a fully initialized default configuration
|
||||
func FullDefaultConfig() *config.Config {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
EnsureDefaults(cfg)
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
// DefaultConfig returns a basic default configuration
|
||||
func DefaultConfig() *config.Config {
|
||||
return &config.Config{
|
||||
Debug: config.Debug{
|
||||
Addr: "127.0.0.1:9224",
|
||||
Token: "",
|
||||
},
|
||||
GRPC: config.GRPCConfig{
|
||||
Addr: "127.0.0.1:9220",
|
||||
Namespace: "qsfera.api",
|
||||
},
|
||||
Service: config.Service{
|
||||
Name: "search",
|
||||
},
|
||||
Reva: shared.DefaultRevaConfig(),
|
||||
Engine: config.Engine{
|
||||
Type: "bleve",
|
||||
Bleve: config.EngineBleve{
|
||||
Datapath: filepath.Join(defaults.BaseDataPath(), "search"),
|
||||
},
|
||||
OpenSearch: config.EngineOpenSearch{
|
||||
ResourceIndex: config.EngineOpenSearchResourceIndex{
|
||||
Name: "qsfera-resource",
|
||||
},
|
||||
},
|
||||
},
|
||||
Extractor: config.Extractor{
|
||||
Type: "basic",
|
||||
CS3AllowInsecure: false,
|
||||
Tika: config.ExtractorTika{
|
||||
TikaURL: "http://127.0.0.1:9998",
|
||||
CleanStopWords: false,
|
||||
},
|
||||
},
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
Cluster: "qsfera-cluster",
|
||||
DebounceDuration: 1000,
|
||||
AsyncUploads: true,
|
||||
NumConsumers: 1,
|
||||
EnableTLS: false,
|
||||
MaxAckPending: 1000,
|
||||
AckWait: 1 * time.Minute,
|
||||
},
|
||||
ContentExtractionSizeLimit: 20 * 1024 * 1024, // Limit content extraction to <20MB files by default
|
||||
BatchSize: 50,
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureDefaults adds default values to the configuration if they are not set yet
|
||||
func EnsureDefaults(cfg *config.Config) {
|
||||
if cfg.LogLevel == "" {
|
||||
cfg.LogLevel = "error"
|
||||
}
|
||||
|
||||
if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil {
|
||||
cfg.TokenManager = &config.TokenManager{
|
||||
JWTSecret: cfg.Commons.TokenManager.JWTSecret,
|
||||
}
|
||||
} else if cfg.TokenManager == nil {
|
||||
cfg.TokenManager = &config.TokenManager{}
|
||||
}
|
||||
|
||||
if cfg.Reva == nil && cfg.Commons != nil {
|
||||
cfg.Reva = structs.CopyOrZeroValue(cfg.Commons.Reva)
|
||||
}
|
||||
if cfg.GRPCClientTLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS)
|
||||
}
|
||||
if cfg.GRPC.TLS == nil && cfg.Commons != nil {
|
||||
cfg.GRPC.TLS = structs.CopyOrZeroValue(cfg.Commons.GRPCServiceTLS)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize sanitizes the configuration
|
||||
func Sanitize(cfg *config.Config) {
|
||||
// no http endpoint to be sanitized
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Engine defines which search engine to use
|
||||
type Engine struct {
|
||||
Type string `yaml:"type" env:"SEARCH_ENGINE_TYPE" desc:"Defines which search engine to use. Defaults to 'bleve'. Supported values are: 'bleve'." introductionVersion:"1.0.0"`
|
||||
Bleve EngineBleve `yaml:"bleve"`
|
||||
OpenSearch EngineOpenSearch `yaml:"open_search"`
|
||||
}
|
||||
|
||||
// EngineBleve configures the bleve engine
|
||||
type EngineBleve struct {
|
||||
Datapath string `yaml:"data_path" env:"SEARCH_ENGINE_BLEVE_DATA_PATH" desc:"The directory where the filesystem will store search data. If not defined, the root directory derives from $OC_BASE_DATA_PATH/search." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// EngineOpenSearch configures the OpenSearch engine
|
||||
type EngineOpenSearch struct {
|
||||
Client EngineOpenSearchClient `yaml:"client"`
|
||||
ResourceIndex EngineOpenSearchResourceIndex `yaml:"resource_index"`
|
||||
}
|
||||
|
||||
// EngineOpenSearchResourceIndex defines the OpenSearch index for resources
|
||||
type EngineOpenSearchResourceIndex struct {
|
||||
Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The name of the OpenSearch index for resources." introductionVersion:"4.0.0"`
|
||||
}
|
||||
|
||||
// EngineOpenSearchClient configures the OpenSearch client
|
||||
type EngineOpenSearchClient struct {
|
||||
Addresses []string `yaml:"addresses" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES" desc:"The addresses of the OpenSearch nodes.." introductionVersion:"4.0.0"`
|
||||
Username string `yaml:"username" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_USERNAME" desc:"Username for HTTP Basic Authentication." introductionVersion:"4.0.0"`
|
||||
Password string `yaml:"password" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_PASSWORD" desc:"Password for HTTP Basic Authentication." introductionVersion:"4.0.0"`
|
||||
Header http.Header `yaml:"header" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_HEADER" desc:"HTTP headers to include in requests." introductionVersion:"4.0.0"`
|
||||
CACert string `yaml:"ca_cert" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_CA_CERT" desc:"Path/File name for the root CA certificate (in PEM format) used to validate TLS server certificates of the opensearch server." introductionVersion:"4.0.0"`
|
||||
RetryOnStatus []int `yaml:"retry_on_status" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_RETRY_ON_STATUS" desc:"HTTP status codes that trigger a retry." introductionVersion:"4.0.0"`
|
||||
DisableRetry bool `yaml:"disable_retry" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_DISABLE_RETRY" desc:"Disable retries on errors." introductionVersion:"4.0.0"`
|
||||
EnableRetryOnTimeout bool `yaml:"enable_retry_on_timeout" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_RETRY_ON_TIMEOUT" desc:"Enable retries on timeout." introductionVersion:"4.0.0"`
|
||||
MaxRetries int `yaml:"max_retries" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_MAX_RETRIES" desc:"Maximum number of retries for requests." introductionVersion:"4.0.0"`
|
||||
CompressRequestBody bool `yaml:"compress_request_body" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_COMPRESS_REQUEST_BODY" desc:"Compress request bodies." introductionVersion:"4.0.0"`
|
||||
DiscoverNodesOnStart bool `yaml:"discover_nodes_on_start" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_DISCOVER_NODES_ON_START" desc:"Discover nodes on service start." introductionVersion:"4.0.0"`
|
||||
DiscoverNodesInterval time.Duration `yaml:"discover_nodes_interval" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_DISCOVER_NODES_INTERVAL" desc:"Interval for discovering nodes." introductionVersion:"4.0.0"`
|
||||
EnableMetrics bool `yaml:"enable_metrics" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_METRICS" desc:"Enable metrics collection." introductionVersion:"4.0.0"`
|
||||
EnableDebugLogger bool `yaml:"enable_debug_logger" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_DEBUG_LOGGER" desc:"Enable debug logging." introductionVersion:"4.0.0"`
|
||||
Insecure bool `yaml:"insecure" env:"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_INSECURE" desc:"Skip TLS certificate verification." introductionVersion:"4.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package config
|
||||
|
||||
import "github.com/qsfera/server/pkg/shared"
|
||||
|
||||
// GRPCConfig defines the available grpc configuration.
|
||||
type GRPCConfig struct {
|
||||
Disabled bool `yaml:"disabled" env:"SEARCH_GRPC_DISABLED" desc:"Disables the GRPC service. Set this to true if the service should only handle events." introductionVersion:"4.0.0"`
|
||||
Addr string `yaml:"addr" env:"SEARCH_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/qsfera/server/pkg/config"
|
||||
"github.com/qsfera/server/pkg/shared"
|
||||
"github.com/qsfera/server/services/search/pkg/config"
|
||||
"github.com/qsfera/server/services/search/pkg/config/defaults"
|
||||
|
||||
"github.com/qsfera/server/pkg/config/envdecode"
|
||||
)
|
||||
|
||||
// ParseConfig loads configuration from known paths.
|
||||
func ParseConfig(cfg *config.Config) error {
|
||||
err := occfg.BindSourcesToStructs(cfg.Service.Name, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defaults.EnsureDefaults(cfg)
|
||||
|
||||
// load all env variables relevant to the config in the current context.
|
||||
if err := envdecode.Decode(cfg); err != nil {
|
||||
// no environment variable set for this config is an expected "error"
|
||||
if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
defaults.Sanitize(cfg)
|
||||
|
||||
return Validate(cfg)
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
return shared.MissingJWTTokenError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
if cfg.ServiceAccount.ServiceAccountID == "" {
|
||||
return shared.MissingServiceAccountID(cfg.Service.Name)
|
||||
}
|
||||
if cfg.ServiceAccount.ServiceAccountSecret == "" {
|
||||
return shared.MissingServiceAccountSecret(cfg.Service.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package config
|
||||
|
||||
// Reva defines all available REVA configuration.
|
||||
type Reva struct {
|
||||
Address string `yaml:"address" env:"OC_REVA_GATEWAY" desc:"The CS3 gateway endpoint." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// TokenManager is the config for using the reva token manager
|
||||
type TokenManager struct {
|
||||
JWTSecret string `yaml:"jwt_secret" env:"OC_JWT_SECRET;SEARCH_JWT_SECRET" desc:"The secret to mint and validate jwt tokens." introductionVersion:"1.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package config
|
||||
|
||||
import "time"
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"SEARCH_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"4.0.0"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;SEARCH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;SEARCH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
AsyncUploads bool `yaml:"async_uploads" env:"OC_ASYNC_UPLOADS;SEARCH_EVENTS_ASYNC_UPLOADS" desc:"Enable asynchronous file uploads." introductionVersion:"1.0.0"`
|
||||
NumConsumers int `yaml:"num_consumers" env:"SEARCH_EVENTS_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for searching files. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"1.0.0"`
|
||||
DebounceDuration int `yaml:"debounce_duration" env:"SEARCH_EVENTS_REINDEX_DEBOUNCE_DURATION" desc:"The duration in milliseconds the reindex debouncer waits before triggering a reindex of a space that was modified." introductionVersion:"1.0.0"`
|
||||
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;SEARCH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;SEARCH_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided SEARCH_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;SEARCH_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_EVENTS_AUTH_USERNAME;SEARCH_EVENTS_AUTH_USERNAME" desc:"The username to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;SEARCH_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the КуСфера service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
|
||||
MaxAckPending int `yaml:"max_ack_pending" env:"SEARCH_EVENTS_MAX_ACK_PENDING" desc:"The maximum number of unacknowledged messages. This is used to limit the number of messages that can be in flight at the same time." introductionVersion:"4.0.0"`
|
||||
AckWait time.Duration `yaml:"ack_wait" env:"SEARCH_EVENTS_ACK_WAIT" desc:"The time to wait for an ack before the message is redelivered. This is used to ensure that messages are not lost if the consumer crashes." introductionVersion:"4.0.0"`
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package config
|
||||
|
||||
// Service defines the available service configuration.
|
||||
type Service struct {
|
||||
Name string `yaml:"-"`
|
||||
}
|
||||
Reference in New Issue
Block a user