Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,100 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package accservice
import (
"encoding/json"
"fmt"
"net/url"
"path"
"strings"
"github.com/pkg/errors"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils/network"
)
// RequestResponse holds the response of an accounts service query.
type RequestResponse struct {
Success bool
Error string
Data interface{}
}
type accountsServiceSettings struct {
URL *url.URL
User string
Password string
}
var settings accountsServiceSettings
// Query performs an account service query.
func Query(endpoint string, params network.URLParams) (*RequestResponse, error) {
fullURL, err := network.GenerateURL(fmt.Sprintf("%v://%v", settings.URL.Scheme, settings.URL.Host), path.Join(settings.URL.Path, endpoint), params)
if err != nil {
return nil, errors.Wrap(err, "error while building the service accounts query URL")
}
data, err := network.ReadEndpoint(fullURL, &network.BasicAuth{User: settings.User, Password: settings.Password}, false)
if err != nil {
return nil, errors.Wrap(err, "unable to query the service accounts endpoint")
}
resp := &RequestResponse{}
if err := json.Unmarshal(data, resp); err != nil {
return nil, errors.Wrap(err, "unable to unmarshal response data")
}
return resp, nil
}
// GetResponseValue gets a value from an account service query using a dotted path notation.
func GetResponseValue(resp *RequestResponse, path string) interface{} {
if data, ok := resp.Data.(map[string]interface{}); ok {
tokens := strings.Split(path, ".")
for i, name := range tokens {
if i == len(tokens)-1 {
if value, ok := data[name]; ok {
return value
}
}
if data, ok = data[name].(map[string]interface{}); !ok {
break
}
}
}
return nil
}
// InitAccountsService initializes the global accounts service.
func InitAccountsService(conf *config.Configuration) error {
URL, err := url.Parse(conf.AccountsService.URL)
if err != nil {
return errors.Wrap(err, "unable to parse the accounts service URL")
}
settings.URL = URL
settings.User = conf.AccountsService.User
settings.Password = conf.AccountsService.Password
return nil
}
@@ -0,0 +1,87 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package config
// Configuration holds the general Mentix configuration.
type Configuration struct {
Prefix string `mapstructure:"prefix"`
Connectors struct {
GOCDB struct {
Address string `mapstructure:"address"`
Scope string `mapstructure:"scope"`
APIKey string `mapstructure:"apikey"`
} `mapstructure:"gocdb"`
} `mapstructure:"connectors"`
UpdateInterval string `mapstructure:"update_interval"`
Services struct {
CriticalTypes []string `mapstructure:"critical_types"`
} `mapstructure:"services"`
Exporters struct {
WebAPI struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
} `mapstructure:"webapi"`
CS3API struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
ElevatedServiceTypes []string `mapstructure:"elevated_service_types"`
} `mapstructure:"cs3api"`
SiteLocations struct {
Endpoint string `mapstructure:"endpoint"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
IsProtected bool `mapstructure:"is_protected"`
} `mapstructure:"siteloc"`
PrometheusSD struct {
OutputPath string `mapstructure:"output_path"`
EnabledConnectors []string `mapstructure:"enabled_connectors"`
} `mapstructure:"promsd"`
Metrics struct {
EnabledConnectors []string `mapstructure:"enabled_connectors"`
} `mapstructure:"metrics"`
} `mapstructure:"exporters"`
AccountsService struct {
URL string `mapstructure:"url"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
} `mapstructure:"accounts"`
// Internal settings
EnabledConnectors []string `mapstructure:"-"`
EnabledImporters []string `mapstructure:"-"`
EnabledExporters []string `mapstructure:"-"`
}
// Init sets sane defaults.
func (c *Configuration) Init() {
if c.Prefix == "" {
c.Prefix = "mentix"
}
// TODO(daniel): add default that works out of the box
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package config
const (
// ConnectorIDGOCDB is the connector identifier for GOCDB.
ConnectorIDGOCDB = "gocdb"
)
const (
// ExporterIDWebAPI is the identifier for the WebAPI exporter.
ExporterIDWebAPI = "webapi"
// ExporterIDCS3API is the identifier for the CS3API exporter.
ExporterIDCS3API = "cs3api"
// ExporterIDSiteLocations is the identifier for the Site Locations exporter.
ExporterIDSiteLocations = "siteloc"
// ExporterIDPrometheusSD is the identifier for the PrometheusSD exporter.
ExporterIDPrometheusSD = "promsd"
// ExporterIDMetrics is the identifier for the Metrics exporter.
ExporterIDMetrics = "metrics"
)
@@ -0,0 +1,67 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"fmt"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// Connector is the interface that all connectors must implement.
type Connector interface {
entity.Entity
// RetrieveMeshData fetches new mesh data.
RetrieveMeshData() (*meshdata.MeshData, error)
// UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that
// should be updated, not the entire data set.
UpdateMeshData(data *meshdata.MeshData) error
}
// BaseConnector implements basic connector functionality common to all connectors.
type BaseConnector struct {
conf *config.Configuration
log *zerolog.Logger
}
// Activate activates the connector.
func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if conf == nil {
return fmt.Errorf("no configuration provided")
}
connector.conf = conf
if log == nil {
return fmt.Errorf("no logger provided")
}
connector.log = log
return nil
}
// UpdateMeshData updates the provided mesh data on the target side. The provided data only contains the data that
// should be updated, not the entire data set.
func (connector *BaseConnector) UpdateMeshData(data *meshdata.MeshData) error {
return fmt.Errorf("the connector doesn't support updating of mesh data")
}
@@ -0,0 +1,68 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
)
var (
registeredConnectors = entity.NewRegistry()
)
// Collection represents a collection of connectors.
type Collection struct {
Connectors []Connector
}
// Entities gets the entities in this collection.
func (collection *Collection) Entities() []entity.Entity {
entities := make([]entity.Entity, 0, len(collection.Connectors))
for _, connector := range collection.Connectors {
entities = append(entities, connector)
}
return entities
}
// ActivateAll activates all entities in the collection.
func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error {
return entity.ActivateEntities(collection, conf, log)
}
// AvailableConnectors returns a collection of all connectors that are enabled in the configuration.
func AvailableConnectors(conf *config.Configuration) (*Collection, error) {
entities, err := registeredConnectors.FindEntities(conf.EnabledConnectors, true, true)
if err != nil {
return nil, err
}
connectors := make([]Connector, 0, len(entities))
for _, entry := range entities {
connectors = append(connectors, entry.(Connector))
}
return &Collection{Connectors: connectors}, nil
}
func registerConnector(connector Connector) {
registeredConnectors.Register(connector)
}
@@ -0,0 +1,317 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package connectors
import (
"encoding/xml"
"fmt"
"net/url"
"path"
"strings"
"time"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/connectors/gocdb"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils/network"
)
// GOCDBConnector is used to read mesh data from a GOCDB instance.
type GOCDBConnector struct {
BaseConnector
gocdbAddress string
}
// Activate activates the connector.
func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := connector.BaseConnector.Activate(conf, log); err != nil {
return err
}
// Check and store GOCDB specific settings
connector.gocdbAddress = conf.Connectors.GOCDB.Address
if len(connector.gocdbAddress) == 0 {
return fmt.Errorf("no GOCDB address configured")
}
return nil
}
// RetrieveMeshData fetches new mesh data.
func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) {
meshData := new(meshdata.MeshData)
// Query all data from GOCDB
if err := connector.queryServiceTypes(meshData); err != nil {
return nil, fmt.Errorf("could not query service types: %v", err)
}
if err := connector.querySites(meshData); err != nil {
return nil, fmt.Errorf("could not query sites: %v", err)
}
for _, site := range meshData.Sites {
// Get services associated with the current site
if err := connector.queryServices(meshData, site); err != nil {
return nil, fmt.Errorf("could not query services of site '%v': %v", site.Name, err)
}
// Get downtimes scheduled for the current site
if err := connector.queryDowntimes(meshData, site); err != nil {
return nil, fmt.Errorf("could not query downtimes of site '%v': %v", site.Name, err)
}
}
meshData.InferMissingData()
return meshData, nil
}
func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error {
var scope string
if hasScope {
scope = connector.conf.Connectors.GOCDB.Scope
}
// Get the data from GOCDB
data, err := gocdb.QueryGOCDB(connector.gocdbAddress, method, isPrivate, scope, connector.conf.Connectors.GOCDB.APIKey, params)
if err != nil {
return err
}
// Unmarshal it
if err := xml.Unmarshal(data, v); err != nil {
return fmt.Errorf("unable to unmarshal data: %v", err)
}
return nil
}
func (connector *GOCDBConnector) queryServiceTypes(meshData *meshdata.MeshData) error {
var serviceTypes gocdb.ServiceTypes
if err := connector.query(&serviceTypes, "get_service_types", false, false, network.URLParams{}); err != nil {
return err
}
// Copy retrieved data into the mesh data
meshData.ServiceTypes = nil
for _, serviceType := range serviceTypes.Types {
meshData.ServiceTypes = append(meshData.ServiceTypes, &meshdata.ServiceType{
Name: serviceType.Name,
Description: serviceType.Description,
})
}
return nil
}
func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error {
var sites gocdb.Sites
if err := connector.query(&sites, "get_site", false, true, network.URLParams{}); err != nil {
return err
}
// Copy retrieved data into the mesh data
meshData.Sites = nil
for _, site := range sites.Sites {
properties := connector.extensionsToMap(&site.Extensions)
// The site ID can be set through a property; by default, the site short name will be used
siteID := meshdata.GetPropertyValue(properties, meshdata.PropertySiteID, site.ShortName)
// See if an organization has been defined using properties; otherwise, use the official name
organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName)
meshsite := &meshdata.Site{
ID: siteID,
Name: site.ShortName,
FullName: site.OfficialName,
Organization: organization,
Domain: site.Domain,
Homepage: site.Homepage,
Email: site.Email,
Description: site.Description,
Country: site.Country,
CountryCode: site.CountryCode,
Longitude: site.Longitude,
Latitude: site.Latitude,
Services: nil,
Properties: properties,
Downtimes: meshdata.Downtimes{},
}
meshData.Sites = append(meshData.Sites, meshsite)
}
return nil
}
func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site *meshdata.Site) error {
var services gocdb.Services
if err := connector.query(&services, "get_service", false, true, network.URLParams{"sitename": site.Name}); err != nil {
return err
}
getServiceURLString := func(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint, host string) string {
urlstr := "https://" + host // Fall back to the provided hostname
if svcURL, err := connector.getServiceURL(service, endpoint); err == nil {
urlstr = svcURL.String()
}
return urlstr
}
// Copy retrieved data into the mesh data
site.Services = nil
for _, service := range services.Services {
host := service.Host
// If a URL is provided, extract the port from it and append it to the host
if len(service.URL) > 0 {
if hostURL, err := url.Parse(service.URL); err == nil {
if port := hostURL.Port(); len(port) > 0 {
host += ":" + port
}
}
}
// Assemble additional endpoints
var endpoints []*meshdata.ServiceEndpoint
for _, endpoint := range service.Endpoints.Endpoints {
endpoints = append(endpoints, &meshdata.ServiceEndpoint{
Type: connector.findServiceType(meshData, endpoint.Type),
Name: endpoint.Name,
RawURL: endpoint.URL,
URL: getServiceURLString(service, endpoint, host),
IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"),
Properties: connector.extensionsToMap(&endpoint.Extensions),
})
}
// Add the service to the site
site.Services = append(site.Services, &meshdata.Service{
ServiceEndpoint: &meshdata.ServiceEndpoint{
Type: connector.findServiceType(meshData, service.Type),
Name: service.Type,
RawURL: service.URL,
URL: getServiceURLString(service, nil, host),
IsMonitored: strings.EqualFold(service.IsMonitored, "Y"),
Properties: connector.extensionsToMap(&service.Extensions),
},
Host: host,
AdditionalEndpoints: endpoints,
})
}
return nil
}
func (connector *GOCDBConnector) queryDowntimes(meshData *meshdata.MeshData, site *meshdata.Site) error {
var downtimes gocdb.Downtimes
if err := connector.query(&downtimes, "get_downtime_nested_services", false, true, network.URLParams{"topentity": site.Name, "ongoing_only": "yes"}); err != nil {
return err
}
// Copy retrieved data into the mesh data
site.Downtimes.Clear()
for _, dt := range downtimes.Downtimes {
if !strings.EqualFold(dt.Severity, "outage") { // Only take real outages into account
continue
}
services := make([]string, 0, len(dt.AffectedServices.Services))
for _, service := range dt.AffectedServices.Services {
// Only add critical services to the list of affected services
if utils.FindInStringArray(service.Type, connector.conf.Services.CriticalTypes, false) != -1 {
services = append(services, service.Type)
}
}
_, _ = site.Downtimes.ScheduleDowntime(time.Unix(dt.StartDate, 0), time.Unix(dt.EndDate, 0), services)
}
return nil
}
func (connector *GOCDBConnector) findServiceType(meshData *meshdata.MeshData, name string) *meshdata.ServiceType {
for _, serviceType := range meshData.ServiceTypes {
if strings.EqualFold(serviceType.Name, name) {
return serviceType
}
}
// If the service type doesn't exist, create a default one
return &meshdata.ServiceType{Name: name, Description: ""}
}
func (connector *GOCDBConnector) extensionsToMap(extensions *gocdb.Extensions) map[string]string {
properties := make(map[string]string)
for _, ext := range extensions.Extensions {
properties[ext.Key] = ext.Value
}
return properties
}
func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint) (*url.URL, error) {
urlstr := service.URL
if len(urlstr) == 0 {
// The URL defaults to the hostname using the HTTPS protocol
urlstr = "https://" + service.Host
}
svcURL, err := url.ParseRequestURI(urlstr)
if err != nil {
return nil, fmt.Errorf("unable to parse URL '%v': %v", urlstr, err)
}
// If an endpoint was provided, use its path
if endpoint != nil {
// If the endpoint URL is an absolute one, just use that; otherwise, make an absolute one out of it
if endpointURL, err := url.ParseRequestURI(endpoint.URL); err == nil && len(endpointURL.Scheme) > 0 {
svcURL = endpointURL
} else {
// Replace entire URL path if the relative path starts with a slash; otherwise, just append
if strings.HasPrefix(endpoint.URL, "/") {
svcURL.Path = endpoint.URL
} else {
svcURL.Path = path.Join(svcURL.Path, endpoint.URL)
if strings.HasSuffix(endpoint.URL, "/") { // Restore trailing slash if necessary
svcURL.Path += "/"
}
}
}
}
return svcURL, nil
}
// GetID returns the ID of the connector.
func (connector *GOCDBConnector) GetID() string {
return config.ConnectorIDGOCDB
}
// GetName returns the display name of the connector.
func (connector *GOCDBConnector) GetName() string {
return "GOCDB"
}
func init() {
registerConnector(&GOCDBConnector{})
}
@@ -0,0 +1,61 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package gocdb
import (
"fmt"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils/network"
)
// QueryGOCDB retrieves data from one of GOCDB's endpoints.
func QueryGOCDB(address string, method string, isPrivate bool, scope string, apiKey string, params network.URLParams) ([]byte, error) {
// The method must always be specified
params["method"] = method
// If a scope or an API key were specified, pass them to the endpoint as well
if len(scope) > 0 {
params["scope"] = scope
}
if len(apiKey) > 0 {
params["apikey"] = apiKey
}
// GOCDB's public API is located at <gocdb-host>/gocdbpi/public, the private one at <gocdb-host>/gocdbpi/private
var path string
if isPrivate {
path = "/gocdbpi/private"
} else {
path = "/gocdbpi/public"
}
// Query the data from GOCDB
endpointURL, err := network.GenerateURL(address, path, params)
if err != nil {
return nil, fmt.Errorf("unable to generate the GOCDB URL: %v", err)
}
data, err := network.ReadEndpoint(endpointURL, nil, true)
if err != nil {
return nil, fmt.Errorf("unable to read GOCDB endpoint: %v", err)
}
return data, nil
}
@@ -0,0 +1,114 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package gocdb
// Extension represents Key-Value pairs in GOCDB.
type Extension struct {
Key string `xml:"KEY"`
Value string `xml:"VALUE"`
}
// Extensions is a list of Extension objects.
type Extensions struct {
Extensions []*Extension `xml:"EXTENSION"`
}
// ServiceType represents a service type in GOCDB.
type ServiceType struct {
Name string `xml:"SERVICE_TYPE_NAME"`
Description string `xml:"SERVICE_TYPE_DESC"`
}
// ServiceTypes is a list of ServiceType objects.
type ServiceTypes struct {
Types []*ServiceType `xml:"SERVICE_TYPE"`
}
// Site represents a site in GOCDB.
type Site struct {
ShortName string `xml:"SHORT_NAME"`
OfficialName string `xml:"OFFICIAL_NAME"`
Description string `xml:"SITE_DESCRIPTION"`
Homepage string `xml:"HOME_URL"`
Email string `xml:"CONTACT_EMAIL"`
Domain string `xml:"DOMAIN>DOMAIN_NAME"`
Country string `xml:"COUNTRY"`
CountryCode string `xml:"COUNTRY_CODE"`
Latitude float32 `xml:"LATITUDE"`
Longitude float32 `xml:"LONGITUDE"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// Sites is a list of Site objects.
type Sites struct {
Sites []*Site `xml:"SITE"`
}
// ServiceEndpoint represents an additional service endpoint of a service in GOCDB.
type ServiceEndpoint struct {
Name string `xml:"NAME"`
URL string `xml:"URL"`
Type string `xml:"INTERFACENAME"`
IsMonitored string `xml:"ENDPOINT_MONITORED"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// ServiceEndpoints is a list of ServiceEndpoint objects.
type ServiceEndpoints struct {
Endpoints []*ServiceEndpoint `xml:"ENDPOINT"`
}
// Service represents a service in GOCDB.
type Service struct {
Host string `xml:"HOSTNAME"`
Type string `xml:"SERVICE_TYPE"`
IsMonitored string `xml:"NODE_MONITORED"`
URL string `xml:"URL"`
Endpoints ServiceEndpoints `xml:"ENDPOINTS"`
Extensions Extensions `xml:"EXTENSIONS"`
}
// Services is a list of Service objects.
type Services struct {
Services []*Service `xml:"SERVICE_ENDPOINT"`
}
// DowntimeService represents a service scheduled for downtime.
type DowntimeService struct {
Type string `xml:"SERVICE_TYPE"`
}
// DowntimeServices represents a list of DowntimeService objects.
type DowntimeServices struct {
Services []*DowntimeService `xml:"SERVICE"`
}
// Downtime is a scheduled downtime for a site.
type Downtime struct {
Severity string `xml:"SEVERITY"`
StartDate int64 `xml:"START_DATE"`
EndDate int64 `xml:"END_DATE"`
AffectedServices DowntimeServices `xml:"SERVICES"`
}
// Downtimes represents a list of Downtime objects.
type Downtimes struct {
Downtimes []*Downtime `xml:"DOWNTIME"`
}
@@ -0,0 +1,64 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package entity
import (
"fmt"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/rs/zerolog"
)
// Collection is an interface for entity collections.
type Collection interface {
// Entities returns a vector of entities within the collection.
Entities() []Entity
}
// ActivateEntities activates the given entities.
func ActivateEntities(collection Collection, conf *config.Configuration, log *zerolog.Logger) error {
for _, exchanger := range collection.Entities() {
if err := exchanger.Activate(conf, log); err != nil {
return fmt.Errorf("unable to activate entity '%v': %v", exchanger.GetName(), err)
}
}
return nil
}
// GetIDs gets a list of entity IDs.
func GetIDs(collection Collection) []string {
entities := collection.Entities()
ids := make([]string, 0, len(entities))
for _, entity := range entities {
ids = append(ids, entity.GetID())
}
return ids
}
// GetNames gets a list of entity names.
func GetNames(collection Collection) []string {
entities := collection.Entities()
names := make([]string, 0, len(entities))
for _, entity := range entities {
names = append(names, entity.GetName())
}
return names
}
@@ -0,0 +1,36 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package entity
import (
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/rs/zerolog"
)
// Entity is the base interface for all Mentix entities.
type Entity interface {
// GetID returns the ID of the entity.
GetID() string
// GetName returns the display name of the entity.
GetName() string
// Activate activates the entity.
Activate(conf *config.Configuration, log *zerolog.Logger) error
}
@@ -0,0 +1,57 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package entity
import "fmt"
// Registry represents a simple id->entity map.
type Registry struct {
Entities map[string]Entity
}
// Register registers a new entity.
func (r *Registry) Register(entity Entity) {
r.Entities[entity.GetID()] = entity
}
// FindEntities returns all entities matching the provided IDs.
// If an entity with a certain ID doesn't exist and mustExist is true, an error is returned.
func (r *Registry) FindEntities(ids []string, mustExist bool, anyRequired bool) ([]Entity, error) {
var entities []Entity
for _, id := range ids {
if entity, ok := r.Entities[id]; ok {
entities = append(entities, entity)
} else if mustExist {
return nil, fmt.Errorf("no entity with ID '%v' registered", id)
}
}
if anyRequired && len(entities) == 0 { // At least one entity must be configured
return nil, fmt.Errorf("no entities available")
}
return entities, nil
}
// NewRegistry returns a new entity registry.
func NewRegistry() *Registry {
return &Registry{
Entities: make(map[string]Entity),
}
}
@@ -0,0 +1,171 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exchangers
import (
"fmt"
"strings"
"sync"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// Exchanger is the base interface for importers and exporters.
type Exchanger interface {
entity.Entity
// Start starts the exchanger; only exchangers which perform periodical background tasks should do something here.
Start() error
// Stop stops any running background activities of the exchanger.
Stop()
// MeshData returns the mesh data.
MeshData() *meshdata.MeshData
// Update is called whenever the mesh data set has changed to reflect these changes.
Update(meshdata.Map) error
}
// BaseExchanger implements basic exchanger functionality common to all exchangers.
type BaseExchanger struct {
Exchanger
conf *config.Configuration
log *zerolog.Logger
enabledConnectors []string
meshData *meshdata.MeshData
locker sync.RWMutex
}
// Activate activates the exchanger.
func (exchanger *BaseExchanger) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if conf == nil {
return fmt.Errorf("no configuration provided")
}
exchanger.conf = conf
if log == nil {
return fmt.Errorf("no logger provided")
}
exchanger.log = log
return nil
}
// Start starts the exchanger; only exchangers which perform periodical background tasks should do something here.
func (exchanger *BaseExchanger) Start() error {
return nil
}
// Stop stops any running background activities of the exchanger.
func (exchanger *BaseExchanger) Stop() {
}
// IsConnectorEnabled checks if the given connector is enabled for the exchanger.
func (exchanger *BaseExchanger) IsConnectorEnabled(id string) bool {
for _, connectorID := range exchanger.enabledConnectors {
if connectorID == "*" || strings.EqualFold(connectorID, id) {
return true
}
}
return false
}
// Update is called whenever the mesh data set has changed to reflect these changes.
func (exchanger *BaseExchanger) Update(meshDataSet meshdata.Map) error {
// Update the stored mesh data set
if err := exchanger.storeMeshDataSet(meshDataSet); err != nil {
return fmt.Errorf("unable to store the mesh data: %v", err)
}
return nil
}
func (exchanger *BaseExchanger) storeMeshDataSet(meshDataSet meshdata.Map) error {
// Store the new mesh data set by cloning it and then merging the cloned data into one object
meshDataSetCloned := make(meshdata.Map)
for connectorID, meshData := range meshDataSet {
if !exchanger.IsConnectorEnabled(connectorID) {
continue
}
meshDataCloned := meshData.Clone()
if meshDataCloned == nil {
return fmt.Errorf("unable to clone the mesh data")
}
meshDataSetCloned[connectorID] = meshDataCloned
}
exchanger.setMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned))
return nil
}
func (exchanger *BaseExchanger) cloneMeshData() *meshdata.MeshData {
exchanger.locker.RLock()
meshDataClone := exchanger.meshData.Clone()
exchanger.locker.RUnlock()
return meshDataClone
}
// Config returns the configuration object.
func (exchanger *BaseExchanger) Config() *config.Configuration {
return exchanger.conf
}
// Log returns the logger object.
func (exchanger *BaseExchanger) Log() *zerolog.Logger {
return exchanger.log
}
// EnabledConnectors returns the list of all enabled connectors for the exchanger.
func (exchanger *BaseExchanger) EnabledConnectors() []string {
return exchanger.enabledConnectors
}
// SetEnabledConnectors sets the list of all enabled connectors for the exchanger.
func (exchanger *BaseExchanger) SetEnabledConnectors(connectors []string) {
exchanger.enabledConnectors = connectors
}
// MeshData returns the stored mesh data. The returned data is cloned to prevent accidental data changes.
// Unauthorized sites are also removed if this exchanger doesn't allow them.
func (exchanger *BaseExchanger) MeshData() *meshdata.MeshData {
return exchanger.cloneMeshData()
}
func (exchanger *BaseExchanger) setMeshData(meshData *meshdata.MeshData) {
exchanger.locker.Lock()
defer exchanger.locker.Unlock()
exchanger.meshData = meshData
}
// Locker returns the locking object.
func (exchanger *BaseExchanger) Locker() *sync.RWMutex {
return &exchanger.locker
}
@@ -0,0 +1,79 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exchangers
import (
"fmt"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
)
// Collection is an interface for exchanger collections.
type Collection interface {
entity.Collection
// Exchangers returns a vector of exchangers within the collection.
Exchangers() []Exchanger
}
type entityCollectionWrapper struct {
entities []entity.Entity
}
func (collection *entityCollectionWrapper) Entities() []entity.Entity {
return collection.entities
}
// AsEntityCollection transforms an exchanger collection into an entity collection.
func AsEntityCollection(collection Collection) entity.Collection {
wrapper := entityCollectionWrapper{}
for _, exchanger := range collection.Exchangers() {
wrapper.entities = append(wrapper.entities, exchanger)
}
return &wrapper
}
// StartExchangers starts the given exchangers.
func StartExchangers(collection Collection) error {
for _, exchanger := range collection.Exchangers() {
if err := exchanger.Start(); err != nil {
return fmt.Errorf("unable to start exchanger '%v': %v", exchanger.GetName(), err)
}
}
return nil
}
// StopExchangers stops the given exchangers.
func StopExchangers(collection Collection) {
for _, exchanger := range collection.Exchangers() {
exchanger.Stop()
}
}
// GetRequestExchangers gets all exchangers from a vector that implement the RequestExchanger interface.
func GetRequestExchangers(collection Collection) []RequestExchanger {
var reqExchangers []RequestExchanger
for _, exporter := range collection.Exchangers() {
if reqExchanger, ok := exporter.(RequestExchanger); ok {
reqExchangers = append(reqExchangers, reqExchanger)
}
}
return reqExchangers
}
@@ -0,0 +1,60 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters/cs3api"
)
// CS3APIExporter implements the CS3API exporter.
type CS3APIExporter struct {
BaseRequestExporter
}
// Activate activates the exporter.
func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil {
return err
}
// Store CS3API specifics
exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint, conf.Exporters.CS3API.IsProtected)
exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors)
exporter.RegisterActionHandler("", cs3api.HandleDefaultQuery)
return nil
}
// GetID returns the ID of the exporter.
func (exporter *CS3APIExporter) GetID() string {
return config.ExporterIDCS3API
}
// GetName returns the display name of the exporter.
func (exporter *CS3APIExporter) GetName() string {
return "CS3API"
}
func init() {
registerExporter(&CS3APIExporter{})
}
@@ -0,0 +1,113 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package cs3api
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// HandleDefaultQuery processes a basic query.
func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (int, []byte, error) {
// Convert the mesh data
ocmData, err := convertMeshDataToOCMData(meshData, conf.Exporters.CS3API.ElevatedServiceTypes)
if err != nil {
return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err)
}
// Marshal the OCM data as JSON
data, err := json.MarshalIndent(ocmData, "", "\t")
if err != nil {
return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the OCM data: %v", err)
}
return http.StatusOK, data, nil
}
func convertMeshDataToOCMData(meshData *meshdata.MeshData, elevatedServiceTypes []string) ([]*ocmprovider.ProviderInfo, error) {
// Convert the mesh data into the corresponding OCM data structures
providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Sites))
for _, site := range meshData.Sites {
// Gather all services from the site
services := make([]*ocmprovider.Service, 0, len(site.Services))
addService := func(host string, endpoint *meshdata.ServiceEndpoint, addEndpoints []*ocmprovider.ServiceEndpoint, apiVersion string) {
services = append(services, &ocmprovider.Service{
Host: host,
Endpoint: convertServiceEndpointToOCMData(endpoint),
AdditionalEndpoints: addEndpoints,
ApiVersion: apiVersion,
})
}
for _, service := range site.Services {
apiVersion := meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, "")
// Gather all additional endpoints of the service
addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints))
for _, endpoint := range service.AdditionalEndpoints {
if utils.FindInStringArray(endpoint.Type.Name, elevatedServiceTypes, false) != -1 {
endpointURL, _ := url.Parse(endpoint.URL)
addService(endpointURL.Host, endpoint, nil, apiVersion)
} else {
addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint))
}
}
addService(service.Host, service.ServiceEndpoint, addEndpoints, apiVersion)
}
// Copy the site info into a ProviderInfo
providers = append(providers, &ocmprovider.ProviderInfo{
Name: site.Name,
FullName: site.FullName,
Description: site.Description,
Organization: site.Organization,
Domain: site.Domain,
Homepage: site.Homepage,
Email: site.Email,
Services: services,
Properties: site.Properties,
})
}
return providers, nil
}
func convertServiceEndpointToOCMData(endpoint *meshdata.ServiceEndpoint) *ocmprovider.ServiceEndpoint {
return &ocmprovider.ServiceEndpoint{
Type: &ocmprovider.ServiceType{
Name: endpoint.Type.Name,
Description: endpoint.Type.Description,
},
Name: endpoint.Name,
Path: endpoint.URL,
IsMonitored: endpoint.IsMonitored,
Properties: endpoint.Properties,
}
}
@@ -0,0 +1,41 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// Exporter is the interface that all exporters must implement.
type Exporter interface {
exchangers.Exchanger
}
// BaseExporter implements basic exporter functionality common to all exporters.
type BaseExporter struct {
exchangers.BaseExchanger
}
// Start starts the exporter.
func (exporter *BaseExporter) Start() error {
// Initialize the exporter with empty data
_ = exporter.Update(meshdata.Map{})
return nil
}
@@ -0,0 +1,90 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
)
// Collection represents a collection of exporters.
type Collection struct {
Exporters []Exporter
}
var (
registeredExporters = entity.NewRegistry()
)
// Entities returns a vector of entities within the collection.
func (collection *Collection) Entities() []entity.Entity {
return exchangers.AsEntityCollection(collection).Entities()
}
// Exchangers returns a vector of exchangers within the collection.
func (collection *Collection) Exchangers() []exchangers.Exchanger {
exchngrs := make([]exchangers.Exchanger, 0, len(collection.Exporters))
for _, connector := range collection.Exporters {
exchngrs = append(exchngrs, connector)
}
return exchngrs
}
// ActivateAll activates all exporters.
func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error {
return entity.ActivateEntities(collection, conf, log)
}
// StartAll starts all exporters.
func (collection *Collection) StartAll() error {
return exchangers.StartExchangers(collection)
}
// StopAll stops all exporters.
func (collection *Collection) StopAll() {
exchangers.StopExchangers(collection)
}
// GetRequestExporters returns all exporters that implement the RequestExchanger interface.
func (collection *Collection) GetRequestExporters() []exchangers.RequestExchanger {
return exchangers.GetRequestExchangers(collection)
}
// AvailableExporters returns a list of all exporters that are enabled in the configuration.
func AvailableExporters(conf *config.Configuration) (*Collection, error) {
// Try to add all exporters configured in the environment
entries, err := registeredExporters.FindEntities(conf.EnabledExporters, true, false)
if err != nil {
return nil, err
}
exporters := make([]Exporter, 0, len(entries))
for _, entry := range entries {
exporters = append(exporters, entry.(Exporter))
}
return &Collection{Exporters: exporters}, nil
}
func registerExporter(exporter Exporter) {
registeredExporters.Register(exporter)
}
@@ -0,0 +1,84 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters/metrics"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
"github.com/pkg/errors"
"github.com/rs/zerolog"
)
// MetricsExporter exposes various Prometheus metrics.
type MetricsExporter struct {
BaseExporter
metrics *metrics.Metrics
}
// Activate activates the exporter.
func (exporter *MetricsExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := exporter.BaseExporter.Activate(conf, log); err != nil {
return err
}
// Create the metrics handler
m, err := metrics.New(conf, log)
if err != nil {
return errors.Wrap(err, "unable to create metrics")
}
exporter.metrics = m
// Store Metrics specifics
exporter.SetEnabledConnectors(conf.Exporters.Metrics.EnabledConnectors)
return nil
}
// Update is called whenever the mesh data set has changed to reflect these changes.
func (exporter *MetricsExporter) Update(meshDataSet meshdata.Map) error {
if err := exporter.BaseExporter.Update(meshDataSet); err != nil {
return err
}
// Data is read, so acquire a read lock
exporter.Locker().RLock()
defer exporter.Locker().RUnlock()
if err := exporter.metrics.Update(exporter.MeshData()); err != nil {
return errors.Wrap(err, "error while updating the metrics")
}
return nil
}
// GetID returns the ID of the exporter.
func (exporter *MetricsExporter) GetID() string {
return config.ExporterIDMetrics
}
// GetName returns the display name of the exporter.
func (exporter *MetricsExporter) GetName() string {
return "Metrics"
}
func init() {
registerExporter(&MetricsExporter{})
}
@@ -0,0 +1,122 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package metrics
import (
"context"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
)
// Metrics exposes various Mentix related metrics via Prometheus.
type Metrics struct {
conf *config.Configuration
log *zerolog.Logger
isScheduledStats *stats.Int64Measure
}
const (
keySiteID = "site_id"
keySiteName = "site"
keyServiceType = "service_type"
)
func (m *Metrics) initialize(conf *config.Configuration, log *zerolog.Logger) error {
if conf == nil {
return errors.Errorf("no configuration provided")
}
m.conf = conf
if log == nil {
return errors.Errorf("no logger provided")
}
m.log = log
if err := m.registerMetrics(); err != nil {
return errors.Wrap(err, "error while registering metrics")
}
return nil
}
func (m *Metrics) registerMetrics() error {
// Create the OpenCensus statistics and a corresponding view
m.isScheduledStats = stats.Int64("site_is_scheduled", "A boolean metric which shows whether the given site is currently scheduled or not", stats.UnitDimensionless)
isScheduledView := &view.View{
Name: m.isScheduledStats.Name(),
Description: m.isScheduledStats.Description(),
Measure: m.isScheduledStats,
TagKeys: []tag.Key{tag.MustNewKey(keySiteID), tag.MustNewKey(keySiteName), tag.MustNewKey(keyServiceType)},
Aggregation: view.LastValue(),
}
if err := view.Register(isScheduledView); err != nil {
return errors.Wrap(err, "unable to register the site schedule status metrics view")
}
return nil
}
// Update is used to update/expose all metrics.
func (m *Metrics) Update(meshData *meshdata.MeshData) error {
for _, site := range meshData.Sites {
if err := m.exportSiteMetrics(site); err != nil {
return errors.Wrapf(err, "error while exporting metrics for site '%v'", site.Name)
}
}
return nil
}
func (m *Metrics) exportSiteMetrics(site *meshdata.Site) error {
mutators := []tag.Mutator{
tag.Insert(tag.MustNewKey(keySiteID), site.ID),
tag.Insert(tag.MustNewKey(keySiteName), site.Name),
tag.Insert(tag.MustNewKey(keyServiceType), "SCIENCEMESH_HCHECK"),
}
// Create a new context to serve the metrics
if ctx, err := tag.New(context.Background(), mutators...); err == nil {
isScheduled := int64(1)
if site.Downtimes.IsAnyActive() {
isScheduled = 0
}
stats.Record(ctx, m.isScheduledStats.M(isScheduled))
} else {
return errors.Wrap(err, "unable to create a context for the site schedule status metrics")
}
return nil
}
// New creates a new Metrics instance.
func New(conf *config.Configuration, log *zerolog.Logger) (*Metrics, error) {
m := &Metrics{}
if err := m.initialize(conf, log); err != nil {
return nil, errors.Wrap(err, "unable to create new metrics object")
}
return m, nil
}
@@ -0,0 +1,25 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package prometheus
// ScrapeConfig represents a scrape configuration in PrometheusSD.
type ScrapeConfig struct {
Targets []string `json:"targets"`
Labels map[string]string `json:"labels"`
}
@@ -0,0 +1,238 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"encoding/json"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters/prometheus"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
type prometheusSDScrapeCreatorCallback = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig
type prometheusSDScrapeCreator struct {
outputFilename string
creatorCallback prometheusSDScrapeCreatorCallback
serviceFilter []string
}
// PrometheusSDExporter implements various Prometheus Service Discovery scrape config exporters.
type PrometheusSDExporter struct {
BaseExporter
scrapeCreators map[string]prometheusSDScrapeCreator
}
const (
labelSiteName = "__meta_mentix_site"
labelSiteID = "__meta_mentix_site_id"
labelSiteCountry = "__meta_mentix_site_country"
labelType = "__meta_mentix_type"
labelURL = "__meta_mentix_url"
labelScheme = "__meta_mentix_scheme"
labelHost = "__meta_mentix_host"
labelPort = "__meta_mentix_port"
labelPath = "__meta_mentix_path"
labelServiceHost = "__meta_mentix_service_host"
labelServiceURL = "__meta_mentix_service_url"
)
func createGenericScrapeConfig(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig {
endpointURL, _ := url.Parse(endpoint.URL)
labels := getScrapeTargetLabels(site, service, endpoint)
return &prometheus.ScrapeConfig{
Targets: []string{endpointURL.Host},
Labels: labels,
}
}
func getScrapeTargetLabels(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) map[string]string {
endpointURL, _ := url.Parse(endpoint.URL)
labels := map[string]string{
labelSiteName: site.Name,
labelSiteID: site.ID,
labelSiteCountry: site.CountryCode,
labelType: endpoint.Type.Name,
labelURL: endpoint.URL,
labelScheme: endpointURL.Scheme,
labelHost: endpointURL.Hostname(),
labelPort: endpointURL.Port(),
labelPath: endpointURL.Path,
labelServiceHost: service.Host,
labelServiceURL: service.URL,
}
return labels
}
func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Configuration) error {
exporter.scrapeCreators = make(map[string]prometheusSDScrapeCreator)
registerCreator := func(name string, outputFilename string, creator prometheusSDScrapeCreatorCallback, serviceFilter []string) error {
if len(outputFilename) > 0 { // Only register the creator if an output filename was configured
exporter.scrapeCreators[name] = prometheusSDScrapeCreator{
outputFilename: outputFilename,
creatorCallback: creator,
serviceFilter: serviceFilter,
}
// Create the output directory for the target file so it exists when exporting
if err := os.MkdirAll(filepath.Dir(outputFilename), 0755); err != nil {
return fmt.Errorf("unable to create output directory tree: %v", err)
}
}
return nil
}
// Register all scrape creators
for _, endpoint := range meshdata.GetServiceEndpoints() {
epName := strings.ToLower(endpoint)
filename := path.Join(conf.Exporters.PrometheusSD.OutputPath, "svc_"+epName+".json")
if err := registerCreator(epName, filename, createGenericScrapeConfig, []string{endpoint}); err != nil {
return fmt.Errorf("unable to register the '%v' scrape config creator: %v", epName, err)
}
}
return nil
}
// Activate activates the exporter.
func (exporter *PrometheusSDExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := exporter.BaseExporter.Activate(conf, log); err != nil {
return err
}
if err := exporter.registerScrapeCreators(conf); err != nil {
return fmt.Errorf("unable to register the scrape creators: %v", err)
}
// Create all output directories
for _, creator := range exporter.scrapeCreators {
if err := os.MkdirAll(filepath.Dir(creator.outputFilename), 0755); err != nil {
return fmt.Errorf("unable to create directory tree: %v", err)
}
}
// Store PrometheusSD specifics
exporter.SetEnabledConnectors(conf.Exporters.PrometheusSD.EnabledConnectors)
return nil
}
// Update is called whenever the mesh data set has changed to reflect these changes.
func (exporter *PrometheusSDExporter) Update(meshDataSet meshdata.Map) error {
if err := exporter.BaseExporter.Update(meshDataSet); err != nil {
return err
}
// Perform exporting the data asynchronously
go exporter.exportMeshData()
return nil
}
func (exporter *PrometheusSDExporter) exportMeshData() {
// Data is read, so acquire a read lock
exporter.Locker().RLock()
defer exporter.Locker().RUnlock()
for name, creator := range exporter.scrapeCreators {
scrapes := exporter.createScrapeConfigs(creator.creatorCallback, creator.serviceFilter)
if err := exporter.exportScrapeConfig(creator.outputFilename, scrapes); err != nil {
exporter.Log().Err(err).Str("kind", name).Str("file", creator.outputFilename).Msg("error exporting Prometheus SD scrape config")
} else {
exporter.Log().Debug().Str("kind", name).Str("file", creator.outputFilename).Msg("exported Prometheus SD scrape config")
}
}
}
func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback prometheusSDScrapeCreatorCallback, serviceFilter []string) []*prometheus.ScrapeConfig {
var scrapes []*prometheus.ScrapeConfig
var addScrape = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) {
if len(serviceFilter) == 0 || utils.FindInStringArray(endpoint.Type.Name, serviceFilter, false) != -1 {
if scrape := creatorCallback(site, service, endpoint); scrape != nil {
scrapes = append(scrapes, scrape)
}
}
}
// Create a scrape config for each service alongside any additional endpoints
for _, site := range exporter.MeshData().Sites {
for _, service := range site.Services {
if !service.IsMonitored {
continue
}
// Add the "main" service to the scrapes
addScrape(site, service, service.ServiceEndpoint)
// Add all additional endpoints as well
for _, endpoint := range service.AdditionalEndpoints {
if endpoint.IsMonitored {
addScrape(site, service, endpoint)
}
}
}
}
if scrapes == nil {
scrapes = []*prometheus.ScrapeConfig{}
}
return scrapes
}
func (exporter *PrometheusSDExporter) exportScrapeConfig(outputFilename string, v interface{}) error {
// Encode scrape config as JSON
data, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return fmt.Errorf("unable to marshal scrape config: %v", err)
}
// Write the data to disk
if err := os.WriteFile(outputFilename, data, 0755); err != nil {
return fmt.Errorf("unable to write scrape config '%v': %v", outputFilename, err)
}
return nil
}
// GetID returns the ID of the exporter.
func (exporter *PrometheusSDExporter) GetID() string {
return config.ExporterIDPrometheusSD
}
// GetName returns the display name of the exporter.
func (exporter *PrometheusSDExporter) GetName() string {
return "Prometheus SD"
}
func init() {
registerExporter(&PrometheusSDExporter{})
}
@@ -0,0 +1,52 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"io"
"net/http"
"net/url"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
)
// BaseRequestExporter implements basic exporter functionality common to all request exporters.
type BaseRequestExporter struct {
BaseExporter
exchangers.BaseRequestExchanger
}
// HandleRequest handles the actual HTTP request.
func (exporter *BaseRequestExporter) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) {
body, _ := io.ReadAll(req.Body)
status, respData, err := exporter.handleQuery(body, req.URL.Query(), conf, log)
if err != nil {
respData = []byte(err.Error())
}
resp.WriteHeader(status)
_, _ = resp.Write(respData)
}
func (exporter *BaseRequestExporter) handleQuery(body []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (int, []byte, error) {
_, status, data, err := exporter.HandleAction(exporter.MeshData(), body, params, false, conf, log)
return status, data, err
}
@@ -0,0 +1,63 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package siteloc
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// HandleDefaultQuery processes a basic query.
func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) {
// Convert the mesh data
locData, err := convertMeshDataToLocationData(meshData)
if err != nil {
return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to location data: %v", err)
}
// Marshal the location data as JSON
data, err := json.MarshalIndent(locData, "", "\t")
if err != nil {
return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the location data: %v", err)
}
return http.StatusOK, data, nil
}
func convertMeshDataToLocationData(meshData *meshdata.MeshData) ([]*SiteLocation, error) {
// Gather the locations of all sites
locations := make([]*SiteLocation, 0, len(meshData.Sites))
for _, site := range meshData.Sites {
locations = append(locations, &SiteLocation{
SiteID: site.ID,
FullName: site.FullName,
Longitude: site.Longitude,
Latitude: site.Latitude,
})
}
return locations, nil
}
@@ -0,0 +1,27 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package siteloc
// SiteLocation represents the location information of a site.
type SiteLocation struct {
SiteID string `json:"key"`
FullName string `json:"name"`
Longitude float32 `json:"longitude"`
Latitude float32 `json:"latitude"`
}
@@ -0,0 +1,60 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters/siteloc"
)
// SiteLocationsExporter implements the Site Locations exporter to use with Grafana.
type SiteLocationsExporter struct {
BaseRequestExporter
}
// Activate activates the exporter.
func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil {
return err
}
// Store SiteLocations specifics
exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint, conf.Exporters.SiteLocations.IsProtected)
exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors)
exporter.RegisterActionHandler("", siteloc.HandleDefaultQuery)
return nil
}
// GetID returns the ID of the exporter.
func (exporter *SiteLocationsExporter) GetID() string {
return config.ExporterIDSiteLocations
}
// GetName returns the display name of the exporter.
func (exporter *SiteLocationsExporter) GetName() string {
return "Site Locations"
}
func init() {
registerExporter(&SiteLocationsExporter{})
}
@@ -0,0 +1,60 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exporters
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters/webapi"
)
// WebAPIExporter implements the generic Web API exporter.
type WebAPIExporter struct {
BaseRequestExporter
}
// Activate activates the exporter.
func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error {
if err := exporter.BaseRequestExporter.Activate(conf, log); err != nil {
return err
}
// Store WebAPI specifics
exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint, conf.Exporters.WebAPI.IsProtected)
exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors)
exporter.RegisterActionHandler("", webapi.HandleDefaultQuery)
return nil
}
// GetID returns the ID of the exporter.
func (exporter *WebAPIExporter) GetID() string {
return config.ExporterIDWebAPI
}
// GetName returns the display name of the exporter.
func (exporter *WebAPIExporter) GetName() string {
return "WebAPI"
}
func init() {
registerExporter(&WebAPIExporter{})
}
@@ -0,0 +1,42 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package webapi
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// HandleDefaultQuery processes a basic query.
func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) {
// Just return the plain, unfiltered data as JSON
data, err := json.MarshalIndent(meshData, "", "\t")
if err != nil {
return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err)
}
return http.StatusOK, data, nil
}
@@ -0,0 +1,94 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package importers
import (
"errors"
"fmt"
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/mentix/connectors"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// Importer is the interface that all importers must implement.
type Importer interface {
exchangers.Exchanger
// Process is called periodically to perform the actual import; if data has been imported, true is returned.
Process(*connectors.Collection) (bool, error)
}
// BaseImporter implements basic importer functionality common to all importers.
type BaseImporter struct {
exchangers.BaseExchanger
meshDataUpdates meshdata.Vector
updatesLocker sync.RWMutex
}
// Process is called periodically to perform the actual import; if data has been imported, true is returned.
func (importer *BaseImporter) Process(connectors *connectors.Collection) (bool, error) {
if importer.meshDataUpdates == nil { // No data present for updating, so nothing to process
return false, nil
}
var processErrs []string
// Data is read, so lock it for writing during the loop
importer.updatesLocker.RLock()
for _, connector := range connectors.Connectors {
if !importer.IsConnectorEnabled(connector.GetID()) {
continue
}
if err := importer.processMeshDataUpdates(connector); err != nil {
processErrs = append(processErrs, fmt.Sprintf("unable to process imported mesh data for connector '%v': %v", connector.GetName(), err))
}
}
importer.updatesLocker.RUnlock()
importer.setMeshDataUpdates(nil)
var err error
if len(processErrs) != 0 {
err = errors.New(strings.Join(processErrs, "; "))
}
return true, err
}
func (importer *BaseImporter) processMeshDataUpdates(connector connectors.Connector) error {
for _, meshData := range importer.meshDataUpdates {
if err := connector.UpdateMeshData(meshData); err != nil {
return fmt.Errorf("error while updating mesh data: %v", err)
}
}
return nil
}
func (importer *BaseImporter) setMeshDataUpdates(meshDataUpdates meshdata.Vector) {
importer.updatesLocker.Lock()
defer importer.updatesLocker.Unlock()
importer.meshDataUpdates = meshDataUpdates
}
@@ -0,0 +1,93 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package importers
import (
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
)
// Collection represents a collection of importers.
type Collection struct {
Importers []Importer
}
var (
registeredImporters = entity.NewRegistry()
)
// Entities returns a vector of entities within the collection.
func (collection *Collection) Entities() []entity.Entity {
return exchangers.AsEntityCollection(collection).Entities()
}
// Exchangers returns a vector of exchangers within the collection.
func (collection *Collection) Exchangers() []exchangers.Exchanger {
exchngrs := make([]exchangers.Exchanger, 0, len(collection.Importers))
for _, connector := range collection.Importers {
exchngrs = append(exchngrs, connector)
}
return exchngrs
}
// ActivateAll activates all importers.
func (collection *Collection) ActivateAll(conf *config.Configuration, log *zerolog.Logger) error {
return entity.ActivateEntities(collection, conf, log)
}
// StartAll starts all importers.
func (collection *Collection) StartAll() error {
return exchangers.StartExchangers(collection)
}
// StopAll stops all importers.
func (collection *Collection) StopAll() {
exchangers.StopExchangers(collection)
}
// GetRequestImporters returns all importers that implement the RequestExchanger interface.
func (collection *Collection) GetRequestImporters() []exchangers.RequestExchanger {
return exchangers.GetRequestExchangers(collection)
}
// AvailableImporters returns a collection of all importers that are enabled in the configuration.
func AvailableImporters(conf *config.Configuration) (*Collection, error) {
// Try to add all importers configured in the environment
entities, err := registeredImporters.FindEntities(conf.EnabledImporters, true, false)
if err != nil {
return nil, err
}
importers := make([]Importer, 0, len(entities))
for _, entry := range entities {
importers = append(importers, entry.(Importer))
}
return &Collection{Importers: importers}, nil
}
// TODO: Uncomment once an importer is actually implemented
/*
func registerImporter(importer Importer) {
registeredImporters.Register(importer)
}
*/
@@ -0,0 +1,73 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package importers
import (
"io"
"net/http"
"net/url"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// BaseRequestImporter implements basic importer functionality common to all request importers.
type BaseRequestImporter struct {
BaseImporter
exchangers.BaseRequestExchanger
}
// HandleRequest handles the actual HTTP request.
func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) {
body, _ := io.ReadAll(req.Body)
meshDataSet, status, respData, err := importer.handleQuery(body, req.URL.Query(), conf, log)
if err == nil {
if len(meshDataSet) > 0 {
importer.mergeImportedMeshDataSet(meshDataSet)
}
} else {
respData = []byte(err.Error())
}
resp.WriteHeader(status)
_, _ = resp.Write(respData)
}
func (importer *BaseRequestImporter) mergeImportedMeshDataSet(meshDataSet meshdata.Vector) {
// Merge the newly imported data with any existing data stored in the importer
if importer.meshDataUpdates != nil {
// Need to manually lock the data for writing
importer.updatesLocker.Lock()
defer importer.updatesLocker.Unlock()
importer.meshDataUpdates = append(importer.meshDataUpdates, meshDataSet...)
} else {
importer.setMeshDataUpdates(meshDataSet) // SetMeshData will do the locking itself
}
}
func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) {
// Data is read, so lock it for writing
importer.Locker().RLock()
defer importer.Locker().RUnlock()
return importer.HandleAction(importer.MeshData(), data, params, true, conf, log)
}
@@ -0,0 +1,126 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package exchangers
import (
"fmt"
"net/http"
"net/url"
"strings"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// RequestExchanger is the interface implemented by exchangers that offer an HTTP endpoint.
type RequestExchanger interface {
// Endpoint returns the (relative) endpoint of the exchanger.
Endpoint() string
// IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization.
IsProtectedEndpoint() bool
// WantsRequest returns whether the exchanger wants to handle the incoming request.
WantsRequest(r *http.Request) bool
// HandleRequest handles the actual HTTP request.
HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger)
}
type queryCallback func(*meshdata.MeshData, url.Values, *config.Configuration, *zerolog.Logger) (int, []byte, error)
type extendedQueryCallback func(*meshdata.MeshData, []byte, url.Values, *config.Configuration, *zerolog.Logger) (meshdata.Vector, int, []byte, error)
// BaseRequestExchanger implements basic exporter functionality common to all request exporters.
type BaseRequestExchanger struct {
RequestExchanger
endpoint string
isProtectedEndpoint bool
actionHandlers map[string]queryCallback
extendedActionHandlers map[string]extendedQueryCallback
}
// Endpoint returns the (relative) endpoint of the exchanger.
func (exchanger *BaseRequestExchanger) Endpoint() string {
// Ensure that the endpoint starts with a /
endpoint := exchanger.endpoint
if !strings.HasPrefix(endpoint, "/") {
endpoint = "/" + endpoint
}
return strings.TrimSpace(endpoint)
}
// IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization.
func (exchanger *BaseRequestExchanger) IsProtectedEndpoint() bool {
return exchanger.isProtectedEndpoint
}
// SetEndpoint sets the (relative) endpoint of the exchanger.
func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string, isProtected bool) {
exchanger.endpoint = endpoint
exchanger.isProtectedEndpoint = isProtected
}
// WantsRequest returns whether the exchanger wants to handle the incoming request.
func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool {
return r.URL.Path == exchanger.Endpoint()
}
// HandleRequest handles the actual HTTP request.
func (exchanger *BaseRequestExchanger) HandleRequest(resp http.ResponseWriter, req *http.Request, conf *config.Configuration, log *zerolog.Logger) error {
return nil
}
// RegisterActionHandler registers a new handler for the specified action.
func (exchanger *BaseRequestExchanger) RegisterActionHandler(action string, callback queryCallback) {
if exchanger.actionHandlers == nil {
exchanger.actionHandlers = make(map[string]queryCallback)
}
exchanger.actionHandlers[action] = callback
}
// RegisterExtendedActionHandler registers a new handler for the specified extended action.
func (exchanger *BaseRequestExchanger) RegisterExtendedActionHandler(action string, callback extendedQueryCallback) {
if exchanger.extendedActionHandlers == nil {
exchanger.extendedActionHandlers = make(map[string]extendedQueryCallback)
}
exchanger.extendedActionHandlers[action] = callback
}
// HandleAction executes the registered handler for the specified action, if any.
func (exchanger *BaseRequestExchanger) HandleAction(meshData *meshdata.MeshData, body []byte, params url.Values, isExtended bool, conf *config.Configuration, log *zerolog.Logger) (meshdata.Vector, int, []byte, error) {
reqAction := params.Get("action")
if isExtended {
for action, handler := range exchanger.extendedActionHandlers {
if strings.EqualFold(action, reqAction) {
return handler(meshData, body, params, conf, log)
}
}
} else {
for action, handler := range exchanger.actionHandlers {
if strings.EqualFold(action, reqAction) {
status, data, err := handler(meshData, params, conf, log)
return nil, status, data, err
}
}
}
return nil, http.StatusNotFound, []byte{}, fmt.Errorf("unhandled query for action '%v'", reqAction)
}
+335
View File
@@ -0,0 +1,335 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package mentix
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/opencloud-eu/reva/v2/pkg/appctx"
"github.com/opencloud-eu/reva/v2/pkg/mentix/accservice"
"github.com/opencloud-eu/reva/v2/pkg/mentix/config"
"github.com/opencloud-eu/reva/v2/pkg/mentix/connectors"
"github.com/opencloud-eu/reva/v2/pkg/mentix/entity"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/exporters"
"github.com/opencloud-eu/reva/v2/pkg/mentix/exchangers/importers"
"github.com/opencloud-eu/reva/v2/pkg/mentix/meshdata"
)
// Mentix represents the main Mentix service object.
type Mentix struct {
conf *config.Configuration
log *zerolog.Logger
connectors *connectors.Collection
importers *importers.Collection
exporters *exporters.Collection
meshDataSet meshdata.Map
updateInterval time.Duration
}
const (
runLoopSleeptime = time.Millisecond * 1000
)
func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) error {
if conf == nil {
return fmt.Errorf("no configuration provided")
}
mntx.conf = conf
if log == nil {
return fmt.Errorf("no logger provided")
}
mntx.log = log
// Initialize the connectors that will be used to gather the mesh data
if err := mntx.initConnectors(); err != nil {
return fmt.Errorf("unable to initialize connector: %v", err)
}
// Initialize the exchangers
if err := mntx.initExchangers(); err != nil {
return fmt.Errorf("unable to initialize exchangers: %v", err)
}
// Get the update interval
duration, err := time.ParseDuration(mntx.conf.UpdateInterval)
if err != nil {
// If the duration can't be parsed, default to one hour
duration = time.Hour
}
mntx.updateInterval = duration
// Create empty mesh data set
mntx.meshDataSet = make(meshdata.Map)
// Log some infos
connectorNames := entity.GetNames(mntx.connectors)
importerNames := entity.GetNames(mntx.importers)
exporterNames := entity.GetNames(mntx.exporters)
log.Info().Msgf("mentix started with connectors: %v; importers: %v; exporters: %v; update interval: %v",
strings.Join(connectorNames, ", "),
strings.Join(importerNames, ", "),
strings.Join(exporterNames, ", "),
duration,
)
return nil
}
func (mntx *Mentix) initConnectors() error {
// Use all connectors exposed by the connectors package
conns, err := connectors.AvailableConnectors(mntx.conf)
if err != nil {
return fmt.Errorf("unable to get registered conns: %v", err)
}
mntx.connectors = conns
if err := mntx.connectors.ActivateAll(mntx.conf, mntx.log); err != nil {
return fmt.Errorf("unable to activate connectors: %v", err)
}
return nil
}
func (mntx *Mentix) initExchangers() error {
// Use all importers exposed by the importers package
imps, err := importers.AvailableImporters(mntx.conf)
if err != nil {
return fmt.Errorf("unable to get registered importers: %v", err)
}
mntx.importers = imps
if err := mntx.importers.ActivateAll(mntx.conf, mntx.log); err != nil {
return fmt.Errorf("unable to activate importers: %v", err)
}
// Use all exporters exposed by the exporters package
exps, err := exporters.AvailableExporters(mntx.conf)
if err != nil {
return fmt.Errorf("unable to get registered exporters: %v", err)
}
mntx.exporters = exps
if err := mntx.exporters.ActivateAll(mntx.conf, mntx.log); err != nil {
return fmt.Errorf("unable to activate exporters: %v", err)
}
return nil
}
func (mntx *Mentix) startExchangers() error {
// Start all importers
if err := mntx.importers.StartAll(); err != nil {
return fmt.Errorf("unable to start importers: %v", err)
}
// Start all exporters
if err := mntx.exporters.StartAll(); err != nil {
return fmt.Errorf("unable to start exporters: %v", err)
}
return nil
}
func (mntx *Mentix) stopExchangers() {
mntx.exporters.StopAll()
mntx.importers.StopAll()
}
func (mntx *Mentix) destroy() {
mntx.stopExchangers()
}
// Run starts the Mentix service that will periodically pull the configured data source and publish this data
// through the enabled exporters.
func (mntx *Mentix) Run(stopSignal <-chan struct{}) error {
defer mntx.destroy()
// Start all im- & exporters; they will be stopped in mntx.destroy
if err := mntx.startExchangers(); err != nil {
return fmt.Errorf("unable to start exchangers: %v", err)
}
updateTimestamp := time.Time{}
loop:
for {
if stopSignal != nil {
// Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully
select {
case <-stopSignal:
break loop
default:
}
}
// Perform all regular actions
mntx.tick(&updateTimestamp)
time.Sleep(runLoopSleeptime)
}
return nil
}
func (mntx *Mentix) tick(updateTimestamp *time.Time) {
// Let all importers do their work first
meshDataUpdated, err := mntx.processImporters()
if err != nil {
mntx.log.Err(err).Msgf("an error occurred while processing the importers: %v", err)
}
// If mesh data has been imported or enough time has passed, update the stored mesh data and all exporters
if meshDataUpdated || time.Since(*updateTimestamp) >= mntx.updateInterval {
// Retrieve and update the mesh data; if the importers modified any data, these changes will
// be reflected automatically here
if meshDataSet, err := mntx.retrieveMeshDataSet(); err == nil {
if err := mntx.applyMeshDataSet(meshDataSet); err != nil {
mntx.log.Err(err).Msg("failed to apply mesh data")
}
} else {
mntx.log.Err(err).Msg("failed to retrieve mesh data")
}
*updateTimestamp = time.Now()
}
}
func (mntx *Mentix) processImporters() (bool, error) {
meshDataUpdated := false
for _, importer := range mntx.importers.Importers {
updated, err := importer.Process(mntx.connectors)
if err != nil {
return false, fmt.Errorf("unable to process importer '%v': %v", importer.GetName(), err)
}
meshDataUpdated = meshDataUpdated || updated
if updated {
mntx.log.Debug().Msgf("mesh data imported from '%v'", importer.GetName())
}
}
return meshDataUpdated, nil
}
func (mntx *Mentix) retrieveMeshDataSet() (meshdata.Map, error) {
meshDataSet := make(meshdata.Map)
for _, connector := range mntx.connectors.Connectors {
meshData, err := connector.RetrieveMeshData()
if err == nil {
meshDataSet[connector.GetID()] = meshData
} else {
mntx.log.Err(err).Msgf("retrieving mesh data from connector '%v' failed", connector.GetName())
}
}
return meshDataSet, nil
}
func (mntx *Mentix) applyMeshDataSet(meshDataSet meshdata.Map) error {
// Check if mesh data from any connector has changed
meshDataChanged := false
for connectorID, meshData := range meshDataSet {
if !meshData.Compare(mntx.meshDataSet[connectorID]) {
meshDataChanged = true
break
}
}
if meshDataChanged {
mntx.log.Debug().Msg("mesh data changed, applying")
mntx.meshDataSet = meshDataSet
exchangers := make([]exchangers.Exchanger, 0, len(mntx.exporters.Exporters)+len(mntx.importers.Importers))
exchangers = append(exchangers, mntx.exporters.Exchangers()...)
exchangers = append(exchangers, mntx.importers.Exchangers()...)
for _, exchanger := range exchangers {
if err := exchanger.Update(mntx.meshDataSet); err != nil {
return fmt.Errorf("unable to update mesh data on exchanger '%v': %v", exchanger.GetName(), err)
}
}
}
return nil
}
// GetRequestImporters returns all exporters that can handle HTTP requests.
func (mntx *Mentix) GetRequestImporters() []exchangers.RequestExchanger {
return mntx.importers.GetRequestImporters()
}
// GetRequestExporters returns all exporters that can handle HTTP requests.
func (mntx *Mentix) GetRequestExporters() []exchangers.RequestExchanger {
return mntx.exporters.GetRequestExporters()
}
// RequestHandler handles any incoming HTTP requests by asking each RequestExchanger whether it wants to
// handle the request (usually based on the relative URL path).
func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
log := appctx.GetLogger(r.Context())
switch r.Method {
case http.MethodGet:
mntx.handleRequest(mntx.GetRequestExporters(), w, r, log)
case http.MethodPost:
mntx.handleRequest(mntx.GetRequestImporters(), w, r, log)
default:
log.Err(fmt.Errorf("unsupported method")).Msg("error handling incoming request")
}
}
func (mntx *Mentix) handleRequest(exchangers []exchangers.RequestExchanger, w http.ResponseWriter, r *http.Request, log *zerolog.Logger) {
// Ask each RequestExchanger if it wants to handle the request
for _, exchanger := range exchangers {
if exchanger.WantsRequest(r) {
exchanger.HandleRequest(w, r, mntx.conf, log)
}
}
}
// New creates a new Mentix service instance.
func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) {
// Configure the accounts service upfront
if err := accservice.InitAccountsService(conf); err != nil {
return nil, fmt.Errorf("unable to initialize the accounts service: %v", err)
}
mntx := new(Mentix)
if err := mntx.initialize(conf, log); err != nil {
return nil, fmt.Errorf("unable to initialize Mentix: %v", err)
}
return mntx, nil
}
@@ -0,0 +1,124 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
import (
"time"
"github.com/pkg/errors"
)
// Downtimes represents all scheduled downtimes of a site.
type Downtimes struct {
Downtimes []*Downtime
}
// ScheduleDowntime schedules a new downtime.
func (dts *Downtimes) ScheduleDowntime(start time.Time, end time.Time, affectedServices []string) (*Downtime, error) {
// Create a new downtime and verify it
dt := &Downtime{
StartDate: start,
EndDate: end,
AffectedServices: affectedServices,
}
dt.InferMissingData()
if err := dt.Verify(); err != nil {
return nil, err
}
// Only schedule the downtime if it hasn't expired yet
if dt.IsExpired() {
return nil, nil
}
dts.Downtimes = append(dts.Downtimes, dt)
return dt, nil
}
// Clear clears all downtimes.
func (dts *Downtimes) Clear() {
dts.Downtimes = make([]*Downtime, 0, 10)
}
// IsAnyActive returns true if any downtime is currently active.
func (dts *Downtimes) IsAnyActive() bool {
for _, dt := range dts.Downtimes {
if dt.IsActive() {
return true
}
}
return false
}
// InferMissingData infers missing data from other data where possible.
func (dts *Downtimes) InferMissingData() {
for _, dt := range dts.Downtimes {
dt.InferMissingData()
}
}
// Verify checks if the downtimes data is valid.
func (dts *Downtimes) Verify() error {
for _, dt := range dts.Downtimes {
if err := dt.Verify(); err != nil {
return err
}
}
return nil
}
// Downtime represents a single scheduled downtime.
type Downtime struct {
StartDate time.Time
EndDate time.Time
AffectedServices []string
}
// IsActive returns true if the downtime is currently active.
func (dt *Downtime) IsActive() bool {
now := time.Now()
return dt.StartDate.Before(now) && dt.EndDate.After(now)
}
// IsPending returns true if the downtime is yet to come.
func (dt *Downtime) IsPending() bool {
return dt.StartDate.After(time.Now())
}
// IsExpired returns true of the downtime has expired (i.e., lies in the past).
func (dt *Downtime) IsExpired() bool {
return dt.EndDate.Before(time.Now())
}
// InferMissingData infers missing data from other data where possible.
func (dt *Downtime) InferMissingData() {
}
// Verify checks if the downtime data is valid.
func (dt *Downtime) Verify() error {
if dt.EndDate.Before(dt.StartDate) {
return errors.Errorf("downtime end is before its start")
}
if len(dt.AffectedServices) == 0 {
return errors.Errorf("no services affected by downtime")
}
return nil
}
@@ -0,0 +1,48 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
const (
// EndpointRevad identifies the main Reva Daemon endpoint
EndpointRevad = "REVAD"
// EndpointGateway identifies the Gateway endpoint
EndpointGateway = "GATEWAY"
// EndpointMetrics identifies the Metrics endpoint
EndpointMetrics = "METRICS"
// EndpointWebdav identifies the Webdav endpoint
EndpointWebdav = "WEBDAV"
// EndpointOCM identifies the OCM endpoint
EndpointOCM = "OCM"
// EndpointMeshDir identifies the Mesh Directory endpoint
EndpointMeshDir = "MESHDIR"
)
// GetServiceEndpoints returns an array of all service endpoint identifiers.
func GetServiceEndpoints() []string {
return []string{
EndpointRevad,
EndpointGateway,
EndpointMetrics,
EndpointWebdav,
EndpointOCM,
EndpointMeshDir,
}
}
@@ -0,0 +1,219 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
import (
"bytes"
"encoding/gob"
"encoding/json"
"fmt"
"reflect"
"strings"
)
const (
// StatusDefault signals that this is just regular data.
StatusDefault = iota
// StatusObsolete flags the mesh data for removal.
StatusObsolete
)
// MeshData holds the entire mesh data managed by Mentix.
type MeshData struct {
Sites []*Site
ServiceTypes []*ServiceType
Status int `json:"-"`
}
// Clear removes all saved data, leaving an empty mesh.
func (meshData *MeshData) Clear() {
meshData.Sites = nil
meshData.ServiceTypes = nil
meshData.Status = StatusDefault
}
// AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten.
func (meshData *MeshData) AddSite(site *Site) {
if siteExisting := meshData.FindSite(site.ID); siteExisting != nil {
*siteExisting = *site
} else {
meshData.Sites = append(meshData.Sites, site)
}
}
// RemoveSite removes the provided site.
func (meshData *MeshData) RemoveSite(site *Site) {
for idx, siteExisting := range meshData.Sites {
if strings.EqualFold(siteExisting.ID, site.ID) { // Remove the site by its ID
lastIdx := len(meshData.Sites) - 1
meshData.Sites[idx] = meshData.Sites[lastIdx]
meshData.Sites[lastIdx] = nil
meshData.Sites = meshData.Sites[:lastIdx]
break
}
}
}
// FindSite searches for a site with the given ID.
func (meshData *MeshData) FindSite(id string) *Site {
for _, site := range meshData.Sites {
if strings.EqualFold(site.ID, id) {
return site
}
}
return nil
}
// AddServiceType adds a new service type; if a type with the same name already exists, the existing one is overwritten.
func (meshData *MeshData) AddServiceType(serviceType *ServiceType) {
if svcTypeExisting := meshData.FindServiceType(serviceType.Name); svcTypeExisting != nil {
*svcTypeExisting = *serviceType
} else {
meshData.ServiceTypes = append(meshData.ServiceTypes, serviceType)
}
}
// RemoveServiceType removes the provided service type.
func (meshData *MeshData) RemoveServiceType(serviceType *ServiceType) {
for idx, svcTypeExisting := range meshData.ServiceTypes {
if strings.EqualFold(svcTypeExisting.Name, serviceType.Name) { // Remove the service type by its name
lastIdx := len(meshData.ServiceTypes) - 1
meshData.ServiceTypes[idx] = meshData.ServiceTypes[lastIdx]
meshData.ServiceTypes[lastIdx] = nil
meshData.ServiceTypes = meshData.ServiceTypes[:lastIdx]
break
}
}
}
// FindServiceType searches for a service type with the given name.
func (meshData *MeshData) FindServiceType(name string) *ServiceType {
for _, serviceType := range meshData.ServiceTypes {
if strings.EqualFold(serviceType.Name, name) {
return serviceType
}
}
return nil
}
// Merge merges data from another MeshData instance into this one.
func (meshData *MeshData) Merge(inData *MeshData) {
for _, site := range inData.Sites {
meshData.AddSite(site)
}
for _, serviceType := range inData.ServiceTypes {
meshData.AddServiceType(serviceType)
}
}
// Unmerge removes data from another MeshData instance from this one.
func (meshData *MeshData) Unmerge(inData *MeshData) {
for _, site := range inData.Sites {
meshData.RemoveSite(site)
}
for _, serviceType := range inData.ServiceTypes {
meshData.RemoveServiceType(serviceType)
}
}
// Verify checks if the mesh data is valid.
func (meshData *MeshData) Verify() error {
// Verify all sites
for _, site := range meshData.Sites {
if err := site.Verify(); err != nil {
return err
}
}
// Verify all service types
for _, serviceType := range meshData.ServiceTypes {
if err := serviceType.Verify(); err != nil {
return err
}
}
return nil
}
// InferMissingData infers missing data from other data where possible.
func (meshData *MeshData) InferMissingData() {
// Infer missing site data
for _, site := range meshData.Sites {
site.InferMissingData()
}
// Infer missing service type data
for _, serviceType := range meshData.ServiceTypes {
serviceType.InferMissingData()
}
}
// ToJSON converts the data to JSON.
func (meshData *MeshData) ToJSON() (string, error) {
data, err := json.MarshalIndent(meshData, "", "\t")
if err != nil {
return "", fmt.Errorf("unable to marshal the mesh data: %v", err)
}
return string(data), nil
}
// FromJSON converts JSON data to mesh data.
func (meshData *MeshData) FromJSON(data string) error {
meshData.Clear()
if err := json.Unmarshal([]byte(data), meshData); err != nil {
return fmt.Errorf("unable to unmarshal the mesh data: %v", err)
}
return nil
}
// Clone creates an exact copy of the mesh data.
func (meshData *MeshData) Clone() *MeshData {
clone := &MeshData{}
// To avoid any "deep copy" packages, use gob en- and decoding instead
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
dec := gob.NewDecoder(&buf)
if err := enc.Encode(meshData); err == nil {
if err := dec.Decode(clone); err != nil {
// In case of an error, clear the data
clone.Clear()
}
}
return clone
}
// Compare checks whether the stored data equals the data of another MeshData object.
func (meshData *MeshData) Compare(other *MeshData) bool {
return reflect.DeepEqual(meshData, other)
}
// New returns a new (empty) MeshData object.
func New() *MeshData {
meshData := &MeshData{}
meshData.Clear()
return meshData
}
@@ -0,0 +1,52 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
import "strings"
const (
// PropertySiteID identifies the site ID property.
PropertySiteID = "site_id"
// PropertyOrganization identifies the organization property.
PropertyOrganization = "organization"
// PropertyAPIVersion identifies the API version property.
PropertyAPIVersion = "api_version"
)
// GetPropertyValue performs a case-insensitive search for the given property.
func GetPropertyValue(props map[string]string, id string, defValue string) string {
for key := range props {
if strings.EqualFold(key, id) {
return props[key]
}
}
return defValue
}
// SetPropertyValue sets a property value.
func SetPropertyValue(props *map[string]string, id string, value string) {
// If the provided properties map is nil, create an empty one
if *props == nil {
*props = make(map[string]string)
}
(*props)[strings.ToUpper(id)] = value
}
@@ -0,0 +1,120 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
import (
"fmt"
"net/url"
"strings"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils/network"
)
// Service represents a service managed by Mentix.
type Service struct {
*ServiceEndpoint
Host string
AdditionalEndpoints []*ServiceEndpoint
}
// FindEndpoint searches for an additional endpoint with the given name.
func (service *Service) FindEndpoint(name string) *ServiceEndpoint {
for _, endpoint := range service.AdditionalEndpoints {
if strings.EqualFold(endpoint.Name, name) {
return endpoint
}
}
return nil
}
// InferMissingData infers missing data from other data where possible.
func (service *Service) InferMissingData() {
service.ServiceEndpoint.InferMissingData()
// Infer missing data
if service.Host == "" {
if serviceURL, err := url.Parse(service.URL); err == nil {
service.Host = network.ExtractDomainFromURL(serviceURL, true)
}
}
}
// Verify checks if the service data is valid.
func (service *Service) Verify() error {
if err := service.ServiceEndpoint.Verify(); err != nil {
return err
}
return nil
}
// ServiceType represents a service type managed by Mentix.
type ServiceType struct {
Name string
Description string
}
// InferMissingData infers missing data from other data where possible.
func (serviceType *ServiceType) InferMissingData() {
// Infer missing data
if serviceType.Description == "" {
serviceType.Description = serviceType.Name
}
}
// Verify checks if the service type data is valid.
func (serviceType *ServiceType) Verify() error {
// Verify data
if serviceType.Name == "" {
return fmt.Errorf("service type name missing")
}
return nil
}
// ServiceEndpoint represents a service endpoint managed by Mentix.
type ServiceEndpoint struct {
Type *ServiceType
Name string
RawURL string
URL string
IsMonitored bool
Properties map[string]string
}
// InferMissingData infers missing data from other data where possible.
func (serviceEndpoint *ServiceEndpoint) InferMissingData() {
}
// Verify checks if the service endpoint data is valid.
func (serviceEndpoint *ServiceEndpoint) Verify() error {
if serviceEndpoint.Type == nil {
return fmt.Errorf("service endpoint type missing")
}
if serviceEndpoint.Name == "" {
return fmt.Errorf("service endpoint name missing")
}
if serviceEndpoint.URL == "" {
return fmt.Errorf("service endpoint URL missing")
}
return nil
}
@@ -0,0 +1,121 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
import (
"fmt"
"net/url"
"strings"
"github.com/opencloud-eu/reva/v2/pkg/mentix/utils/network"
)
// Site represents a single site managed by Mentix.
type Site struct {
// Internal settings
ID string
Name string
FullName string
Organization string
Domain string
Homepage string
Email string
Description string
Country string
CountryCode string
Location string
Latitude float32
Longitude float32
Services []*Service
Properties map[string]string
Downtimes Downtimes `json:"-"`
}
// AddService adds a new service; if a service with the same name already exists, the existing one is overwritten.
func (site *Site) AddService(service *Service) {
if serviceExisting := site.FindService(service.Name); serviceExisting != nil {
*service = *serviceExisting
} else {
site.Services = append(site.Services, service)
}
}
// RemoveService removes the provided service.
func (site *Site) RemoveService(name string) {
if service := site.FindService(name); service != nil {
for idx, serviceExisting := range site.Services {
if serviceExisting == service {
lastIdx := len(site.Services) - 1
site.Services[idx] = site.Services[lastIdx]
site.Services[lastIdx] = nil
site.Services = site.Services[:lastIdx]
break
}
}
}
}
// FindService searches for a service with the given name.
func (site *Site) FindService(name string) *Service {
for _, service := range site.Services {
if strings.EqualFold(service.Name, name) {
return service
}
}
return nil
}
// Verify checks if the site data is valid.
func (site *Site) Verify() error {
// Verify data
if site.Name == "" {
return fmt.Errorf("site name missing")
}
if site.Domain == "" && site.Homepage == "" {
return fmt.Errorf("site URL missing")
}
// Verify services
for _, service := range site.Services {
if err := service.Verify(); err != nil {
return err
}
}
return nil
}
// InferMissingData infers missing data from other data where possible.
func (site *Site) InferMissingData() {
// Infer missing data
if site.Homepage == "" {
site.Homepage = fmt.Sprintf("http://www.%v", site.Domain)
} else if site.Domain == "" {
if URL, err := url.Parse(site.Homepage); err == nil {
site.Domain = network.ExtractDomainFromURL(URL, false)
}
}
// Infer missing for services
for _, service := range site.Services {
service.InferMissingData()
}
}
@@ -0,0 +1,34 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package meshdata
// Vector represents a vector of MeshData objects.
type Vector = []*MeshData
// Map represents a map of MeshData objects.
type Map = map[string]*MeshData
// MergeMeshDataMap merges all mesh data objects within a map.
func MergeMeshDataMap(meshDataSet Map) *MeshData {
mergedMeshData := &MeshData{}
for _, meshData := range meshDataSet {
mergedMeshData.Merge(meshData)
}
return mergedMeshData
}
@@ -0,0 +1,131 @@
// Copyright 2018-2021 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package network
import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
p "path"
"strings"
)
// URLParams holds Key-Value URL parameters; it is a simpler form of url.Values.
type URLParams map[string]string
// ResponseParams holds parameters of an HTTP response.
type ResponseParams map[string]interface{}
// BasicAuth holds user credentials for basic HTTP authentication.
type BasicAuth struct {
User string
Password string
}
// GenerateURL creates a URL object from a host, path and optional parameters.
func GenerateURL(host string, path string, params URLParams) (*url.URL, error) {
fullURL, err := url.Parse(host)
if err != nil {
return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", host, path, params)
}
if len(fullURL.Scheme) == 0 {
fullURL.Scheme = "https"
}
fullURL.Path = p.Join(fullURL.Path, path)
if len(params) > 0 {
query := make(url.Values)
for key, value := range params {
query.Set(key, value)
}
fullURL.RawQuery = query.Encode()
}
return fullURL, nil
}
func queryEndpoint(method string, endpointURL *url.URL, auth *BasicAuth, checkStatus bool) ([]byte, error) {
// Prepare the request
req, err := http.NewRequest(method, endpointURL.String(), nil)
if err != nil {
return nil, fmt.Errorf("unable to create HTTP request: %v", err)
}
if auth != nil {
req.SetBasicAuth(auth.User, auth.Password)
}
// Fetch the data and read the body
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("unable to get data from endpoint: %v", err)
}
defer resp.Body.Close()
if checkStatus && resp.StatusCode >= 400 {
return nil, fmt.Errorf("invalid response received: %v", resp.Status)
}
body, _ := io.ReadAll(resp.Body)
return body, nil
}
// ReadEndpoint reads data from an HTTP endpoint via GET.
func ReadEndpoint(endpointURL *url.URL, auth *BasicAuth, checkStatus bool) ([]byte, error) {
return queryEndpoint(http.MethodGet, endpointURL, auth, checkStatus)
}
// WriteEndpoint sends data to an HTTP endpoint via POST.
func WriteEndpoint(endpointURL *url.URL, auth *BasicAuth, checkStatus bool) ([]byte, error) {
return queryEndpoint(http.MethodPost, endpointURL, auth, checkStatus)
}
// CreateResponse creates a generic HTTP response in JSON format.
func CreateResponse(msg string, params ResponseParams) []byte {
if params == nil {
params = make(map[string]interface{})
}
params["message"] = msg
jsonData, _ := json.MarshalIndent(params, "", "\t")
return jsonData
}
// ExtractDomainFromURL extracts the domain name (domain.tld or subdomain.domain.tld) from a URL.
func ExtractDomainFromURL(hostURL *url.URL, keepSubdomain bool) string {
// Remove host port if present
host, _, err := net.SplitHostPort(hostURL.Host)
if err != nil {
host = hostURL.Host
}
if !keepSubdomain {
// Remove subdomain
if idx := strings.Index(host, "."); idx != -1 {
host = host[idx+1:]
}
}
return host
}
@@ -0,0 +1,31 @@
// Copyright 2018-2020 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package utils
import "strings"
// FindInStringArray searches for the given string in the given string array.
func FindInStringArray(s string, arr []string, caseSensitive bool) int {
for i, a := range arr {
if (caseSensitive && a == s) || (!caseSensitive && strings.EqualFold(a, s)) {
return i
}
}
return -1
}