Initial QSfera import

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/config/config.go

@@ -0,0 +1,47 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package config
+
+// Config holds the config options that need to be passed down to the metrics reader(driver)
+type Config struct {
+	MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"`
+	MetricsDataLocation   string `mapstructure:"metrics_data_location"`
+	MetricsRecordInterval int    `mapstructure:"metrics_record_interval"`
+	XcloudInstance        string `mapstructure:"xcloud_instance"`
+	XcloudPullInterval    int    `mapstructure:"xcloud_pull_interval"`
+	InsecureSkipVerify    bool   `mapstructure:"insecure_skip_verify"`
+}
+
+// Init sets sane defaults
+func (c *Config) Init() {
+	if c.MetricsDataDriverType == "json" {
+		// default values
+		if c.MetricsDataLocation == "" {
+			c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json"
+		}
+	}
+
+	if c.MetricsRecordInterval == 0 {
+		c.MetricsRecordInterval = 5000
+	}
+
+	if c.XcloudPullInterval == 0 {
+		c.XcloudPullInterval = 5
+	}
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/dummy/dummy.go

@@ -0,0 +1,60 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package dummy
+
+import (
+	"math/rand"
+
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/config"
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/driver/registry"
+)
+
+func init() {
+	driver := &MetricsDummyDriver{}
+	registry.Register(driverName(), driver)
+}
+
+func driverName() string {
+	return "dummy"
+}
+
+// MetricsDummyDriver the MetricsDummyDriver struct
+type MetricsDummyDriver struct {
+}
+
+// Configure configures this driver
+func (d *MetricsDummyDriver) Configure(c *config.Config) error {
+	// no configuration necessary
+	return nil
+}
+
+// GetNumUsers returns the number of site users; it's a random number
+func (d *MetricsDummyDriver) GetNumUsers() int64 {
+	return int64(rand.Intn(30000))
+}
+
+// GetNumGroups returns the number of site groups; it's a random number
+func (d *MetricsDummyDriver) GetNumGroups() int64 {
+	return int64(rand.Intn(200))
+}
+
+// GetAmountStorage returns the amount of site storage used; it's a random amount
+func (d *MetricsDummyDriver) GetAmountStorage() int64 {
+	return rand.Int63n(70000000000)
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/json/json.go

@@ -0,0 +1,97 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/driver/registry"
+
+	"github.com/opencloud-eu/reva/v2/pkg/logger"
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/config"
+	"github.com/rs/zerolog"
+)
+
+var log zerolog.Logger
+
+func init() {
+	log = logger.New().With().Int("pid", os.Getpid()).Logger()
+	driver := &MetricsJSONDriver{}
+	registry.Register(driverName(), driver)
+}
+
+func driverName() string {
+	return "json"
+}
+
+// readJSON always returns a data object but logs the error in case reading the json fails.
+func readJSON(driver *MetricsJSONDriver) *data {
+	data := &data{}
+
+	file, err := os.ReadFile(driver.metricsDataLocation)
+	if err != nil {
+		log.Error().Err(err).Str("location", driver.metricsDataLocation).Msg("Unable to read json file from location.")
+	}
+	err = json.Unmarshal(file, data)
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to unmarshall json file.")
+	}
+
+	return data
+}
+
+type data struct {
+	NumUsers      int64 `json:"cs3_org_sciencemesh_site_total_num_users"`
+	NumGroups     int64 `json:"cs3_org_sciencemesh_site_total_num_groups"`
+	AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"`
+}
+
+// MetricsJSONDriver the JsonDriver struct
+type MetricsJSONDriver struct {
+	metricsDataLocation string
+}
+
+// Configure configures this driver
+func (d *MetricsJSONDriver) Configure(c *config.Config) error {
+	if c.MetricsDataLocation == "" {
+		err := errors.New("unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?")
+		return err
+	}
+
+	d.metricsDataLocation = c.MetricsDataLocation
+
+	return nil
+}
+
+// GetNumUsers returns the number of site users
+func (d *MetricsJSONDriver) GetNumUsers() int64 {
+	return readJSON(d).NumUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *MetricsJSONDriver) GetNumGroups() int64 {
+	return readJSON(d).NumGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *MetricsJSONDriver) GetAmountStorage() int64 {
+	return readJSON(d).AmountStorage
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/loader/loader.go

@@ -0,0 +1,27 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package loader
+
+import (
+	// Load metrics drivers.
+	_ "github.com/opencloud-eu/reva/v2/pkg/metrics/driver/dummy"
+	_ "github.com/opencloud-eu/reva/v2/pkg/metrics/driver/json"
+	_ "github.com/opencloud-eu/reva/v2/pkg/metrics/driver/xcloud"
+	// Add your own here
+)

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/registry/registry.go

@@ -0,0 +1,42 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package registry
+
+import (
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/reader"
+)
+
+var drivers map[string]reader.Reader // map key is driver type name
+
+// Register register a driver
+func Register(driverName string, r reader.Reader) {
+	if drivers == nil {
+		drivers = make(map[string]reader.Reader)
+	}
+	drivers[driverName] = r
+}
+
+// GetDriver returns the registered driver for the specified driver name, or nil if it is not registered
+func GetDriver(driverName string) reader.Reader {
+	driver, found := drivers[driverName]
+	if found {
+		return driver
+	}
+	return nil
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/driver/xcloud/xcloud.go

@@ -0,0 +1,180 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/driver/registry"
+
+	"github.com/rs/zerolog"
+
+	"github.com/opencloud-eu/reva/v2/pkg/logger"
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/config"
+)
+
+var log zerolog.Logger
+
+func init() {
+	log = logger.New().With().Int("pid", os.Getpid()).Logger()
+	driver := &CloudDriver{CloudData: &CloudData{}}
+	registry.Register(driverName(), driver)
+}
+
+func driverName() string {
+	return "xcloud"
+}
+
+// CloudDriver is the driver to use for Sciencemesh apps
+type CloudDriver struct {
+	instance     string
+	pullInterval int
+	CloudData    *CloudData
+	sync.Mutex
+	client *http.Client
+}
+
+func (d *CloudDriver) refresh() error {
+	// endpoint example: https://mybox.com
+	endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance)
+
+	req, err := http.NewRequest("GET", endpoint, nil)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error creating request to %s", d.instance)
+		return err
+	}
+
+	resp, err := d.client.Do(req)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", d.instance, resp.StatusCode)
+		log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+		return err
+	}
+	defer resp.Body.Close()
+
+	// read response body
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", d.instance)
+		return err
+	}
+
+	cd := &CloudData{}
+	if err := json.Unmarshal(data, cd); err != nil {
+		log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data))
+		return err
+	}
+
+	d.Lock()
+	defer d.Unlock()
+	d.CloudData = cd
+	log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd)
+
+	return nil
+}
+
+// Configure configures this driver
+func (d *CloudDriver) Configure(c *config.Config) error {
+	if c.XcloudInstance == "" {
+		err := errors.New("xcloud: missing xcloud_instance config parameter")
+		return err
+	}
+
+	if c.XcloudPullInterval == 0 {
+		c.XcloudPullInterval = 5 // seconds
+	}
+
+	d.instance = c.XcloudInstance
+	d.pullInterval = c.XcloudPullInterval
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: c.InsecureSkipVerify},
+	}
+	client := &http.Client{Transport: tr}
+
+	d.client = client
+
+	ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second)
+	quit := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				err := d.refresh()
+				if err != nil {
+					log.Err(err).Msgf("xcloud: error from refresh goroutine")
+				}
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
+// GetNumUsers returns the number of site users
+func (d *CloudDriver) GetNumUsers() int64 {
+	return d.CloudData.Metrics.TotalUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *CloudDriver) GetNumGroups() int64 {
+	return d.CloudData.Metrics.TotalGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *CloudDriver) GetAmountStorage() int64 {
+	return d.CloudData.Metrics.TotalStorage
+}
+
+// CloudData represents the information obtained from the sciencemesh app
+type CloudData struct {
+	Metrics  CloudDataMetrics  `json:"metrics"`
+	Settings CloudDataSettings `json:"settings"`
+}
+
+// CloudDataMetrics reprents the metrics gathered from the sciencemesh app
+type CloudDataMetrics struct {
+	TotalUsers   int64 `json:"numusers"`
+	TotalGroups  int64 `json:"numgroups"`
+	TotalStorage int64 `json:"numstorage"`
+}
+
+// CloudDataSettings represents the metrics gathered
+type CloudDataSettings struct {
+	IOPUrl   string `json:"iopurl"`
+	Sitename string `json:"sitename"`
+	Siteurl  string `json:"siteurl"`
+	Country  string `json:"country"`
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/metrics.go

@@ -0,0 +1,147 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package metrics
+
+/*
+Metrics registers OpenCensus data views of the metrics.
+Metrics initializes the driver as specified in the configuration.
+*/
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/config"
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/driver/registry"
+	"github.com/opencloud-eu/reva/v2/pkg/metrics/reader"
+
+	"github.com/opencloud-eu/reva/v2/pkg/logger"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+)
+
+// Init intializes metrics according to the specified configuration
+func Init(conf *config.Config) error {
+	log := logger.New().With().Int("pid", os.Getpid()).Logger()
+
+	driver := registry.GetDriver(conf.MetricsDataDriverType)
+
+	if driver == nil {
+		log.Info().Msg("No metrics are being recorded.")
+		// No error but just don't proceed with metrics
+		return nil
+	}
+
+	// configure the driver
+	err := driver.Configure(conf)
+	if err != nil {
+		return err
+	}
+
+	m := &Metrics{
+		dataDriver:           driver,
+		NumUsersMeasure:      stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless),
+		NumGroupsMeasure:     stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless),
+		AmountStorageMeasure: stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes),
+	}
+
+	if err := view.Register(
+		m.getNumUsersView(),
+		m.getNumGroupsView(),
+		m.getAmountStorageView(),
+	); err != nil {
+		return err
+	}
+
+	// periodically record metrics data
+	go func() {
+		for {
+			if err := m.recordMetrics(); err != nil {
+				log.Error().Err(err).Msg("Metrics recording failed.")
+			}
+			<-time.After(time.Millisecond * time.Duration(conf.MetricsRecordInterval))
+		}
+	}()
+
+	return nil
+}
+
+// Metrics the metrics struct
+type Metrics struct {
+	dataDriver           reader.Reader // the metrics data driver is an implemention of Reader
+	NumUsersMeasure      *stats.Int64Measure
+	NumGroupsMeasure     *stats.Int64Measure
+	AmountStorageMeasure *stats.Int64Measure
+}
+
+// RecordMetrics records the latest metrics from the metrics data source as OpenCensus stats views.
+func (m *Metrics) recordMetrics() error {
+	// record all latest metrics
+	if m.dataDriver != nil {
+		m.recordNumUsers()
+		m.recordNumGroups()
+		m.recordAmountStorage()
+	}
+	return nil
+}
+
+// recordNumUsers records the latest number of site users figure
+func (m *Metrics) recordNumUsers() {
+	ctx := context.Background()
+	stats.Record(ctx, m.NumUsersMeasure.M(m.dataDriver.GetNumUsers()))
+}
+
+func (m *Metrics) getNumUsersView() *view.View {
+	return &view.View{
+		Name:        m.NumUsersMeasure.Name(),
+		Description: m.NumUsersMeasure.Description(),
+		Measure:     m.NumUsersMeasure,
+		Aggregation: view.LastValue(),
+	}
+}
+
+// recordNumGroups records the latest number of site groups figure
+func (m *Metrics) recordNumGroups() {
+	ctx := context.Background()
+	stats.Record(ctx, m.NumGroupsMeasure.M(m.dataDriver.GetNumGroups()))
+}
+
+func (m *Metrics) getNumGroupsView() *view.View {
+	return &view.View{
+		Name:        m.NumGroupsMeasure.Name(),
+		Description: m.NumGroupsMeasure.Description(),
+		Measure:     m.NumGroupsMeasure,
+		Aggregation: view.LastValue(),
+	}
+}
+
+// recordAmountStorage records the latest amount storage figure
+func (m *Metrics) recordAmountStorage() {
+	ctx := context.Background()
+	stats.Record(ctx, m.AmountStorageMeasure.M(m.dataDriver.GetAmountStorage()))
+}
+
+func (m *Metrics) getAmountStorageView() *view.View {
+	return &view.View{
+		Name:        m.AmountStorageMeasure.Name(),
+		Description: m.AmountStorageMeasure.Description(),
+		Measure:     m.AmountStorageMeasure,
+		Aggregation: view.LastValue(),
+	}
+}

Server/vendor/github.com/opencloud-eu/reva/v2/pkg/metrics/reader/reader.go

@@ -0,0 +1,49 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package reader
+
+/*
+Reader is the interface that defines the metrics to read.
+Any metrics data driver must implement this interface.
+Each metric function should return the current/latest available metrics figure relevant to that function.
+*/
+
+import "github.com/opencloud-eu/reva/v2/pkg/metrics/config"
+
+// Reader the Reader interface
+type Reader interface {
+
+	// Configure configures the reader according to the specified configuration
+	Configure(c *config.Config) error
+
+	// GetNumUsersView returns an OpenCensus stats view which records the
+	// number of users registered in the mesh provider.
+	// Metric name: cs3_org_sciencemesh_site_total_num_users
+	GetNumUsers() int64
+
+	// GetNumGroupsView returns an OpenCensus stats view which records the
+	// number of user groups registered in the mesh provider.
+	// Metric name: cs3_org_sciencemesh_site_total_num_groups
+	GetNumGroups() int64
+
+	// GetAmountStorageView returns an OpenCensus stats view which records the
+	// amount of storage in the system.
+	// Metric name: cs3_org_sciencemesh_site_total_amount_storage
+	GetAmountStorage() int64
+}