sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[full-ci] enhancement: use reva client pool selectors (#6452)

Browse Source 
* enhancement: use reva client pool selectors

register mock service to registry and pass tests

* enhancement: bump reva

* Fix a couple of linter issues

---------

Co-authored-by: Ralf Haferkamp <rhaferkamp@owncloud.com>
This commit is contained in:
Florian Schade
2023-06-08 12:41:04 +02:00
committed by GitHub
co-authored by Ralf Haferkamp
parent 021c9fcdd9
commit 4f26424db6
157 changed files with 2845 additions and 1901 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/userlog/pkg/command/server.go
+5 -3
View File
@@ -11,6 +11,7 @@ import (
"github.com/oklog/run"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
"github.com/owncloud/ocis/v2/ocis-pkg/handlers"
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/service/debug"
ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
"github.com/owncloud/ocis/v2/ocis-pkg/version"
@@ -90,13 +91,14 @@ func Server(cfg *config.Config) *cli.Command {
if err != nil {
return err
}
gwclient, err := pool.GetGatewayServiceClient(
gatewaySelector, err := pool.GatewaySelector(
cfg.RevaGateway,
pool.WithTLSCACert(cfg.GRPCClientTLS.CACert),
pool.WithTLSMode(tm),
pool.WithRegistry(registry.GetRegistry()),
)
if err != nil {
return fmt.Errorf("could not get reva client: %s", err)
return fmt.Errorf("could not get reva client selector: %s", err)
}
hClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", ogrpc.DefaultClient())
@@ -109,7 +111,7 @@ func Server(cfg *config.Config) *cli.Command {
http.Metrics(mtrcs),
http.Store(st),
http.Consumer(consumer),
http.Gateway(gwclient),
http.GatewaySelector(gatewaySelector),
http.History(hClient),
http.RegisteredEvents(_registeredEvents),
)
services/userlog/pkg/server/http/option.go
+5 -4
View File
@@ -5,6 +5,7 @@ import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
"github.com/owncloud/ocis/v2/services/userlog/pkg/config"
@@ -26,7 +27,7 @@ type Options struct {
Namespace string
Store store.Store
Consumer events.Consumer
GatewayClient gateway.GatewayAPIClient
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
HistoryClient ehsvc.EventHistoryService
RegisteredEvents []events.Unmarshaller
}
@@ -98,10 +99,10 @@ func Consumer(consumer events.Consumer) Option {
}
}
// Gateway provides a function to configure the gateway client
func Gateway(gw gateway.GatewayAPIClient) Option {
// GatewaySelector provides a function to configure the gateway client selector
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewayClient = gw
o.GatewaySelector = gatewaySelector
}
}
services/userlog/pkg/server/http/server.go
+1 -1
View File
@@ -75,7 +75,7 @@ func Server(opts ...Option) (http.Service, error) {
svc.Store(options.Store),
svc.Config(options.Config),
svc.HistoryClient(options.HistoryClient),
svc.GatewayClient(options.GatewayClient),
svc.GatewaySelector(options.GatewaySelector),
svc.RegisteredEvents(options.RegisteredEvents),
)
if err != nil {
services/userlog/pkg/service/conversion.go
+8 -7
View File
@@ -16,6 +16,7 @@ import (
collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/leonelquinteros/gotext"
@@ -52,7 +53,7 @@ type OC10Notification struct {
// Converter is responsible for converting eventhistory events to OC10Notifications
type Converter struct {
locale string
gwClient gateway.GatewayAPIClient
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
machineAuthAPIKey string
serviceName string
registeredEvents map[string]events.Unmarshaller
@@ -66,10 +67,10 @@ type Converter struct {
}
// NewConverter returns a new Converter
func NewConverter(loc string, gwc gateway.GatewayAPIClient, machineAuthAPIKey string, name string, translationPath string, registeredEvents map[string]events.Unmarshaller) *Converter {
func NewConverter(loc string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], machineAuthAPIKey string, name string, translationPath string, registeredEvents map[string]events.Unmarshaller) *Converter {
return &Converter{
locale: loc,
gwClient: gwc,
gatewaySelector: gatewaySelector,
machineAuthAPIKey: machineAuthAPIKey,
serviceName: name,
registeredEvents: registeredEvents,
@@ -306,7 +307,7 @@ func (c *Converter) authenticate(usr *user.User) (context.Context, error) {
if ctx, ok := c.contexts[usr.GetId().GetOpaqueId()]; ok {
return ctx, nil
}
ctx, err := authenticate(usr, c.gwClient, c.machineAuthAPIKey)
ctx, err := authenticate(usr, c.gatewaySelector, c.machineAuthAPIKey)
if err == nil {
c.contexts[usr.GetId().GetOpaqueId()] = ctx
}
@@ -317,7 +318,7 @@ func (c *Converter) getSpace(ctx context.Context, spaceID string) (*storageprovi
if space, ok := c.spaces[spaceID]; ok {
return space, nil
}
space, err := getSpace(ctx, spaceID, c.gwClient)
space, err := getSpace(ctx, spaceID, c.gatewaySelector)
if err == nil {
c.spaces[spaceID] = space
}
@@ -328,7 +329,7 @@ func (c *Converter) getResource(ctx context.Context, resourceID *storageprovider
if r, ok := c.resources[resourceID.GetOpaqueId()]; ok {
return r, nil
}
resource, err := getResource(ctx, resourceID, c.gwClient)
resource, err := getResource(ctx, resourceID, c.gatewaySelector)
if err == nil {
c.resources[resourceID.GetOpaqueId()] = resource
}
@@ -339,7 +340,7 @@ func (c *Converter) getUser(ctx context.Context, userID *user.UserId) (*user.Use
if u, ok := c.users[userID.GetOpaqueId()]; ok {
return u, nil
}
usr, err := getUser(ctx, userID, c.gwClient)
usr, err := getUser(ctx, userID, c.gatewaySelector)
if err == nil {
c.users[userID.GetOpaqueId()] = usr
}
services/userlog/pkg/service/http.go
+1 -1
View File
@@ -31,7 +31,7 @@ func (ul *UserlogService) HandleGetEvents(w http.ResponseWriter, r *http.Request
return
}
conv := NewConverter(r.Header.Get(HeaderAcceptLanguage), ul.gwClient, ul.cfg.MachineAuthAPIKey, ul.cfg.Service.Name, ul.cfg.TranslationPath, ul.registeredEvents)
conv := NewConverter(r.Header.Get(HeaderAcceptLanguage), ul.gatewaySelector, ul.cfg.MachineAuthAPIKey, ul.cfg.Service.Name, ul.cfg.TranslationPath, ul.registeredEvents)
resp := GetEventResponseOC10{}
for _, e := range evs {
services/userlog/pkg/service/options.go
+5 -4
View File
@@ -3,6 +3,7 @@ package service
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/go-chi/chi/v5"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
@@ -21,7 +22,7 @@ type Options struct {
Store store.Store
Config *config.Config
HistoryClient ehsvc.EventHistoryService
GatewayClient gateway.GatewayAPIClient
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
RegisteredEvents []events.Unmarshaller
}
@@ -67,10 +68,10 @@ func HistoryClient(hc ehsvc.EventHistoryService) Option {
}
}
// GatewayClient adds a grpc client for the gateway service
func GatewayClient(gwc gateway.GatewayAPIClient) Option {
// GatewaySelector adds a grpc client selector for the gateway service
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewayClient = gwc
o.GatewaySelector = gatewaySelector
}
}
services/userlog/pkg/service/service.go
+42 -16
View File
@@ -14,6 +14,7 @@ import (
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/go-chi/chi/v5"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
@@ -31,7 +32,7 @@ type UserlogService struct {
store store.Store
cfg *config.Config
historyClient ehsvc.EventHistoryService
gwClient gateway.GatewayAPIClient
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
registeredEvents map[string]events.Unmarshaller
translationPath string
}
@@ -58,7 +59,7 @@ func NewUserlogService(opts ...Option) (*UserlogService, error) {
store: o.Store,
cfg: o.Config,
historyClient: o.HistoryClient,
gwClient: o.GatewayClient,
gatewaySelector: o.GatewaySelector,
registeredEvents: make(map[string]events.Unmarshaller),
}
@@ -283,7 +284,7 @@ func (ul *UserlogService) findSpaceMembers(ctx context.Context, spaceID string,
return nil, errors.New("need authenticated context to find space members")
}
space, err := getSpace(ctx, spaceID, ul.gwClient)
space, err := getSpace(ctx, spaceID, ul.gatewaySelector)
if err != nil {
return nil, err
}
@@ -361,7 +362,7 @@ func (ul *UserlogService) resolveID(ctx context.Context, userid *user.UserId, gr
// resolves the users of a group
func (ul *UserlogService) resolveGroup(ctx context.Context, groupID string) ([]string, error) {
grp, err := getGroup(ctx, groupID, ul.gwClient)
grp, err := getGroup(ctx, groupID, ul.gatewaySelector)
if err != nil {
return nil, err
}
@@ -380,13 +381,13 @@ func (ul *UserlogService) impersonate(uid *user.UserId) context.Context {
return nil
}
u, err := getUser(context.Background(), uid, ul.gwClient)
u, err := getUser(context.Background(), uid, ul.gatewaySelector)
if err != nil {
ul.log.Error().Err(err).Msg("cannot get user")
return nil
}
ctx, err := authenticate(u, ul.gwClient, ul.cfg.MachineAuthAPIKey)
ctx, err := authenticate(u, ul.gatewaySelector, ul.cfg.MachineAuthAPIKey)
if err != nil {
ul.log.Error().Err(err).Str("userid", u.GetId().GetOpaqueId()).Msg("failed to impersonate user")
return nil
@@ -394,9 +395,14 @@ func (ul *UserlogService) impersonate(uid *user.UserId) context.Context {
return ctx
}
func authenticate(usr *user.User, gwc gateway.GatewayAPIClient, machineAuthAPIKey string) (context.Context, error) {
func authenticate(usr *user.User, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], machineAuthAPIKey string) (context.Context, error) {
gatewayClient, err := gatewaySelector.Next()
if err != nil {
return nil, err
}
ctx := revactx.ContextSetUser(context.Background(), usr)
authRes, err := gwc.Authenticate(ctx, &gateway.AuthenticateRequest{
authRes, err := gatewayClient.Authenticate(ctx, &gateway.AuthenticateRequest{
Type: "machine",
ClientId: "userid:" + usr.GetId().GetOpaqueId(),
ClientSecret: machineAuthAPIKey,
@@ -411,8 +417,13 @@ func authenticate(usr *user.User, gwc gateway.GatewayAPIClient, machineAuthAPIKe
return metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, authRes.Token), nil
}
func getSpace(ctx context.Context, spaceID string, gwc gateway.GatewayAPIClient) (*storageprovider.StorageSpace, error) {
res, err := gwc.ListStorageSpaces(ctx, listStorageSpaceRequest(spaceID))
func getSpace(ctx context.Context, spaceID string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*storageprovider.StorageSpace, error) {
gatewayClient, err := gatewaySelector.Next()
if err != nil {
return nil, err
}
res, err := gatewayClient.ListStorageSpaces(ctx, listStorageSpaceRequest(spaceID))
if err != nil {
return nil, err
}
@@ -428,8 +439,13 @@ func getSpace(ctx context.Context, spaceID string, gwc gateway.GatewayAPIClient)
return res.StorageSpaces[0], nil
}
func getUser(ctx context.Context, userid *user.UserId, gwc gateway.GatewayAPIClient) (*user.User, error) {
getUserResponse, err := gwc.GetUser(context.Background(), &user.GetUserRequest{
func getUser(ctx context.Context, userid *user.UserId, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*user.User, error) {
gatewayClient, err := gatewaySelector.Next()
if err != nil {
return nil, err
}
getUserResponse, err := gatewayClient.GetUser(context.Background(), &user.GetUserRequest{
UserId: userid,
})
if err != nil {
@@ -443,8 +459,13 @@ func getUser(ctx context.Context, userid *user.UserId, gwc gateway.GatewayAPICli
return getUserResponse.GetUser(), nil
}
func getGroup(ctx context.Context, groupid string, gwc gateway.GatewayAPIClient) (*group.Group, error) {
r, err := gwc.GetGroup(ctx, &group.GetGroupRequest{GroupId: &group.GroupId{OpaqueId: groupid}})
func getGroup(ctx context.Context, groupid string, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*group.Group, error) {
gatewayClient, err := gatewaySelector.Next()
if err != nil {
return nil, err
}
r, err := gatewayClient.GetGroup(ctx, &group.GetGroupRequest{GroupId: &group.GroupId{OpaqueId: groupid}})
if err != nil {
return nil, err
}
@@ -456,8 +477,13 @@ func getGroup(ctx context.Context, groupid string, gwc gateway.GatewayAPIClient)
return r.GetGroup(), nil
}
func getResource(ctx context.Context, resourceid *storageprovider.ResourceId, gwc gateway.GatewayAPIClient) (*storageprovider.ResourceInfo, error) {
res, err := gwc.Stat(ctx, &storageprovider.StatRequest{Ref: &storageprovider.Reference{ResourceId: resourceid}})
func getResource(ctx context.Context, resourceid *storageprovider.ResourceId, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*storageprovider.ResourceInfo, error) {
gatewayClient, err := gatewaySelector.Next()
if err != nil {
return nil, err
}
res, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{Ref: &storageprovider.Reference{ResourceId: resourceid}})
if err != nil {
return nil, err
}
services/userlog/pkg/service/service_suit_test.go
+13
View File
@@ -3,10 +3,23 @@ package service_test
import (
"testing"
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
mRegistry "go-micro.dev/v4/registry"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func init() {
registry.Configure("memory")
r := registry.GetRegistry()
service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "")
service.Nodes = []*mRegistry.Node{{
Address: "any",
}}
_ = r.Register(service)
}
func TestSearch(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Userlog service Suite")
services/userlog/pkg/service/service_test.go
+20 -5
View File
@@ -11,6 +11,7 @@ import (
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/store"
"github.com/cs3org/reva/v2/pkg/utils"
cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks"
@@ -27,6 +28,7 @@ import (
"github.com/test-go/testify/mock"
microevents "go-micro.dev/v4/events"
microstore "go-micro.dev/v4/store"
"google.golang.org/grpc"
)
var _ = Describe("UserlogService", func() {
@@ -37,7 +39,9 @@ var _ = Describe("UserlogService", func() {
bus testBus
sto microstore.Store
gwc cs3mocks.GatewayAPIClient
gatewayClient *cs3mocks.GatewayAPIClient
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
ehc mocks.EventHistoryService
)
@@ -45,15 +49,26 @@ var _ = Describe("UserlogService", func() {
var err error
sto = store.Create()
bus = testBus(make(chan events.Event))
pool.RemoveSelector("GatewaySelector" + "com.owncloud.api.gateway")
gatewayClient = &cs3mocks.GatewayAPIClient{}
gatewaySelector = pool.GetSelector[gateway.GatewayAPIClient](
"GatewaySelector",
"com.owncloud.api.gateway",
func(cc *grpc.ClientConn) gateway.GatewayAPIClient {
return gatewayClient
},
)
o := utils.AppendJSONToOpaque(nil, "grants", map[string]*provider.ResourcePermissions{"userid": {Stat: true}})
gwc.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{StorageSpaces: []*provider.StorageSpace{
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{StorageSpaces: []*provider.StorageSpace{
{
Opaque: o,
SpaceType: "project",
},
}, Status: &rpc.Status{Code: rpc.Code_CODE_OK}}, nil)
gwc.On("GetUser", mock.Anything, mock.Anything).Return(&user.GetUserResponse{User: &user.User{Id: &user.UserId{OpaqueId: "userid"}}, Status: &rpc.Status{Code: rpc.Code_CODE_OK}}, nil)
gwc.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{Status: &rpc.Status{Code: rpc.Code_CODE_OK}}, nil)
gatewayClient.On("GetUser", mock.Anything, mock.Anything).Return(&user.GetUserResponse{User: &user.User{Id: &user.UserId{OpaqueId: "userid"}}, Status: &rpc.Status{Code: rpc.Code_CODE_OK}}, nil)
gatewayClient.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{Status: &rpc.Status{Code: rpc.Code_CODE_OK}}, nil)
ul, err = service.NewUserlogService(
service.Config(cfg),
@@ -61,7 +76,7 @@ var _ = Describe("UserlogService", func() {
service.Store(sto),
service.Logger(log.NewLogger()),
service.Mux(chi.NewMux()),
service.GatewayClient(&gwc),
service.GatewaySelector(gatewaySelector),
service.HistoryClient(&ehc),
service.RegisteredEvents([]events.Unmarshaller{
events.SpaceDisabled{},