From 8e32b4f5da49b1f7bf3c067acd67aee6eeb99643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Wed, 5 Apr 2023 12:39:05 +0200 Subject: [PATCH 1/9] Integrate keycloak and events data into graph. This PR adds the data from keycloak and events into the GDPR export of the graph service. --- services/graph/pkg/config/config.go | 13 ++++++++++++- services/graph/pkg/server/http/server.go | 16 ++++++++++++++++ services/graph/pkg/service/v0/graph.go | 4 ++++ services/graph/pkg/service/v0/option.go | 18 ++++++++++++++++++ services/graph/pkg/service/v0/personaldata.go | 14 +++++++++++--- services/graph/pkg/service/v0/service.go | 1 + services/invitations/pkg/config/config.go | 12 ++++++------ 7 files changed, 68 insertions(+), 10 deletions(-) diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index 0dac204c2..da11aa48d 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -30,7 +30,8 @@ type Config struct { Identity Identity `yaml:"identity"` Events Events `yaml:"events"` - MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."` + Keycloak Keycloak `yaml:"keycloak"` Context context.Context `yaml:"-"` } @@ -121,3 +122,13 @@ type CORS struct { AllowedHeaders []string `yaml:"allow_headers" env:"OCIS_CORS_ALLOW_HEADERS;GRAPH_CORS_ALLOW_HEADERS" desc:"A comma-separated list of allowed CORS headers. See following chapter for more details: *Access-Control-Request-Headers* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Request-Headers."` AllowCredentials bool `yaml:"allow_credentials" env:"OCIS_CORS_ALLOW_CREDENTIALS;GRAPH_CORS_ALLOW_CREDENTIALS" desc:"Allow credentials for CORS.See following chapter for more details: *Access-Control-Allow-Credentials* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials."` } + +// Keycloak configuration +type Keycloak struct { + BasePath string `yaml:"base_path" env:"KEYCLOAK_BASE_PATH;GRAPH_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` + ClientID string `yaml:"client_id" env:"KEYCLOAK_CLIENT_ID;GRAPH_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` + ClientSecret string `yaml:"client_secret" env:"KEYCLOAK_CLIENT_SECRET;GRAPH_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` + ClientRealm string `yaml:"client_realm" env:"KEYCLOAK_CLIENT_REALM;GRAPH_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` + UserRealm string `yaml:"user_realm" env:"KEYCLOAK_USER_REALM;GRAPH_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` + InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"KEYCLOAK_INSECURE_SKIP_VERIFY;GRAPH_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` +} diff --git a/services/graph/pkg/server/http/server.go b/services/graph/pkg/server/http/server.go index 83643968a..5e7f89bd1 100644 --- a/services/graph/pkg/server/http/server.go +++ b/services/graph/pkg/server/http/server.go @@ -15,10 +15,13 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/account" "github.com/owncloud/ocis/v2/ocis-pkg/cors" ociscrypto "github.com/owncloud/ocis/v2/ocis-pkg/crypto" + "github.com/owncloud/ocis/v2/ocis-pkg/keycloak" "github.com/owncloud/ocis/v2/ocis-pkg/middleware" "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" + ogrpc "github.com/owncloud/ocis/v2/ocis-pkg/service/grpc" "github.com/owncloud/ocis/v2/ocis-pkg/service/http" "github.com/owncloud/ocis/v2/ocis-pkg/version" + ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0" searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" graphMiddleware "github.com/owncloud/ocis/v2/services/graph/pkg/middleware" @@ -132,6 +135,18 @@ func Server(opts ...Option) (http.Service, error) { // no gatewayclient needed } + // Keycloak client is optional, so if it stays nil, it's fine. + var keyCloakClient keycloak.Client + if options.Config.Keycloak.BasePath != "" { + kcc := options.Config.Keycloak + if kcc.ClientID == "" || kcc.ClientSecret == "" || kcc.ClientRealm == "" || kcc.UserRealm == "" { + return nil, errors.New("keycloak client id, secret, client realm and user realm must be set") + } + keyCloakClient = keycloak.New(kcc.BasePath, kcc.ClientID, kcc.ClientSecret, kcc.ClientRealm, kcc.InsecureSkipVerify) + } + + hClient := ehsvc.NewEventHistoryService("com.owncloud.api.eventhistory", ogrpc.DefaultClient()) + var handle svc.Service handle, err = svc.NewService( svc.Logger(options.Logger), @@ -142,6 +157,7 @@ func Server(opts ...Option) (http.Service, error) { svc.WithRequireAdminMiddleware(requireAdminMiddleware), svc.WithGatewayClient(gatewayClient), svc.WithSearchService(searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient())), + svc.KeycloakCient(keyCloakClient), ) if err != nil { diff --git a/services/graph/pkg/service/v0/graph.go b/services/graph/pkg/service/v0/graph.go index e4fc0d8f5..d5bd379b3 100644 --- a/services/graph/pkg/service/v0/graph.go +++ b/services/graph/pkg/service/v0/graph.go @@ -13,7 +13,9 @@ import ( "github.com/go-chi/chi/v5" "github.com/jellydator/ttlcache/v3" libregraph "github.com/owncloud/libre-graph-api-go" + "github.com/owncloud/ocis/v2/ocis-pkg/keycloak" "github.com/owncloud/ocis/v2/ocis-pkg/log" + ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0" searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" "github.com/owncloud/ocis/v2/services/graph/pkg/config" @@ -68,6 +70,8 @@ type Graph struct { groupsCache *ttlcache.Cache[string, libregraph.Group] eventsPublisher events.Publisher searchService searchsvc.SearchProviderService + keycloakClient keycloak.Client + historyClient ehsvc.EventHistoryService } // ServeHTTP implements the Service interface. diff --git a/services/graph/pkg/service/v0/option.go b/services/graph/pkg/service/v0/option.go index ccbde28b4..16ca102a0 100644 --- a/services/graph/pkg/service/v0/option.go +++ b/services/graph/pkg/service/v0/option.go @@ -5,8 +5,10 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/cs3org/reva/v2/pkg/events" + "github.com/owncloud/ocis/v2/ocis-pkg/keycloak" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/roles" + ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0" searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0" "github.com/owncloud/ocis/v2/services/graph/pkg/config" @@ -30,6 +32,8 @@ type Options struct { RoleManager *roles.Manager EventsPublisher events.Publisher SearchService searchsvc.SearchProviderService + KeycloakClient keycloak.Client + EventHistoryClient ehsvc.EventHistoryService } // newOptions initializes the available default options. @@ -126,3 +130,17 @@ func EventsPublisher(val events.Publisher) Option { o.EventsPublisher = val } } + +// KeycloakCient provides a function to set the KeycloakCient option. +func KeycloakCient(val keycloak.Client) Option { + return func(o *Options) { + o.KeycloakClient = val + } +} + +// EventHistoryClient provides a function to set the EventHistoryClient option. +func KeycloakCient(val keycloak.Client) Option { + return func(o *Options) { + o.KeycloakClient = val + } +} diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index b934acb96..a8ae63a9d 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -68,19 +68,28 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { } // go start gathering - go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh) + go g.GatherPersonalData(ctx, u, ref, r.Header.Get(revactx.TokenHeader), marsh) w.WriteHeader(http.StatusCreated) } // GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space -func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { +func (g Graph) GatherPersonalData(ctx context.Context, usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { // create data data := make(map[string]interface{}) // reva user data["user"] = usr + // Check if we have a keycloak client, and if so, get the keycloak export. + if g.keycloakClient != nil { + kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.ClientID, usr) + if err != nil { + g.logger.Error().Err(err).Str("userID").Msg("cannot get keycloak personal data") + } + data["keycloak"] = kcd + } + // marshal by, err := marsh(data) if err != nil { @@ -198,7 +207,6 @@ func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.Gat } } return nil - } func getLocation(r *http.Request) string { diff --git a/services/graph/pkg/service/v0/service.go b/services/graph/pkg/service/v0/service.go index 8541167af..08c345aeb 100644 --- a/services/graph/pkg/service/v0/service.go +++ b/services/graph/pkg/service/v0/service.go @@ -143,6 +143,7 @@ func NewService(opts ...Option) (Graph, error) { gatewayClient: options.GatewayClient, searchService: options.SearchService, identityEducationBackend: options.IdentityEducationBackend, + keycloakClient: options.KeycloakClient, } if err := setIdentityBackends(options, &svc); err != nil { diff --git a/services/invitations/pkg/config/config.go b/services/invitations/pkg/config/config.go index ca499aeae..944813329 100644 --- a/services/invitations/pkg/config/config.go +++ b/services/invitations/pkg/config/config.go @@ -26,10 +26,10 @@ type Config struct { // Keycloak configuration type Keycloak struct { - BasePath string `yaml:"base_path" env:"INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` - ClientID string `yaml:"client_id" env:"INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` - ClientSecret string `yaml:"client_secret" env:"INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` - ClientRealm string `yaml:"client_realm" env:"INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` - UserRealm string `yaml:"user_realm" env:"INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` - InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` + BasePath string `yaml:"base_path" env:"KEYCLOAK_BASE_PATH;INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` + ClientID string `yaml:"client_id" env:"KEYCLOAK_CLIENT_ID;INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` + ClientSecret string `yaml:"client_secret" env:"KEYCLOAK_CLIENT_SECRET;INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` + ClientRealm string `yaml:"client_realm" env:"KEYCLOAK_CLIENT_REALM;INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` + UserRealm string `yaml:"user_realm" env:"KEYCLOAK_USER_REALM;INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` + InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"KEYCLOAK_INSECURE_SKIP_VERIFY;INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` } From cf6b1eac25785cb51433e8aa6ac0523a74b6b01d Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 5 Apr 2023 15:07:31 +0200 Subject: [PATCH 2/9] add events to personalDataExport Signed-off-by: jkoberg --- services/graph/pkg/server/http/server.go | 5 ++- services/graph/pkg/service/v0/option.go | 8 ++-- services/graph/pkg/service/v0/personaldata.go | 43 ++++++++++++++++--- services/graph/pkg/service/v0/service.go | 1 + 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/services/graph/pkg/server/http/server.go b/services/graph/pkg/server/http/server.go index 5e7f89bd1..fe9e34d75 100644 --- a/services/graph/pkg/server/http/server.go +++ b/services/graph/pkg/server/http/server.go @@ -140,7 +140,7 @@ func Server(opts ...Option) (http.Service, error) { if options.Config.Keycloak.BasePath != "" { kcc := options.Config.Keycloak if kcc.ClientID == "" || kcc.ClientSecret == "" || kcc.ClientRealm == "" || kcc.UserRealm == "" { - return nil, errors.New("keycloak client id, secret, client realm and user realm must be set") + return http.Service{}, errors.New("keycloak client id, secret, client realm and user realm must be set") } keyCloakClient = keycloak.New(kcc.BasePath, kcc.ClientID, kcc.ClientSecret, kcc.ClientRealm, kcc.InsecureSkipVerify) } @@ -157,7 +157,8 @@ func Server(opts ...Option) (http.Service, error) { svc.WithRequireAdminMiddleware(requireAdminMiddleware), svc.WithGatewayClient(gatewayClient), svc.WithSearchService(searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient())), - svc.KeycloakCient(keyCloakClient), + svc.KeycloakClient(keyCloakClient), + svc.EventHistoryClient(hClient), ) if err != nil { diff --git a/services/graph/pkg/service/v0/option.go b/services/graph/pkg/service/v0/option.go index 16ca102a0..51581ade7 100644 --- a/services/graph/pkg/service/v0/option.go +++ b/services/graph/pkg/service/v0/option.go @@ -131,16 +131,16 @@ func EventsPublisher(val events.Publisher) Option { } } -// KeycloakCient provides a function to set the KeycloakCient option. -func KeycloakCient(val keycloak.Client) Option { +// KeycloakClient provides a function to set the KeycloakCient option. +func KeycloakClient(val keycloak.Client) Option { return func(o *Options) { o.KeycloakClient = val } } // EventHistoryClient provides a function to set the EventHistoryClient option. -func KeycloakCient(val keycloak.Client) Option { +func EventHistoryClient(val ehsvc.EventHistoryService) Option { return func(o *Options) { - o.KeycloakClient = val + o.EventHistoryClient = val } } diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index a8ae63a9d..b583b3405 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -19,6 +19,8 @@ import ( "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rhttp" "github.com/cs3org/reva/v2/pkg/utils" + libregraph "github.com/owncloud/libre-graph-api-go" + ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0" ) var ( @@ -68,13 +70,19 @@ func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) { } // go start gathering - go g.GatherPersonalData(ctx, u, ref, r.Header.Get(revactx.TokenHeader), marsh) + go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh) w.WriteHeader(http.StatusCreated) } // GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space -func (g Graph) GatherPersonalData(ctx context.Context, usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { +func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) { + // the context might already be cancelled. We need to impersonate the acting user again + ctx, err := utils.ImpersonateUser(usr, g.gatewayClient, g.config.MachineAuthAPIKey) + if err != nil { + g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot impersonate user") + + } // create data data := make(map[string]interface{}) @@ -82,14 +90,24 @@ func (g Graph) GatherPersonalData(ctx context.Context, usr *user.User, ref *prov data["user"] = usr // Check if we have a keycloak client, and if so, get the keycloak export. - if g.keycloakClient != nil { - kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.ClientID, usr) + if ctx != nil && g.keycloakClient != nil { + id := usr.GetId().GetOpaqueId() + kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.ClientID, &libregraph.User{Id: &id}) if err != nil { - g.logger.Error().Err(err).Str("userID").Msg("cannot get keycloak personal data") + g.logger.Error().Err(err).Str("userID", id).Msg("cannot get keycloak personal data") } data["keycloak"] = kcd } + // get event data + if ctx != nil && g.historyClient != nil { + resp, err := g.historyClient.GetEventsForUser(ctx, &ehsvc.GetEventsForUserRequest{UserID: usr.GetId().GetOpaqueId()}) + if err != nil { + g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get event personal data") + } + data["events"] = convertEvents(resp) + } + // marshal by, err := marsh(data) if err != nil { @@ -222,3 +240,18 @@ func getLocation(r *http.Request) string { return _backupFileName } + +// we want the events to look nice in the file, don't we? +func convertEvents(resp *ehsvc.GetEventsResponse) []map[string]interface{} { + var out []map[string]interface{} + for _, e := range resp.GetEvents() { + var content map[string]interface{} + _ = json.Unmarshal(e.Event, &content) + out = append(out, map[string]interface{}{ + "id": e.GetId(), + "type": e.GetType(), + "event": content, + }) + } + return out +} diff --git a/services/graph/pkg/service/v0/service.go b/services/graph/pkg/service/v0/service.go index 08c345aeb..8ed8f6419 100644 --- a/services/graph/pkg/service/v0/service.go +++ b/services/graph/pkg/service/v0/service.go @@ -144,6 +144,7 @@ func NewService(opts ...Option) (Graph, error) { searchService: options.SearchService, identityEducationBackend: options.IdentityEducationBackend, keycloakClient: options.KeycloakClient, + historyClient: options.EventHistoryClient, } if err := setIdentityBackends(options, &svc); err != nil { From c9d0e876f76727033f9597481ee14dfeb6731cb2 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 5 Apr 2023 16:53:34 +0200 Subject: [PATCH 3/9] use mail instead id to get user Signed-off-by: jkoberg --- ocis-pkg/keycloak/client.go | 4 ++-- ocis-pkg/keycloak/types.go | 2 +- services/graph/pkg/service/v0/personaldata.go | 21 +++++++++---------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ocis-pkg/keycloak/client.go b/ocis-pkg/keycloak/client.go index 8f39dda53..3b20eafe0 100644 --- a/ocis-pkg/keycloak/client.go +++ b/ocis-pkg/keycloak/client.go @@ -116,8 +116,8 @@ func (c *ConcreteClient) GetUserByEmail(ctx context.Context, realm, mail string) } // GetPIIReport returns a structure with all the PII for the user. -func (c *ConcreteClient) GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*PIIReport, error) { - u, err := c.GetUserByEmail(ctx, realm, *user.Mail) +func (c *ConcreteClient) GetPIIReport(ctx context.Context, realm string, email string) (*PIIReport, error) { + u, err := c.GetUserByEmail(ctx, realm, email) if err != nil { return nil, err } diff --git a/ocis-pkg/keycloak/types.go b/ocis-pkg/keycloak/types.go index 1f5ddfa64..2d5de64a5 100644 --- a/ocis-pkg/keycloak/types.go +++ b/ocis-pkg/keycloak/types.go @@ -35,5 +35,5 @@ type Client interface { CreateUser(ctx context.Context, realm string, user *libregraph.User, userActions []UserAction) (string, error) SendActionsMail(ctx context.Context, realm, userID string, userActions []UserAction) error GetUserByEmail(ctx context.Context, realm, email string) (*libregraph.User, error) - GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*PIIReport, error) + GetPIIReport(ctx context.Context, realm string, email string) (*PIIReport, error) } diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index b583b3405..6a7b360fd 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -19,7 +19,7 @@ import ( "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rhttp" "github.com/cs3org/reva/v2/pkg/utils" - libregraph "github.com/owncloud/libre-graph-api-go" + ehmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/eventhistory/v0" ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0" ) @@ -91,10 +91,9 @@ func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token // Check if we have a keycloak client, and if so, get the keycloak export. if ctx != nil && g.keycloakClient != nil { - id := usr.GetId().GetOpaqueId() - kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.ClientID, &libregraph.User{Id: &id}) + kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.UserRealm, usr.GetMail()) if err != nil { - g.logger.Error().Err(err).Str("userID", id).Msg("cannot get keycloak personal data") + g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get keycloak personal data") } data["keycloak"] = kcd } @@ -105,7 +104,7 @@ func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token if err != nil { g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get event personal data") } - data["events"] = convertEvents(resp) + data["events"] = convertEvents(resp.GetEvents()) } // marshal @@ -242,16 +241,16 @@ func getLocation(r *http.Request) string { } // we want the events to look nice in the file, don't we? -func convertEvents(resp *ehsvc.GetEventsResponse) []map[string]interface{} { - var out []map[string]interface{} - for _, e := range resp.GetEvents() { +func convertEvents(evs []*ehmsg.Event) []map[string]interface{} { + out := make([]map[string]interface{}, len(evs)) + for i, e := range evs { var content map[string]interface{} - _ = json.Unmarshal(e.Event, &content) - out = append(out, map[string]interface{}{ + _ = json.Unmarshal(e.GetEvent(), &content) + out[i] = map[string]interface{}{ "id": e.GetId(), "type": e.GetType(), "event": content, - }) + } } return out } From 3ec5c0f6ed9b7ba6c33a7e5ee8b5e1ff8cfbdc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Wed, 12 Apr 2023 12:11:22 +0200 Subject: [PATCH 4/9] Don't check for attributes if no attributes set. If no attributes are set, the attributes property is nil. This causes a panic if keycloak has no custom attributes. This patch checks for nil before accessing the attributes. --- ocis-pkg/keycloak/client.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ocis-pkg/keycloak/client.go b/ocis-pkg/keycloak/client.go index 3b20eafe0..49cd3e981 100644 --- a/ocis-pkg/keycloak/client.go +++ b/ocis-pkg/keycloak/client.go @@ -164,17 +164,20 @@ func (c *ConcreteClient) getToken(ctx context.Context) (*gocloak.JWT, error) { } func (c *ConcreteClient) keycloakUserToLibregraph(u *gocloak.User) *libregraph.User { - attrs := *u.Attributes - ldapID := "" - ldapIDs, ok := attrs[_idAttr] - if ok { - ldapID = ldapIDs[0] - } - + var ldapID string var userType *string - userTypes, ok := attrs[_userTypeAttr] - if ok { - userType = &userTypes[0] + + if u.Attributes != nil { + attrs := *u.Attributes + ldapIDs, ok := attrs[_idAttr] + if ok { + ldapID = ldapIDs[0] + } + + userTypes, ok := attrs[_userTypeAttr] + if ok { + userType = &userTypes[0] + } } return &libregraph.User{ From d81065c1b6ffb80e80a5be8a6327ada95c27adab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Wed, 12 Apr 2023 12:13:33 +0200 Subject: [PATCH 5/9] Adds JSON tags to PII report. This will make the JSON export look a bit prettier. --- ocis-pkg/keycloak/types.go | 4 ++-- services/graph/pkg/service/v0/personaldata.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ocis-pkg/keycloak/types.go b/ocis-pkg/keycloak/types.go index 2d5de64a5..366a550e3 100644 --- a/ocis-pkg/keycloak/types.go +++ b/ocis-pkg/keycloak/types.go @@ -26,8 +26,8 @@ var userActionsToString = map[UserAction]string{ // PIIReport is a structure of all the PersonalIdentifiableInformation contained in keycloak. type PIIReport struct { - UserData *libregraph.User - Credentials []*gocloak.CredentialRepresentation + UserData *libregraph.User `json:"user_data,omitempty"` + Credentials []*gocloak.CredentialRepresentation `json:"credentials,omitempty"` } // Client represents a keycloak client. diff --git a/services/graph/pkg/service/v0/personaldata.go b/services/graph/pkg/service/v0/personaldata.go index 6a7b360fd..2017c6031 100644 --- a/services/graph/pkg/service/v0/personaldata.go +++ b/services/graph/pkg/service/v0/personaldata.go @@ -81,7 +81,6 @@ func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token ctx, err := utils.ImpersonateUser(usr, g.gatewayClient, g.config.MachineAuthAPIKey) if err != nil { g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot impersonate user") - } // create data data := make(map[string]interface{}) From ed27261262578ab6df1a133d010aeff6d761f7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Wed, 12 Apr 2023 12:21:44 +0200 Subject: [PATCH 6/9] Add documentation about keycloak configuration. --- services/graph/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/services/graph/README.md b/services/graph/README.md index 0bcc40a4d..f65d623b0 100644 --- a/services/graph/README.md +++ b/services/graph/README.md @@ -30,3 +30,20 @@ The `graph` service can use a configured store via `GRAPH_STORE_TYPE`. Possible 2. Though usually not necessary, a database name and a database table can be configured for event stores if the event store supports this. Generally not applicapable for stores of type `in-memory`. These settings are blank by default which means that the standard settings of the configured store applies. 3. The graph service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances. 4. When using `redis-sentinel`, the Redis master to use is configured via `GRAPH_CACHE_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. + + +## Keycloak configuration for the personal data export + +If Keycloak is used for authentication, the GDPR requires us to add all the personal identifiable +information that keycloak has about the user to the personal data export. To do this, set the following +environment variables: + +* `KEYCLOAK_BASE_PATH` - The URL to the keycloak instance. +* `KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. +* `KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak. +* `KEYCLOAK_CLIENT_REALM` - The realm the client is defined in. +* `KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in. +* `KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified. + +These variables are the same as the ones used by the invitations service, but can be configured separateley +by prefixing `GRAPH_` to the variable name. From 43cf811db13c2f156513441527465297b3d3bd35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Wed, 12 Apr 2023 13:45:06 +0200 Subject: [PATCH 7/9] Format markdown a bit cleaner. --- services/graph/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/services/graph/README.md b/services/graph/README.md index f65d623b0..296f0b85a 100644 --- a/services/graph/README.md +++ b/services/graph/README.md @@ -31,19 +31,18 @@ The `graph` service can use a configured store via `GRAPH_STORE_TYPE`. Possible 3. The graph service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances. 4. When using `redis-sentinel`, the Redis master to use is configured via `GRAPH_CACHE_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. - ## Keycloak configuration for the personal data export If Keycloak is used for authentication, the GDPR requires us to add all the personal identifiable information that keycloak has about the user to the personal data export. To do this, set the following environment variables: -* `KEYCLOAK_BASE_PATH` - The URL to the keycloak instance. -* `KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. -* `KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak. -* `KEYCLOAK_CLIENT_REALM` - The realm the client is defined in. -* `KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in. -* `KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified. + * `KEYCLOAK_BASE_PATH` - The URL to the keycloak instance. + * `KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. + * `KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak. + * `KEYCLOAK_CLIENT_REALM` - The realm the client is defined in. + * `KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in. + * `KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified. These variables are the same as the ones used by the invitations service, but can be configured separateley by prefixing `GRAPH_` to the variable name. From 2e8b74153083cc08742a38b1203b237ce89b4a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Thu, 13 Apr 2023 09:19:42 +0200 Subject: [PATCH 8/9] improve readme Co-authored-by: Martin --- services/graph/README.md | 32 ++++++++++++++--------- services/graph/pkg/config/config.go | 12 ++++----- services/invitations/pkg/config/config.go | 12 ++++----- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/services/graph/README.md b/services/graph/README.md index 296f0b85a..6813c2f48 100644 --- a/services/graph/README.md +++ b/services/graph/README.md @@ -31,18 +31,26 @@ The `graph` service can use a configured store via `GRAPH_STORE_TYPE`. Possible 3. The graph service can be scaled if not using `in-memory` stores and the stores are configured identically over all instances. 4. When using `redis-sentinel`, the Redis master to use is configured via `GRAPH_CACHE_STORE_NODES` in the form of `:/` like `10.10.0.200:26379/mymaster`. -## Keycloak configuration for the personal data export +## Keycloak Configuration For The Personal Data Export -If Keycloak is used for authentication, the GDPR requires us to add all the personal identifiable -information that keycloak has about the user to the personal data export. To do this, set the following -environment variables: +If Keycloak is used for authentication, GDPR regulations require to add all personal identifiable information that Keycloak has about the user to the personal data export. To do this, the following environment variables must be set: - * `KEYCLOAK_BASE_PATH` - The URL to the keycloak instance. - * `KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. - * `KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak. - * `KEYCLOAK_CLIENT_REALM` - The realm the client is defined in. - * `KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in. - * `KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified. +* `OCIS_KEYCLOAK_BASE_PATH` - The URL to the keycloak instance. +* `OCIS_KEYCLOAK_CLIENT_ID` - The client ID of the client that is used to authenticate with keycloak, this client has to be able to list users and get the credential data. +* `OCIS_KEYCLOAK_CLIENT_SECRET` - The client secret of the client that is used to authenticate with keycloak. +* `OCIS_KEYCLOAK_CLIENT_REALM` - The realm the client is defined in. +* `OCIS_KEYCLOAK_USER_REALM` - The realm the oCIS users are defined in. +* `OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY` - If set to true, the TLS certificate of the keycloak instance is not verified. -These variables are the same as the ones used by the invitations service, but can be configured separateley -by prefixing `GRAPH_` to the variable name. +### Keycloak Client Configuration + +The client that is used to authenticate with keycloak has to be able to list users and get the credential data. To do this, the following roles have to be assigned to the client and they have to be about the realm that contains the oCIS users: + +* `view-users` +* `view-identity-providers` +* `view-realm` +* `view-clients` +* `view-events` +* `view-authorization` + +Note that these roles are only available to assign if the client is in the `master` realm. diff --git a/services/graph/pkg/config/config.go b/services/graph/pkg/config/config.go index da11aa48d..cd9623b81 100644 --- a/services/graph/pkg/config/config.go +++ b/services/graph/pkg/config/config.go @@ -125,10 +125,10 @@ type CORS struct { // Keycloak configuration type Keycloak struct { - BasePath string `yaml:"base_path" env:"KEYCLOAK_BASE_PATH;GRAPH_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` - ClientID string `yaml:"client_id" env:"KEYCLOAK_CLIENT_ID;GRAPH_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` - ClientSecret string `yaml:"client_secret" env:"KEYCLOAK_CLIENT_SECRET;GRAPH_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` - ClientRealm string `yaml:"client_realm" env:"KEYCLOAK_CLIENT_REALM;GRAPH_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` - UserRealm string `yaml:"user_realm" env:"KEYCLOAK_USER_REALM;GRAPH_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` - InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"KEYCLOAK_INSECURE_SKIP_VERIFY;GRAPH_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` + BasePath string `yaml:"base_path" env:"OCIS_KEYCLOAK_BASE_PATH;GRAPH_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` + ClientID string `yaml:"client_id" env:"OCIS_KEYCLOAK_CLIENT_ID;GRAPH_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` + ClientSecret string `yaml:"client_secret" env:"OCIS_KEYCLOAK_CLIENT_SECRET;GRAPH_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` + ClientRealm string `yaml:"client_realm" env:"OCIS_KEYCLOAK_CLIENT_REALM;GRAPH_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` + UserRealm string `yaml:"user_realm" env:"OCIS_KEYCLOAK_USER_REALM;GRAPH_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` + InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;GRAPH_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` } diff --git a/services/invitations/pkg/config/config.go b/services/invitations/pkg/config/config.go index 944813329..27391684c 100644 --- a/services/invitations/pkg/config/config.go +++ b/services/invitations/pkg/config/config.go @@ -26,10 +26,10 @@ type Config struct { // Keycloak configuration type Keycloak struct { - BasePath string `yaml:"base_path" env:"KEYCLOAK_BASE_PATH;INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` - ClientID string `yaml:"client_id" env:"KEYCLOAK_CLIENT_ID;INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` - ClientSecret string `yaml:"client_secret" env:"KEYCLOAK_CLIENT_SECRET;INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` - ClientRealm string `yaml:"client_realm" env:"KEYCLOAK_CLIENT_REALM;INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` - UserRealm string `yaml:"user_realm" env:"KEYCLOAK_USER_REALM;INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` - InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"KEYCLOAK_INSECURE_SKIP_VERIFY;INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` + BasePath string `yaml:"base_path" env:"OCIS_KEYCLOAK_BASE_PATH;INVITATIONS_KEYCLOAK_BASE_PATH" desc:"The URL to access keycloak."` + ClientID string `yaml:"client_id" env:"OCIS_KEYCLOAK_CLIENT_ID;INVITATIONS_KEYCLOAK_CLIENT_ID" desc:"The client id to authenticate with keycloak."` + ClientSecret string `yaml:"client_secret" env:"OCIS_KEYCLOAK_CLIENT_SECRET;INVITATIONS_KEYCLOAK_CLIENT_SECRET" desc:"The client secret to use in authentication."` + ClientRealm string `yaml:"client_realm" env:"OCIS_KEYCLOAK_CLIENT_REALM;INVITATIONS_KEYCLOAK_CLIENT_REALM" desc:"The realm the client is defined in."` + UserRealm string `yaml:"user_realm" env:"OCIS_KEYCLOAK_USER_REALM;INVITATIONS_KEYCLOAK_USER_REALM" desc:"The realm users are defined."` + InsecureSkipVerify bool `yaml:"insecure_skip_verify" env:"OCIS_KEYCLOAK_INSECURE_SKIP_VERIFY;INVITATIONS_KEYCLOAK_INSECURE_SKIP_VERIFY" desc:"Disable TLS certificate validation for Keycloak connections. Do not set this in production environments."` } From 2d8fb2549a41d5f5b3f7872ae0eb87f0ea6d84a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Franke?= Date: Thu, 13 Apr 2023 13:07:24 +0200 Subject: [PATCH 9/9] Add modified mock. --- services/invitations/pkg/mocks/client.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/services/invitations/pkg/mocks/client.go b/services/invitations/pkg/mocks/client.go index eddaf9cec..42efdfbf0 100644 --- a/services/invitations/pkg/mocks/client.go +++ b/services/invitations/pkg/mocks/client.go @@ -40,25 +40,25 @@ func (_m *Client) CreateUser(ctx context.Context, realm string, user *libregraph return r0, r1 } -// GetPIIReport provides a mock function with given fields: ctx, realm, user -func (_m *Client) GetPIIReport(ctx context.Context, realm string, user *libregraph.User) (*keycloak.PIIReport, error) { - ret := _m.Called(ctx, realm, user) +// GetPIIReport provides a mock function with given fields: ctx, realm, email +func (_m *Client) GetPIIReport(ctx context.Context, realm string, email string) (*keycloak.PIIReport, error) { + ret := _m.Called(ctx, realm, email) var r0 *keycloak.PIIReport var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, *libregraph.User) (*keycloak.PIIReport, error)); ok { - return rf(ctx, realm, user) + if rf, ok := ret.Get(0).(func(context.Context, string, string) (*keycloak.PIIReport, error)); ok { + return rf(ctx, realm, email) } - if rf, ok := ret.Get(0).(func(context.Context, string, *libregraph.User) *keycloak.PIIReport); ok { - r0 = rf(ctx, realm, user) + if rf, ok := ret.Get(0).(func(context.Context, string, string) *keycloak.PIIReport); ok { + r0 = rf(ctx, realm, email) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*keycloak.PIIReport) } } - if rf, ok := ret.Get(1).(func(context.Context, string, *libregraph.User) error); ok { - r1 = rf(ctx, realm, user) + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, realm, email) } else { r1 = ret.Error(1) }