sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add gdpr export endpoint

Browse Source 
Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
2023-03-29 16:02:42 +02:00
parent 526e4c6459
commit 442c53f796
12 changed files with 342 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/graph/pkg/config/config.go
+2
View File
@@ -30,6 +30,8 @@ type Config struct {
Identity Identity `yaml:"identity"`
Events Events `yaml:"events"`
MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;USERLOG_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary to access resources from other services."`
Context context.Context `yaml:"-"`
}
services/graph/pkg/config/defaults/defaultconfig.go
+4
View File
@@ -149,6 +149,10 @@ func EnsureDefaults(cfg *config.Config) {
if cfg.Commons != nil {
cfg.HTTP.TLS = cfg.Commons.HTTPServiceTLS
}
if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" {
cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey
}
}
// Sanitize sanitized the configuration
services/graph/pkg/identity/mocks/education_backend.go
+65 -17
View File
@@ -1,4 +1,4 @@
// Code generated by mockery v2.14.1. DO NOT EDIT.
// Code generated by mockery v2.22.1. DO NOT EDIT.
package mocks
@@ -62,6 +62,10 @@ func (_m *EducationBackend) CreateEducationClass(ctx context.Context, class libr
ret := _m.Called(ctx, class)
var r0 *libregraph.EducationClass
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationClass) (*libregraph.EducationClass, error)); ok {
return rf(ctx, class)
}
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationClass) *libregraph.EducationClass); ok {
r0 = rf(ctx, class)
} else {
@@ -70,7 +74,6 @@ func (_m *EducationBackend) CreateEducationClass(ctx context.Context, class libr
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationClass) error); ok {
r1 = rf(ctx, class)
} else {
@@ -85,6 +88,10 @@ func (_m *EducationBackend) CreateEducationSchool(ctx context.Context, group lib
ret := _m.Called(ctx, group)
var r0 *libregraph.EducationSchool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationSchool) (*libregraph.EducationSchool, error)); ok {
return rf(ctx, group)
}
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationSchool) *libregraph.EducationSchool); ok {
r0 = rf(ctx, group)
} else {
@@ -93,7 +100,6 @@ func (_m *EducationBackend) CreateEducationSchool(ctx context.Context, group lib
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationSchool) error); ok {
r1 = rf(ctx, group)
} else {
@@ -108,6 +114,10 @@ func (_m *EducationBackend) CreateEducationUser(ctx context.Context, user libreg
ret := _m.Called(ctx, user)
var r0 *libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationUser) (*libregraph.EducationUser, error)); ok {
return rf(ctx, user)
}
if rf, ok := ret.Get(0).(func(context.Context, libregraph.EducationUser) *libregraph.EducationUser); ok {
r0 = rf(ctx, user)
} else {
@@ -116,7 +126,6 @@ func (_m *EducationBackend) CreateEducationUser(ctx context.Context, user libreg
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, libregraph.EducationUser) error); ok {
r1 = rf(ctx, user)
} else {
@@ -173,6 +182,10 @@ func (_m *EducationBackend) GetEducationClass(ctx context.Context, namedOrID str
ret := _m.Called(ctx, namedOrID)
var r0 *libregraph.EducationClass
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationClass, error)); ok {
return rf(ctx, namedOrID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationClass); ok {
r0 = rf(ctx, namedOrID)
} else {
@@ -181,7 +194,6 @@ func (_m *EducationBackend) GetEducationClass(ctx context.Context, namedOrID str
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, namedOrID)
} else {
@@ -196,6 +208,10 @@ func (_m *EducationBackend) GetEducationClassMembers(ctx context.Context, nameOr
ret := _m.Called(ctx, nameOrID)
var r0 []*libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok {
return rf(ctx, nameOrID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok {
r0 = rf(ctx, nameOrID)
} else {
@@ -204,7 +220,6 @@ func (_m *EducationBackend) GetEducationClassMembers(ctx context.Context, nameOr
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, nameOrID)
} else {
@@ -219,6 +234,10 @@ func (_m *EducationBackend) GetEducationClassTeachers(ctx context.Context, class
ret := _m.Called(ctx, classID)
var r0 []*libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok {
return rf(ctx, classID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok {
r0 = rf(ctx, classID)
} else {
@@ -227,7 +246,6 @@ func (_m *EducationBackend) GetEducationClassTeachers(ctx context.Context, class
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, classID)
} else {
@@ -242,6 +260,10 @@ func (_m *EducationBackend) GetEducationClasses(ctx context.Context) ([]*libregr
ret := _m.Called(ctx)
var r0 []*libregraph.EducationClass
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationClass, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationClass); ok {
r0 = rf(ctx)
} else {
@@ -250,7 +272,6 @@ func (_m *EducationBackend) GetEducationClasses(ctx context.Context) ([]*libregr
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
@@ -265,6 +286,10 @@ func (_m *EducationBackend) GetEducationSchool(ctx context.Context, nameOrID str
ret := _m.Called(ctx, nameOrID)
var r0 *libregraph.EducationSchool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationSchool, error)); ok {
return rf(ctx, nameOrID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationSchool); ok {
r0 = rf(ctx, nameOrID)
} else {
@@ -273,7 +298,6 @@ func (_m *EducationBackend) GetEducationSchool(ctx context.Context, nameOrID str
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, nameOrID)
} else {
@@ -288,6 +312,10 @@ func (_m *EducationBackend) GetEducationSchoolClasses(ctx context.Context, schoo
ret := _m.Called(ctx, schoolNumberOrID)
var r0 []*libregraph.EducationClass
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationClass, error)); ok {
return rf(ctx, schoolNumberOrID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationClass); ok {
r0 = rf(ctx, schoolNumberOrID)
} else {
@@ -296,7 +324,6 @@ func (_m *EducationBackend) GetEducationSchoolClasses(ctx context.Context, schoo
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, schoolNumberOrID)
} else {
@@ -311,6 +338,10 @@ func (_m *EducationBackend) GetEducationSchoolUsers(ctx context.Context, id stri
ret := _m.Called(ctx, id)
var r0 []*libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]*libregraph.EducationUser, error)); ok {
return rf(ctx, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []*libregraph.EducationUser); ok {
r0 = rf(ctx, id)
} else {
@@ -319,7 +350,6 @@ func (_m *EducationBackend) GetEducationSchoolUsers(ctx context.Context, id stri
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, id)
} else {
@@ -334,6 +364,10 @@ func (_m *EducationBackend) GetEducationSchools(ctx context.Context) ([]*libregr
ret := _m.Called(ctx)
var r0 []*libregraph.EducationSchool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationSchool, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationSchool); ok {
r0 = rf(ctx)
} else {
@@ -342,7 +376,6 @@ func (_m *EducationBackend) GetEducationSchools(ctx context.Context) ([]*libregr
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
@@ -357,6 +390,10 @@ func (_m *EducationBackend) GetEducationUser(ctx context.Context, nameOrID strin
ret := _m.Called(ctx, nameOrID)
var r0 *libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (*libregraph.EducationUser, error)); ok {
return rf(ctx, nameOrID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) *libregraph.EducationUser); ok {
r0 = rf(ctx, nameOrID)
} else {
@@ -365,7 +402,6 @@ func (_m *EducationBackend) GetEducationUser(ctx context.Context, nameOrID strin
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, nameOrID)
} else {
@@ -380,6 +416,10 @@ func (_m *EducationBackend) GetEducationUsers(ctx context.Context) ([]*libregrap
ret := _m.Called(ctx)
var r0 []*libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]*libregraph.EducationUser, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []*libregraph.EducationUser); ok {
r0 = rf(ctx)
} else {
@@ -388,7 +428,6 @@ func (_m *EducationBackend) GetEducationUsers(ctx context.Context) ([]*libregrap
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
@@ -445,6 +484,10 @@ func (_m *EducationBackend) UpdateEducationClass(ctx context.Context, id string,
ret := _m.Called(ctx, id, class)
var r0 *libregraph.EducationClass
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationClass) (*libregraph.EducationClass, error)); ok {
return rf(ctx, id, class)
}
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationClass) *libregraph.EducationClass); ok {
r0 = rf(ctx, id, class)
} else {
@@ -453,7 +496,6 @@ func (_m *EducationBackend) UpdateEducationClass(ctx context.Context, id string,
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationClass) error); ok {
r1 = rf(ctx, id, class)
} else {
@@ -468,6 +510,10 @@ func (_m *EducationBackend) UpdateEducationSchool(ctx context.Context, numberOrI
ret := _m.Called(ctx, numberOrID, school)
var r0 *libregraph.EducationSchool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationSchool) (*libregraph.EducationSchool, error)); ok {
return rf(ctx, numberOrID, school)
}
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationSchool) *libregraph.EducationSchool); ok {
r0 = rf(ctx, numberOrID, school)
} else {
@@ -476,7 +522,6 @@ func (_m *EducationBackend) UpdateEducationSchool(ctx context.Context, numberOrI
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationSchool) error); ok {
r1 = rf(ctx, numberOrID, school)
} else {
@@ -491,6 +536,10 @@ func (_m *EducationBackend) UpdateEducationUser(ctx context.Context, nameOrID st
ret := _m.Called(ctx, nameOrID, user)
var r0 *libregraph.EducationUser
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationUser) (*libregraph.EducationUser, error)); ok {
return rf(ctx, nameOrID, user)
}
if rf, ok := ret.Get(0).(func(context.Context, string, libregraph.EducationUser) *libregraph.EducationUser); ok {
r0 = rf(ctx, nameOrID, user)
} else {
@@ -499,7 +548,6 @@ func (_m *EducationBackend) UpdateEducationUser(ctx context.Context, nameOrID st
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, libregraph.EducationUser) error); ok {
r1 = rf(ctx, nameOrID, user)
} else {
services/graph/pkg/service/v0/graph.go
+4
View File
@@ -45,6 +45,10 @@ type GatewayClient interface {
// Returns the resource information at the provided reference.
// MUST return CODE_NOT_FOUND if the reference does not exist.
Stat(ctx context.Context, in *provider.StatRequest, opts ...grpc.CallOption) (*provider.StatResponse, error)
// TouchFile allows to touch a file
TouchFile(ctx context.Context, req *provider.TouchFileRequest, opts ...grpc.CallOption) (*provider.TouchFileResponse, error)
// Initiates the upload of a file using an out-of-band data transfer mechanism.
InitiateFileUpload(ctx context.Context, in *provider.InitiateFileUploadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileUploadResponse, error)
// Initiates the download of a file using an
// out-of-band data transfer mechanism.
InitiateFileDownload(ctx context.Context, in *provider.InitiateFileDownloadRequest, opts ...grpc.CallOption) (*gateway.InitiateFileDownloadResponse, error)
services/graph/pkg/service/v0/personaldata.go
+126
View File
@@ -0,0 +1,126 @@
package svc
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revaCtx "github.com/cs3org/reva/v2/pkg/ctx"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/cs3org/reva/v2/pkg/utils"
)
var (
_backupFileName = "personal_data_export.json"
// TokenTransportHeader holds the header key for the reva transfer token
TokenTransportHeader = "X-Reva-Transfer"
)
// ExportPersonalDataRequest is the body of the request
type ExportPersonalDataRequest struct {
StorageLocation string `json:"storageLocation"`
}
// ExportPersonalData exports all personal data the system holds
func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
u := revactx.ContextMustGetUser(ctx)
// Get location from request
loc := ""
if loc == "" {
loc = _backupFileName
}
ref := &provider.Reference{
ResourceId: &provider.ResourceId{SpaceId: u.GetId().GetOpaqueId(), OpaqueId: u.GetId().GetOpaqueId()},
Path: loc,
}
// touch file
gwc := g.GetGatewayClient()
resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{
Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"),
Ref: ref,
})
if err != nil || resp.GetStatus().GetCode() != rpc.Code_CODE_OK {
g.logger.Error().Err(err).Str("status", resp.GetStatus().GetMessage()).Msg("error touching file")
w.WriteHeader(http.StatusInternalServerError)
return
}
// go start gathering
go func() {
time.Sleep(10 * time.Second)
by, _ := json.Marshal(map[string]string{u.GetId().GetOpaqueId(): "no data stored"})
b := bytes.NewBuffer(by)
th := r.Header.Get(revaCtx.TokenHeader)
err := g.upload(u, b, ref, th)
fmt.Println("Upload error", err)
}()
w.WriteHeader(http.StatusOK)
return
}
func (g Graph) upload(u *user.User, data io.Reader, ref *provider.Reference, th string) error {
uReq := &provider.InitiateFileUploadRequest{
Ref: ref,
//Opaque: &typespb.Opaque{
//Map: map[string]*typespb.OpaqueEntry{
//"Upload-Length": {
//Decoder: "plain",
//// TODO: handle case where size is not known in advance
//Value: []byte(strconv.FormatUint(cp.sourceInfo.GetSize(), 10)),
//},
//},
//},
}
gwc := g.GetGatewayClient()
ctx, _, err := utils.Impersonate(u.GetId(), gwc.(gateway.GatewayAPIClient), g.config.MachineAuthAPIKey)
ctx = revaCtx.ContextSetToken(ctx, th)
uRes, err := gwc.InitiateFileUpload(ctx, uReq)
if err != nil {
return err
}
if uRes.Status.Code != rpc.Code_CODE_OK {
return fmt.Errorf("wrong status code while initiating upload: %s", uRes.GetStatus().GetMessage())
}
var uploadEP, uploadToken string
for _, p := range uRes.Protocols {
if p.Protocol == "simple" {
uploadEP, uploadToken = p.UploadEndpoint, p.Token
}
}
httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, data)
if err != nil {
return err
}
httpUploadReq.Header.Set(TokenTransportHeader, uploadToken)
httpUploadRes, err := rhttp.GetHTTPClient(
// rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))),
rhttp.Insecure(true),
).Do(httpUploadReq)
if err != nil {
return err
}
defer httpUploadRes.Body.Close()
if httpUploadRes.StatusCode != http.StatusOK {
return fmt.Errorf("wrong status uploading file: %d", httpUploadRes.StatusCode)
}
return nil
}
services/graph/pkg/service/v0/service.go
+1
View File
@@ -207,6 +207,7 @@ func NewService(opts ...Option) (Graph, error) {
r.Route("/{userID}", func(r chi.Router) {
r.Get("/", svc.GetUser)
r.Get("/drive", svc.GetUserDrive)
r.Post("/exportPersonalData", svc.ExportPersonalData)
r.With(requireAdmin).Delete("/", svc.DeleteUser)
r.With(requireAdmin).Patch("/", svc.PatchUser)
if svc.roleService != nil {