Initial QSfera import

Server/services/graph/pkg/service/v0/personaldata.go

@@ -0,0 +1,331 @@
+package svc
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strconv"
+
+	"github.com/qsfera/server/services/graph/pkg/errorcode"
+
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
+	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/go-chi/chi/v5"
+	ehmsg "github.com/qsfera/server/protogen/gen/qsfera/messages/eventhistory/v0"
+	ehsvc "github.com/qsfera/server/protogen/gen/qsfera/services/eventhistory/v0"
+	revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
+	"github.com/opencloud-eu/reva/v2/pkg/events"
+	"github.com/opencloud-eu/reva/v2/pkg/rhttp"
+	"github.com/opencloud-eu/reva/v2/pkg/utils"
+)
+
+var (
+	_backupFileName = "personal_data_export.json"
+
+	// TokenTransportHeader holds the header key for the reva transfer token
+	TokenTransportHeader = "X-Reva-Transfer"
+)
+
+// Marshaller is the common interface for a marshaller
+type Marshaller func(any) ([]byte, error)
+
+// ExportPersonalDataRequest is the body of the request
+type ExportPersonalDataRequest struct {
+	StorageLocation string `json:"storageLocation"`
+}
+
+func (g Graph) getPersonalSpace(ctx context.Context, u *user.UserId) (*provider.StorageSpace, error) {
+
+	// we cannot assume the space id is the same as the user id, so we have to look up his personal space first
+	lReq := &storageprovider.ListStorageSpacesRequest{
+		Filters: []*storageprovider.ListStorageSpacesRequest_Filter{
+			{
+				Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_OWNER,
+				Term: &storageprovider.ListStorageSpacesRequest_Filter_Owner{
+					Owner: u,
+				},
+			},
+			{
+				Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE,
+				Term: &storageprovider.ListStorageSpacesRequest_Filter_SpaceType{
+					SpaceType: "personal",
+				},
+			},
+		},
+	}
+
+	gatewayClient, err := g.gatewaySelector.Next()
+	if err != nil {
+		return nil, err
+	}
+	res, err := gatewayClient.ListStorageSpaces(ctx, lReq)
+	if err != nil {
+		return nil, err
+	}
+	if res.Status.Code != rpc.Code_CODE_OK {
+		return nil, errorcode.FromCS3Status(res.Status, fmt.Errorf("could not list storage spaces"))
+	}
+	if len(res.GetStorageSpaces()) == 0 {
+		return nil, fmt.Errorf("could not find personal space for user %s", u.GetOpaqueId())
+	}
+
+	return res.GetStorageSpaces()[0], nil
+}
+
+// ExportPersonalData exports all personal data the system holds
+func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	u := revactx.ContextMustGetUser(ctx)
+	if reqUserID := chi.URLParam(r, "userID"); reqUserID != u.GetId().GetOpaqueId() {
+		g.logger.Info().Msg("uid mismatch")
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte("personal data export for other users are not permitted"))
+		return
+	}
+	// Get location from request
+	loc := getLocation(r)
+
+	// prepare marshaller
+	var marsh Marshaller
+	switch filepath.Ext(loc) {
+	default:
+		g.logger.Info().Str("path", loc).Msg("invalid location")
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte("only json format is supported for personal data export"))
+		return
+	case ".json":
+		marsh = json.Marshal
+	}
+
+	personalSpace, err := g.getPersonalSpace(ctx, u.GetId())
+	if err != nil {
+		g.logger.Error().Err(err).Msg("could not get personal space")
+		errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, "could not get personal space, aborting")
+		return
+	}
+
+	ref := &provider.Reference{
+		ResourceId: personalSpace.GetRoot(),
+		Path:       loc,
+	}
+
+	gatewayClient, err := g.gatewaySelector.Next()
+	if err != nil {
+		g.logger.Error().Err(err).Msg("could not select next gateway client")
+		errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, "could not select next gateway client, aborting")
+		return
+	}
+
+	// touch file
+	if err := mustTouchFile(ctx, ref, gatewayClient); err != nil {
+		g.logger.Error().Err(err).Msg("error touching file")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// go start gathering
+	go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh)
+
+	w.WriteHeader(http.StatusAccepted)
+}
+
+// GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space
+func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) {
+	gatewayClient, err := g.gatewaySelector.Next()
+	if err != nil {
+		g.logger.Error().Err(err).Msg("could not select next gateway client")
+		return
+	}
+
+	// the context might already be cancelled. We need to impersonate the acting user again
+	ctx, err := utils.GetServiceUserContext(g.config.ServiceAccount.ServiceAccountID, gatewayClient, g.config.ServiceAccount.ServiceAccountSecret)
+	if err != nil {
+		g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot impersonate user")
+	}
+	// create data
+	data := make(map[string]any)
+
+	// reva user
+	data["user"] = usr
+
+	// Check if we have a keycloak client, and if so, get the keycloak export.
+	if ctx != nil && g.keycloakClient != nil {
+		kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.UserRealm, usr.GetUsername())
+		if err != nil {
+			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get keycloak personal data")
+		}
+		data["keycloak"] = kcd
+	}
+
+	// get event data
+	if ctx != nil && g.historyClient != nil {
+		resp, err := g.historyClient.GetEventsForUser(ctx, &ehsvc.GetEventsForUserRequest{UserID: usr.GetId().GetOpaqueId()})
+		if err != nil {
+			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get event personal data")
+		}
+		data["events"] = convertEvents(resp.GetEvents())
+	}
+
+	// marshal
+	by, err := marsh(data)
+	if err != nil {
+		g.logger.Error().Err(err).Msg("cannot marshal personal user data")
+		return
+	}
+
+	// upload
+	var errmsg string
+	if err := g.upload(usr, by, ref, token); err != nil {
+		g.logger.Error().Err(err).Msg("failed uploading personal data export")
+		errmsg = err.Error()
+	}
+
+	if err := events.Publish(ctx, g.eventsPublisher, events.PersonalDataExtracted{
+		Executant: usr.GetId(),
+		Timestamp: utils.TSNow(),
+		ErrorMsg:  errmsg,
+	}); err != nil {
+		g.logger.Error().Err(err).Msg("cannot publish event")
+	}
+}
+
+func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th string) error {
+	uReq := &provider.InitiateFileUploadRequest{
+		Ref:    ref,
+		Opaque: utils.AppendPlainToOpaque(nil, "Upload-Length", strconv.FormatUint(uint64(len(data)), 10)),
+	}
+
+	gatewayClient, err := g.gatewaySelector.Next()
+	if err != nil {
+		g.logger.Error().Err(err).Msg("could not select next gateway client")
+		return err
+	}
+
+	ctx, err := utils.GetServiceUserContext(g.config.ServiceAccount.ServiceAccountID, gatewayClient, g.config.ServiceAccount.ServiceAccountSecret)
+	if err != nil {
+		return err
+	}
+	client, err := g.gatewaySelector.Next()
+	if err != nil {
+		return err
+	}
+	ctx = revactx.ContextSetToken(ctx, th)
+	uRes, err := client.InitiateFileUpload(ctx, uReq)
+	if err != nil {
+		return err
+	}
+
+	if uRes.Status.Code != rpc.Code_CODE_OK {
+		return fmt.Errorf("wrong status code while initiating upload: %s", uRes.GetStatus().GetMessage())
+	}
+
+	var uploadEP, uploadToken string
+	for _, p := range uRes.Protocols {
+		if p.Protocol == "simple" {
+			uploadEP, uploadToken = p.UploadEndpoint, p.Token
+		}
+	}
+
+	httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, bytes.NewBuffer(data))
+	if err != nil {
+		return err
+	}
+	httpUploadReq.Header.Set(TokenTransportHeader, uploadToken)
+
+	httpUploadRes, err := rhttp.GetHTTPClient(rhttp.Insecure(true)).Do(httpUploadReq)
+	if err != nil {
+		return err
+	}
+	defer httpUploadRes.Body.Close()
+	if httpUploadRes.StatusCode != http.StatusOK {
+		return fmt.Errorf("wrong status uploading file: %d", httpUploadRes.StatusCode)
+	}
+
+	return nil
+}
+
+// touches the file, creating folders if necessary
+func mustTouchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
+	if err := touchFile(ctx, ref, gwc); err == nil {
+		return nil
+	}
+
+	if err := createFolders(ctx, ref, gwc); err != nil {
+		return err
+	}
+
+	return touchFile(ctx, ref, gwc)
+}
+
+func touchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
+	resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{
+		Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"),
+		Ref:    ref,
+	})
+	if err != nil {
+		return err
+	}
+	if resp.GetStatus().GetCode() != rpc.Code_CODE_OK {
+		return fmt.Errorf("unexpected statuscode while touching file: %d %s", resp.GetStatus().GetCode(), resp.GetStatus().GetMessage())
+	}
+	return nil
+}
+
+func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
+	var paths []string
+	p := filepath.Dir(ref.GetPath())
+	for p != "." {
+		paths = append([]string{p}, paths...)
+		p = filepath.Dir(p)
+	}
+
+	for _, p := range paths {
+		r := &provider.Reference{ResourceId: ref.GetResourceId(), Path: p}
+		resp, err := gwc.CreateContainer(ctx, &provider.CreateContainerRequest{Ref: r})
+		if err != nil {
+			return err
+		}
+
+		code := resp.GetStatus().GetCode()
+		if code != rpc.Code_CODE_OK && code != rpc.Code_CODE_ALREADY_EXISTS {
+			return fmt.Errorf("unexpected statuscode while creating folder: %d %s", code, resp.GetStatus().GetMessage())
+		}
+	}
+	return nil
+}
+
+func getLocation(r *http.Request) string {
+	// from body
+	var req ExportPersonalDataRequest
+	if b, err := io.ReadAll(r.Body); err == nil {
+		if err := json.Unmarshal(b, &req); err == nil && req.StorageLocation != "" {
+			return req.StorageLocation
+		}
+	}
+
+	// from header?
+
+	return _backupFileName
+}
+
+// we want the events to look nice in the file, don't we?
+func convertEvents(evs []*ehmsg.Event) []map[string]any {
+	out := make([]map[string]any, len(evs))
+	for i, e := range evs {
+		var content map[string]any
+		_ = json.Unmarshal(e.GetEvent(), &content)
+		out[i] = map[string]any{
+			"id":    e.GetId(),
+			"type":  e.GetType(),
+			"event": content,
+		}
+	}
+	return out
+}