QSfera/Server/services/graph/pkg/service/v0/personaldata.go

package svc

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"strconv"

	"github.com/qsfera/server/services/graph/pkg/errorcode"

	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
	storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
	"github.com/go-chi/chi/v5"
	ehmsg "github.com/qsfera/server/protogen/gen/qsfera/messages/eventhistory/v0"
	ehsvc "github.com/qsfera/server/protogen/gen/qsfera/services/eventhistory/v0"
	revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
	"github.com/opencloud-eu/reva/v2/pkg/events"
	"github.com/opencloud-eu/reva/v2/pkg/rhttp"
	"github.com/opencloud-eu/reva/v2/pkg/utils"
)

var (
	_backupFileName = "personal_data_export.json"

	// TokenTransportHeader holds the header key for the reva transfer token
	TokenTransportHeader = "X-Reva-Transfer"
)

// Marshaller is the common interface for a marshaller
type Marshaller func(any) ([]byte, error)

// ExportPersonalDataRequest is the body of the request
type ExportPersonalDataRequest struct {
	StorageLocation string `json:"storageLocation"`
}

func (g Graph) getPersonalSpace(ctx context.Context, u *user.UserId) (*provider.StorageSpace, error) {

	// we cannot assume the space id is the same as the user id, so we have to look up his personal space first
	lReq := &storageprovider.ListStorageSpacesRequest{
		Filters: []*storageprovider.ListStorageSpacesRequest_Filter{
			{
				Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_OWNER,
				Term: &storageprovider.ListStorageSpacesRequest_Filter_Owner{
					Owner: u,
				},
			},
			{
				Type: storageprovider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE,
				Term: &storageprovider.ListStorageSpacesRequest_Filter_SpaceType{
					SpaceType: "personal",
				},
			},
		},
	}

	gatewayClient, err := g.gatewaySelector.Next()
	if err != nil {
		return nil, err
	}
	res, err := gatewayClient.ListStorageSpaces(ctx, lReq)
	if err != nil {
		return nil, err
	}
	if res.Status.Code != rpc.Code_CODE_OK {
		return nil, errorcode.FromCS3Status(res.Status, fmt.Errorf("could not list storage spaces"))
	}
	if len(res.GetStorageSpaces()) == 0 {
		return nil, fmt.Errorf("could not find personal space for user %s", u.GetOpaqueId())
	}

	return res.GetStorageSpaces()[0], nil
}

// ExportPersonalData exports all personal data the system holds
func (g Graph) ExportPersonalData(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	u := revactx.ContextMustGetUser(ctx)
	if reqUserID := chi.URLParam(r, "userID"); reqUserID != u.GetId().GetOpaqueId() {
		g.logger.Info().Msg("uid mismatch")
		w.WriteHeader(http.StatusBadRequest)
		_, _ = w.Write([]byte("personal data export for other users are not permitted"))
		return
	}
	// Get location from request
	loc := getLocation(r)

	// prepare marshaller
	var marsh Marshaller
	switch filepath.Ext(loc) {
	default:
		g.logger.Info().Str("path", loc).Msg("invalid location")
		w.WriteHeader(http.StatusBadRequest)
		_, _ = w.Write([]byte("only json format is supported for personal data export"))
		return
	case ".json":
		marsh = json.Marshal
	}

	personalSpace, err := g.getPersonalSpace(ctx, u.GetId())
	if err != nil {
		g.logger.Error().Err(err).Msg("could not get personal space")
		errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, "could not get personal space, aborting")
		return
	}

	ref := &provider.Reference{
		ResourceId: personalSpace.GetRoot(),
		Path:       loc,
	}

	gatewayClient, err := g.gatewaySelector.Next()
	if err != nil {
		g.logger.Error().Err(err).Msg("could not select next gateway client")
		errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, "could not select next gateway client, aborting")
		return
	}

	// touch file
	if err := mustTouchFile(ctx, ref, gatewayClient); err != nil {
		g.logger.Error().Err(err).Msg("error touching file")
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	// go start gathering
	go g.GatherPersonalData(u, ref, r.Header.Get(revactx.TokenHeader), marsh)

	w.WriteHeader(http.StatusAccepted)
}

// GatherPersonalData will all gather all personal data of the user and save it to a file in the users personal space
func (g Graph) GatherPersonalData(usr *user.User, ref *provider.Reference, token string, marsh Marshaller) {
	gatewayClient, err := g.gatewaySelector.Next()
	if err != nil {
		g.logger.Error().Err(err).Msg("could not select next gateway client")
		return
	}

	// the context might already be cancelled. We need to impersonate the acting user again
	ctx, err := utils.GetServiceUserContext(g.config.ServiceAccount.ServiceAccountID, gatewayClient, g.config.ServiceAccount.ServiceAccountSecret)
	if err != nil {
		g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot impersonate user")
	}
	// create data
	data := make(map[string]any)

	// reva user
	data["user"] = usr

	// Check if we have a keycloak client, and if so, get the keycloak export.
	if ctx != nil && g.keycloakClient != nil {
		kcd, err := g.keycloakClient.GetPIIReport(ctx, g.config.Keycloak.UserRealm, usr.GetUsername())
		if err != nil {
			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get keycloak personal data")
		}
		data["keycloak"] = kcd
	}

	// get event data
	if ctx != nil && g.historyClient != nil {
		resp, err := g.historyClient.GetEventsForUser(ctx, &ehsvc.GetEventsForUserRequest{UserID: usr.GetId().GetOpaqueId()})
		if err != nil {
			g.logger.Error().Err(err).Str("userID", usr.GetId().GetOpaqueId()).Msg("cannot get event personal data")
		}
		data["events"] = convertEvents(resp.GetEvents())
	}

	// marshal
	by, err := marsh(data)
	if err != nil {
		g.logger.Error().Err(err).Msg("cannot marshal personal user data")
		return
	}

	// upload
	var errmsg string
	if err := g.upload(usr, by, ref, token); err != nil {
		g.logger.Error().Err(err).Msg("failed uploading personal data export")
		errmsg = err.Error()
	}

	if err := events.Publish(ctx, g.eventsPublisher, events.PersonalDataExtracted{
		Executant: usr.GetId(),
		Timestamp: utils.TSNow(),
		ErrorMsg:  errmsg,
	}); err != nil {
		g.logger.Error().Err(err).Msg("cannot publish event")
	}
}

func (g Graph) upload(u *user.User, data []byte, ref *provider.Reference, th string) error {
	uReq := &provider.InitiateFileUploadRequest{
		Ref:    ref,
		Opaque: utils.AppendPlainToOpaque(nil, "Upload-Length", strconv.FormatUint(uint64(len(data)), 10)),
	}

	gatewayClient, err := g.gatewaySelector.Next()
	if err != nil {
		g.logger.Error().Err(err).Msg("could not select next gateway client")
		return err
	}

	ctx, err := utils.GetServiceUserContext(g.config.ServiceAccount.ServiceAccountID, gatewayClient, g.config.ServiceAccount.ServiceAccountSecret)
	if err != nil {
		return err
	}
	client, err := g.gatewaySelector.Next()
	if err != nil {
		return err
	}
	ctx = revactx.ContextSetToken(ctx, th)
	uRes, err := client.InitiateFileUpload(ctx, uReq)
	if err != nil {
		return err
	}

	if uRes.Status.Code != rpc.Code_CODE_OK {
		return fmt.Errorf("wrong status code while initiating upload: %s", uRes.GetStatus().GetMessage())
	}

	var uploadEP, uploadToken string
	for _, p := range uRes.Protocols {
		if p.Protocol == "simple" {
			uploadEP, uploadToken = p.UploadEndpoint, p.Token
		}
	}

	httpUploadReq, err := rhttp.NewRequest(ctx, "PUT", uploadEP, bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	httpUploadReq.Header.Set(TokenTransportHeader, uploadToken)

	httpUploadRes, err := rhttp.GetHTTPClient(rhttp.Insecure(true)).Do(httpUploadReq)
	if err != nil {
		return err
	}
	defer httpUploadRes.Body.Close()
	if httpUploadRes.StatusCode != http.StatusOK {
		return fmt.Errorf("wrong status uploading file: %d", httpUploadRes.StatusCode)
	}

	return nil
}

// touches the file, creating folders if necessary
func mustTouchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
	if err := touchFile(ctx, ref, gwc); err == nil {
		return nil
	}

	if err := createFolders(ctx, ref, gwc); err != nil {
		return err
	}

	return touchFile(ctx, ref, gwc)
}

func touchFile(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
	resp, err := gwc.TouchFile(ctx, &provider.TouchFileRequest{
		Opaque: utils.AppendPlainToOpaque(nil, "markprocessing", "true"),
		Ref:    ref,
	})
	if err != nil {
		return err
	}
	if resp.GetStatus().GetCode() != rpc.Code_CODE_OK {
		return fmt.Errorf("unexpected statuscode while touching file: %d %s", resp.GetStatus().GetCode(), resp.GetStatus().GetMessage())
	}
	return nil
}

func createFolders(ctx context.Context, ref *provider.Reference, gwc gateway.GatewayAPIClient) error {
	var paths []string
	p := filepath.Dir(ref.GetPath())
	for p != "." {
		paths = append([]string{p}, paths...)
		p = filepath.Dir(p)
	}

	for _, p := range paths {
		r := &provider.Reference{ResourceId: ref.GetResourceId(), Path: p}
		resp, err := gwc.CreateContainer(ctx, &provider.CreateContainerRequest{Ref: r})
		if err != nil {
			return err
		}

		code := resp.GetStatus().GetCode()
		if code != rpc.Code_CODE_OK && code != rpc.Code_CODE_ALREADY_EXISTS {
			return fmt.Errorf("unexpected statuscode while creating folder: %d %s", code, resp.GetStatus().GetMessage())
		}
	}
	return nil
}

func getLocation(r *http.Request) string {
	// from body
	var req ExportPersonalDataRequest
	if b, err := io.ReadAll(r.Body); err == nil {
		if err := json.Unmarshal(b, &req); err == nil && req.StorageLocation != "" {
			return req.StorageLocation
		}
	}

	// from header?

	return _backupFileName
}

// we want the events to look nice in the file, don't we?
func convertEvents(evs []*ehmsg.Event) []map[string]any {
	out := make([]map[string]any, len(evs))
	for i, e := range evs {
		var content map[string]any
		_ = json.Unmarshal(e.GetEvent(), &content)
		out[i] = map[string]any{
			"id":    e.GetId(),
			"type":  e.GetType(),
			"event": content,
		}
	}
	return out
}