File events audit logging (#3332)

* use feature reva & file config

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* pre-implementation

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* add changelog

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* prerequesists for unit tests

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* file created event

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* remaining tests & events

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* use edge reva

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* improvements from manual testing

Signed-off-by: jkoberg <jkoberg@owncloud.com>

* use utils package to generate itemID

Signed-off-by: jkoberg <jkoberg@owncloud.com>

audit/pkg/service/service.go

@@ -61,6 +61,20 @@ func StartAuditLogger(ctx context.Context, ch <-chan interface{}, log log.Logger
 				auditEvent = types.LinkAccessed(ev)
 			case events.LinkAccessFailed:
 				auditEvent = types.LinkAccessFailed(ev)
+			case events.FileUploaded:
+				auditEvent = types.FileUploaded(ev)
+			case events.FileDownloaded:
+				auditEvent = types.FileDownloaded(ev)
+			case events.ItemMoved:
+				auditEvent = types.ItemMoved(ev)
+			case events.ItemTrashed:
+				auditEvent = types.ItemTrashed(ev)
+			case events.ItemPurged:
+				auditEvent = types.ItemPurged(ev)
+			case events.ItemRestored:
+				auditEvent = types.ItemRestored(ev)
+			case events.FileVersionRestored:
+				auditEvent = types.FileVersionRestored(ev)
 			default:
 				log.Error().Interface("event", ev).Msg(fmt.Sprintf("can't handle event of type '%T'", ev))
 				continue

audit/pkg/service/service_test.go

@@ -294,6 +294,124 @@ var testCases = []struct {
 			require.Equal(t, "token-123", ev.ShareToken)
 			require.Equal(t, false, ev.Success)
 		},
+	}, {
+		Alias: "File created",
+		SystemEvent: events.FileUploaded{
+			FileID: reference("sto-123", "iid-123", "./item"),
+			Owner:  userID("uid-123"), // NOTE: owner not yet implemented in reva
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileCreated{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was created", "file_create")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+		},
+	}, {
+		Alias: "File read",
+		SystemEvent: events.FileDownloaded{
+			FileID: reference("sto-123", "iid-123", "./item"),
+			Owner:  userID("uid-123"), // NOTE: owner not yet implemented in reva
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileRead{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was read", "file_read")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+		},
+	}, {
+		Alias: "File trashed",
+		SystemEvent: events.ItemTrashed{
+			FileID: reference("sto-123", "iid-123", "./item"),
+			Owner:  userID("uid-123"), // NOTE: owner not yet implemented in reva
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileDeleted{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was trashed", "file_delete")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+		},
+	}, {
+		Alias: "File renamed",
+		SystemEvent: events.ItemMoved{
+			FileID:       reference("sto-123", "iid-123", "./item"),
+			OldReference: reference("sto-123", "iid-123", "./anotheritem"),
+			Owner:        userID("uid-123"), // NOTE: owner not yet implemented in reva
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileRenamed{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was moved from './anotheritem' to './item'", "file_rename")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+			// AuditEventFileRenamed fields
+			require.Equal(t, "./anotheritem", ev.OldPath)
+
+		},
+	}, {
+		Alias: "File purged",
+		SystemEvent: events.ItemPurged{
+			FileID: reference("sto-123", "iid-123", "./item"),
+			Owner:  userID("uid-123"), // NOTE: owner not yet implemented in reva
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFilePurged{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was removed from trashbin", "file_trash_delete")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+		},
+	}, {
+		Alias: "File restored",
+		SystemEvent: events.ItemRestored{
+			FileID:       reference("sto-123", "iid-123", "./item"),
+			Owner:        userID("uid-123"), // NOTE: owner not yet implemented in reva
+			OldReference: reference("sto-123", "sto-123!iid-123/item", "./oldpath"),
+			Key:          "",
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileRestored{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was restored from trashbin to './item'", "file_trash_restore")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+			// AuditEventFileRestored fields
+			require.Equal(t, "./oldpath", ev.OldPath)
+
+		},
+	}, {
+		Alias: "File version restored",
+		SystemEvent: events.FileVersionRestored{
+			FileID: reference("sto-123", "iid-123", "./item"),
+			Owner:  userID("uid-123"), // NOTE: owner not yet implemented in reva
+			Key:    "v1",
+		},
+		CheckAuditEvent: func(t *testing.T, b []byte) {
+			ev := types.AuditEventFileVersionRestored{}
+			require.NoError(t, json.Unmarshal(b, &ev))
+
+			// AuditEvent fields
+			checkBaseAuditEvent(t, ev.AuditEvent, "uid-123", "", "File 'sto-123!iid-123/item' was restored in version 'v1'", "file_version_restore")
+			// AuditEventSharing fields
+			checkFilesAuditEvent(t, ev.AuditEventFiles, "sto-123!iid-123/item", "uid-123", "./item")
+			// AuditEventFileRestored fields
+			require.Equal(t, "v1", ev.Key)
+
+		},
 	},
 }
 
@@ -343,6 +461,12 @@ func checkSharingAuditEvent(t *testing.T, ev types.AuditEventSharing, itemID str
 	require.Equal(t, shareID, ev.ShareID)
 }
 
+func checkFilesAuditEvent(t *testing.T, ev types.AuditEventFiles, itemID string, owner string, path string) {
+	require.Equal(t, itemID, ev.FileID)
+	require.Equal(t, owner, ev.Owner)
+	require.Equal(t, path, ev.Path)
+}
+
 func shareID(id string) *collaboration.ShareId {
 	return &collaboration.ShareId{
 		OpaqueId: id,
@@ -376,6 +500,13 @@ func resourceID(sid, oid string) *provider.ResourceId {
 	}
 }
 
+func reference(sid, oid, path string) *provider.Reference {
+	return &provider.Reference{
+		ResourceId: resourceID(sid, oid),
+		Path:       path,
+	}
+}
+
 func timestamp(seconds uint64) *rtypes.Timestamp {
 	return &rtypes.Timestamp{
 		Seconds: seconds,
@@ -394,6 +525,7 @@ func linkPermissions(perms ...string) *link.PublicSharePermissions {
 		Permissions: permissions(perms...),
 	}
 }
+
 func permissions(permissions ...string) *provider.ResourcePermissions {
 	perms := &provider.ResourcePermissions{}
 

audit/pkg/types/constants.go

@@ -4,6 +4,7 @@ import "fmt"
 
 // short identifiers for audit actions
 const (
+	// Sharing
 	ActionShareCreated            = "file_shared"
 	ActionSharePermissionUpdated  = "share_permission_updated"
 	ActionShareDisplayNameUpdated = "share_name_updated"
@@ -13,6 +14,15 @@ const (
 	ActionShareAccepted           = "share_accepted"
 	ActionShareDeclined           = "share_declined"
 	ActionLinkAccessed            = "public_link_accessed"
+
+	// Files
+	ActionFileCreated         = "file_create"
+	ActionFileRead            = "file_read"
+	ActionFileTrashed         = "file_delete"
+	ActionFileRenamed         = "file_rename"
+	ActionFilePurged          = "file_trash_delete"
+	ActionFileRestored        = "file_trash_restore"
+	ActionFileVersionRestored = "file_version_restore"
 )
 
 // MessageShareCreated returns the human readable string that describes the action
@@ -59,3 +69,38 @@ func MessageShareDeclined(userid, shareid, sharerid string) string {
 func MessageLinkAccessed(linkid string, success bool) string {
 	return fmt.Sprintf("link '%s' was accessed. Success: %v", linkid, success)
 }
+
+// MessageFileCreated returns the human readable string that describes the action
+func MessageFileCreated(item string) string {
+	return fmt.Sprintf("File '%s' was created", item)
+}
+
+// MessageFileRead returns the human readable string that describes the action
+func MessageFileRead(item string) string {
+	return fmt.Sprintf("File '%s' was read", item)
+}
+
+// MessageFileTrashed returns the human readable string that describes the action
+func MessageFileTrashed(item string) string {
+	return fmt.Sprintf("File '%s' was trashed", item)
+}
+
+// MessageFileRenamed returns the human readable string that describes the action
+func MessageFileRenamed(item, oldpath, newpath string) string {
+	return fmt.Sprintf("File '%s' was moved from '%s' to '%s'", item, oldpath, newpath)
+}
+
+// MessageFilePurged returns the human readable string that describes the action
+func MessageFilePurged(item string) string {
+	return fmt.Sprintf("File '%s' was removed from trashbin", item)
+}
+
+// MessageFileRestored returns the human readable string that describes the action
+func MessageFileRestored(item, path string) string {
+	return fmt.Sprintf("File '%s' was restored from trashbin to '%s'", item, path)
+}
+
+// MessageFileVersionRestored returns the human readable string that describes the action
+func MessageFileVersionRestored(item string, version string) string {
+	return fmt.Sprintf("File '%s' was restored in version '%s'", item, version)
+}

audit/pkg/types/conversion.go

@@ -5,9 +5,11 @@ import (
 	"time"
 
 	"github.com/cs3org/reva/v2/pkg/events"
+	"github.com/cs3org/reva/v2/pkg/utils"
 
 	group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
 	user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
 )
 
@@ -220,6 +222,94 @@ func LinkAccessFailed(ev events.LinkAccessFailed) AuditEventLinkAccessed {
 	}
 }
 
+// FilesAuditEvent creates an AuditEventFiles from the given values
+func FilesAuditEvent(base AuditEvent, itemid, owner, path string) AuditEventFiles {
+	return AuditEventFiles{
+		AuditEvent: base,
+		FileID:     itemid,
+		Owner:      owner,
+		Path:       path,
+	}
+}
+
+// FileUploaded converts a FileUploaded event to an AuditEventFileCreated
+func FileUploaded(ev events.FileUploaded) AuditEventFileCreated {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+	base := BasicAuditEvent(uid, "", MessageFileCreated(iid), ActionFileCreated)
+	return AuditEventFileCreated{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+	}
+}
+
+// FileDownloaded converts a FileDownloaded event to an AuditEventFileRead
+func FileDownloaded(ev events.FileDownloaded) AuditEventFileRead {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+	base := BasicAuditEvent(uid, "", MessageFileRead(iid), ActionFileRead)
+	return AuditEventFileRead{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+	}
+}
+
+// ItemMoved converts a ItemMoved event to an AuditEventFileRenamed
+func ItemMoved(ev events.ItemMoved) AuditEventFileRenamed {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+
+	oldpath := ""
+	if ev.OldReference != nil {
+		oldpath = ev.OldReference.GetPath()
+	}
+
+	base := BasicAuditEvent(uid, "", MessageFileRenamed(iid, oldpath, path), ActionFileRenamed)
+	return AuditEventFileRenamed{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+		OldPath:         oldpath,
+	}
+}
+
+// ItemTrashed converts a ItemTrashed event to an AuditEventFileDeleted
+func ItemTrashed(ev events.ItemTrashed) AuditEventFileDeleted {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+	base := BasicAuditEvent(uid, "", MessageFileTrashed(iid), ActionFileTrashed)
+	return AuditEventFileDeleted{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+	}
+}
+
+// ItemPurged converts a ItemPurged event to an AuditEventFilePurged
+func ItemPurged(ev events.ItemPurged) AuditEventFilePurged {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+	base := BasicAuditEvent(uid, "", MessageFilePurged(iid), ActionFilePurged)
+	return AuditEventFilePurged{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+	}
+}
+
+// ItemRestored converts a ItemRestored event to an AuditEventFileRestored
+func ItemRestored(ev events.ItemRestored) AuditEventFileRestored {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+
+	oldpath := ""
+	if ev.OldReference != nil {
+		oldpath = ev.OldReference.GetPath()
+	}
+
+	base := BasicAuditEvent(uid, "", MessageFileRestored(iid, path), ActionFileRestored)
+	return AuditEventFileRestored{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+		OldPath:         oldpath,
+	}
+}
+
+// FileVersionRestored converts a FileVersionRestored event to an AuditEventFileVersionRestored
+func FileVersionRestored(ev events.FileVersionRestored) AuditEventFileVersionRestored {
+	iid, path, uid := extractFileDetails(ev.FileID, ev.Owner)
+	base := BasicAuditEvent(uid, "", MessageFileVersionRestored(iid, ev.Key), ActionFileVersionRestored)
+	return AuditEventFileVersionRestored{
+		AuditEventFiles: FilesAuditEvent(base, iid, uid, path),
+		Key:             ev.Key,
+	}
+}
+
 func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) {
 	switch {
 	case uid != nil && uid.OpaqueId != "":
@@ -231,6 +321,20 @@ func extractGrantee(uid *user.UserId, gid *group.GroupId) (string, string) {
 	return "", ""
 }
 
+func extractFileDetails(ref *provider.Reference, owner *user.UserId) (string, string, string) {
+	id, path := "", ""
+	if ref != nil {
+		path = ref.GetPath()
+		id, _ = utils.FormatStorageSpaceReference(ref)
+	}
+
+	uid := ""
+	if owner != nil {
+		uid = owner.GetOpaqueId()
+	}
+	return id, path, uid
+}
+
 func formatTime(t *types.Timestamp) string {
 	if t == nil {
 		return ""

audit/pkg/types/events.go

@@ -16,5 +16,12 @@ func RegisteredEvents() []events.Unmarshaller {
 		events.ReceivedShareUpdated{},
 		events.LinkAccessed{},
 		events.LinkAccessFailed{},
+		events.FileUploaded{},
+		events.FileDownloaded{},
+		events.ItemTrashed{},
+		events.ItemMoved{},
+		events.ItemPurged{},
+		events.ItemRestored{},
+		events.FileVersionRestored{},
 	}
 }

audit/pkg/types/types.go

@@ -15,6 +15,10 @@ type AuditEvent struct {
 	Level      int    // the log level of the entry (usually 1 for audit events)
 }
 
+/*
+   Sharing
+*/
+
 // AuditEventSharing is the basic audit event for shares
 type AuditEventSharing struct {
 	AuditEvent
@@ -76,3 +80,77 @@ type AuditEventLinkAccessed struct {
 	Success    bool   // If the request was successful.
 	ItemType   string // file or folder
 }
+
+/*
+   Files
+*/
+
+// AuditEventFiles is the basic audit event for files
+type AuditEventFiles struct {
+	AuditEvent
+
+	Path   string // The full path to the create file.
+	Owner  string // The UID of the owner of the file.
+	FileID string // The newly created files identifier.
+}
+
+// AuditEventFileCreated is the event logged when a file is created
+type AuditEventFileCreated struct {
+	AuditEventFiles
+}
+
+// AuditEventFileRead is the event logged when a file is read (aka downloaded)
+type AuditEventFileRead struct {
+	AuditEventFiles
+}
+
+// AuditEventFileUpdated is the event logged when a file is updated
+// TODO: How to differentiate between new uploads and new version uploads?
+// FIXME: implement
+type AuditEventFileUpdated struct {
+	AuditEventFiles
+}
+
+// AuditEventFileDeleted is the event logged when a file is deleted (aka trashed)
+type AuditEventFileDeleted struct {
+	AuditEventFiles
+}
+
+// AuditEventFileCopied is the event logged when a file is copied
+// TODO: copy is a download&upload for now. How to know it was a copy?
+// FIXME: implement
+type AuditEventFileCopied struct {
+	AuditEventFiles
+}
+
+// AuditEventFileRenamed is the event logged when a file is renamed (moved)
+type AuditEventFileRenamed struct {
+	AuditEventFiles
+
+	OldPath string
+}
+
+// AuditEventFilePurged is the event logged when a file is purged (deleted from trashbin)
+type AuditEventFilePurged struct {
+	AuditEventFiles
+}
+
+// AuditEventFileRestored is the event logged when a file is restored (from trashbin)
+type AuditEventFileRestored struct {
+	AuditEventFiles
+
+	OldPath string
+}
+
+// AuditEventFileVersionRestored is the event logged when a file version is restored
+type AuditEventFileVersionRestored struct {
+	AuditEventFiles
+
+	Key string
+}
+
+// AuditEventFileVersionDeleted is the event logged when a file version is deleted
+// TODO: is this even possible?
+type AuditEventFileVersionDeleted struct {
+	AuditEventFiles
+}

changelog/unreleased/file-events-audit-logging.md

@@ -0,0 +1,5 @@
+Enhancement: Audit logger will now log file events
+
+See full list of supported events in `audit/pkg/types/types.go`
+
+https://github.com/owncloud/ocis/pull/3332

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.3.1
 	github.com/coreos/go-oidc/v3 v3.1.0
 	github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19
-	github.com/cs3org/reva/v2 v2.0.0-20220316045927-99115670eb33
+	github.com/cs3org/reva/v2 v2.0.0-20220317153101-5a93e519610c
 	github.com/disintegration/imaging v1.6.2
 	github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733
 	github.com/go-chi/chi/v5 v5.0.7

go.sum

@@ -341,8 +341,8 @@ github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD9
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
 github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19 h1:1jqPH58jCxvbaJ9WLIJ7W2/m622bWS6ChptzljSG6IQ=
 github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
-github.com/cs3org/reva/v2 v2.0.0-20220316045927-99115670eb33 h1:XK88Fs9FteY9a+iKXqPhUK38zNQbP1jj60zy+Tx7SPI=
+github.com/cs3org/reva/v2 v2.0.0-20220317153101-5a93e519610c h1:tTAuVwgbDNPyeqNJPjMrT1xZ4jZYGSJ2AWqDkvSpXuA=
-github.com/cs3org/reva/v2 v2.0.0-20220316045927-99115670eb33/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0=
+github.com/cs3org/reva/v2 v2.0.0-20220317153101-5a93e519610c/go.mod h1:XNtK1HEClNzmz5vyQa2DUw4KH3oqBjQoEsV1LhAGlV0=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

storage/pkg/command/storageusers.go

@@ -112,6 +112,14 @@ func storageUsersConfigFromStruct(c *cli.Context, cfg *config.Config) map[string
 					"tmp_folder":         cfg.Reva.StorageUsers.TempFolder,
 				},
 			},
+			"interceptors": map[string]interface{}{
+				"eventsmiddleware": map[string]interface{}{
+					"group":     "sharing",
+					"type":      "nats",
+					"address":   cfg.Reva.Sharing.Events.Address,
+					"clusterID": cfg.Reva.Sharing.Events.ClusterID,
+				},
+			},
 		},
 		"http": map[string]interface{}{
 			"network": cfg.Reva.StorageUsers.HTTPNetwork,