enhancement: add mimetype to file extension rego function (#6133)

* enhancement: add mimetype to file extension rego function

add rego function to detect the resource extension by mimetype, at the same time this pr introduces a custom ocis namespace for the rego functions.

* enhancement: add custom logPrinter to opa policies service

* fix: imports and test

TypeByExtension which is used to resolve extension by mimetype relies on MIME-info database which differs at my local env (macos <-> drone). This is fixed by using one of the builtinTypes for testing

---------

Signed-off-by: Christian Richter <crichter@owncloud.com>
Co-authored-by: Christian Richter <crichter@owncloud.com>

changelog/unreleased/enhancement-add-postprocessing-mimetype-to-extension.md

@@ -0,0 +1,6 @@
+Enhancement: Add postprocessing mimetype to extension helper
+
+Add rego helper to resolve extensions from mimetype `ocis.mimetype.extensions(mimetype)`.
+Besides that, a rego print helper is included also `print("PRINT MESSAGE EXAMPLE")`
+
+https://github.com/owncloud/ocis/pull/6133

deployments/examples/service_policies/policies/postprocessing.rego

@@ -6,5 +6,12 @@ import data.utils
 default granted := true
 
 granted = false if {
-    not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.resource.name)
+    not utils.is_extension_allowed(input.resource.name)
+}
+
+granted = false if {
+    bytes := ocis.resource.download(input.resource.url)
+    mimetype := ocis.mimetype.detect(bytes)
+
+    not utils.is_mimetype_allowed(mimetype)
 }

deployments/examples/service_policies/policies/proxy.rego

@@ -6,13 +6,14 @@ import data.utils
 default granted := true
 
 granted = false if {
-    utils.is_request_type_put
+    print("PRINT MESSAGE EXAMPLE")
-    not input.request.path == "/data"
+    input.request.method == "PUT"
-    not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.request.path)
+    not startswith(input.request.path, "/ocs")
+    not utils.is_extension_allowed(input.request.path)
 }
 
 granted = false if {
-    utils.is_request_type_post
+    input.request.method == "POST"
     startswith(input.request.path, "/remote.php")
-    not utils.collection_contains(utils.ALLOWED_FILE_EXTENSIONS, input.resource.name)
+    not utils.is_extension_allowed(input.resource.name)
 }

deployments/examples/service_policies/policies/utils.rego

@@ -1,8 +1,6 @@
 package utils
 
-import future.keywords.if
+ALLOWED_RESOURCE_EXTENSIONS := [
-
-ALLOWED_FILE_EXTENSIONS := [
     ".apk", ".avi", ".bat", ".bmp", ".css", ".csv", ".doc", ".docm", ".docx",
     ".docxf", ".dotx", ".eml", ".epub", ".htm", ".html", ".ipa", ".jar", ".java",
     ".jpg", ".js", ".json", ".mp3", ".mp4", ".msg", ".odp", ".ods", ".odt", ".oform",
@@ -11,43 +9,14 @@ ALLOWED_FILE_EXTENSIONS := [
     ".txt", ".xls", ".xlsm", ".xlsx", ".xltm", ".xltx", ".xml", ".zip", ".md"
 ]
 
-##
+is_extension_allowed(identifier) {
-
+     extension := ALLOWED_RESOURCE_EXTENSIONS[_]
-is_stage_http {
+     endswith(identifier, extension)
-    input.stage == "http"
 }
 
-is_stage_pp {
+is_mimetype_allowed(mimetype) {
-    input.stage == "pp"
+     extensions := ocis.mimetype.extensions(mimetype)
-}
+     extension := extensions[_]
-
+     is_extension_allowed(extension)
-##
-
-is_user_admin {
-    input.user.username == "admin"
-}
-
-##
-
-is_request_type_put {
-    is_stage_http
-    input.request.method == "PUT"
-}
-
-is_request_type_post {
-    is_stage_http
-    input.request.method == "POST"
-}
-
-is_request_type_mkcol {
-    is_stage_http
-    input.request.method == "MKCOL"
-}
-
-##
-
-collection_contains(collection, source) {
-     current := collection[_]
-     endswith(source, current)
 }
 

services/policies/pkg/command/server.go

@@ -21,7 +21,7 @@ import (
 	svcProtogen "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0"
 	"github.com/owncloud/ocis/v2/services/policies/pkg/config"
 	"github.com/owncloud/ocis/v2/services/policies/pkg/config/parser"
-	"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
+	"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
 	svcEvent "github.com/owncloud/ocis/v2/services/policies/pkg/service/event"
 	svcGRPC "github.com/owncloud/ocis/v2/services/policies/pkg/service/grpc"
 	"github.com/urfave/cli/v2"
@@ -51,11 +51,11 @@ func Server(cfg *config.Config) *cli.Command {
 					log.Pretty(cfg.Log.Pretty),
 					log.Color(cfg.Log.Color),
 					log.File(cfg.Log.File),
-				)
+				).SubloggerWithRequestID(ctx)
 			)
 			defer cancel()
 
-			e, err := engine.NewOPA(cfg.Engine.Timeout, cfg.Engine)
+			e, err := opa.NewOPA(cfg.Engine.Timeout, logger, cfg.Engine)
 			if err != nil {
 				return err
 			}

services/policies/pkg/engine/opa.go

@@ -1,124 +0,0 @@
-package engine
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/cs3org/reva/v2/pkg/rhttp"
-	"github.com/gabriel-vasile/mimetype"
-	"github.com/open-policy-agent/opa/ast"
-	"github.com/open-policy-agent/opa/rego"
-	"github.com/open-policy-agent/opa/types"
-	"github.com/owncloud/ocis/v2/services/policies/pkg/config"
-)
-
-// OPA wraps open policy agent makes it possible to ask if an action is granted.
-type OPA struct {
-	policies []string
-	timeout  time.Duration
-}
-
-// NewOPA returns a ready to use opa engine.
-func NewOPA(timeout time.Duration, conf config.Engine) (OPA, error) {
-	return OPA{
-			policies: conf.Policies,
-			timeout:  timeout,
-		},
-		nil
-}
-
-// Evaluate evaluates the opa policies and returns the result.
-func (o OPA) Evaluate(ctx context.Context, qs string, env Environment) (bool, error) {
-	ctx, cancel := context.WithTimeout(ctx, o.timeout)
-	defer cancel()
-
-	q, err := rego.New(
-		rego.Query(qs),
-		rego.Load(o.policies, nil),
-		GetMimetype,
-		GetResource,
-	).PrepareForEval(ctx)
-	if err != nil {
-		return false, err
-	}
-
-	result, err := q.Eval(ctx, rego.EvalInput(env))
-	if err != nil {
-		return false, err
-	}
-
-	return result.Allowed(), nil
-}
-
-var GetResource = rego.Function1(
-	&rego.Function{
-		Name:             "ocis_get_resource",
-		Decl:             types.NewFunction(types.Args(types.S), types.A),
-		Memoize:          true,
-		Nondeterministic: true,
-	},
-	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
-		var url string
-
-		if err := ast.As(a.Value, &url); err != nil {
-			return nil, err
-		}
-
-		req, err := http.NewRequest(http.MethodGet, url, nil)
-		if err != nil {
-			return nil, err
-		}
-
-		client := rhttp.GetHTTPClient(rhttp.Insecure(true))
-		res, err := client.Do(req)
-		if err != nil {
-			return nil, err
-		}
-		defer res.Body.Close()
-
-		if res.StatusCode != http.StatusOK {
-			return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
-		}
-
-		buf := new(bytes.Buffer)
-		if _, err := buf.ReadFrom(res.Body); err != nil {
-			return nil, err
-		}
-
-		v, err := ast.InterfaceToValue(buf.Bytes())
-		if err != nil {
-			return nil, err
-		}
-
-		return ast.NewTerm(v), nil
-	},
-)
-
-var GetMimetype = rego.Function1(
-	&rego.Function{
-		Name:             "ocis_get_mimetype",
-		Decl:             types.NewFunction(types.Args(types.A), types.S),
-		Memoize:          true,
-		Nondeterministic: true,
-	},
-	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
-		var body []byte
-
-		if err := ast.As(a.Value, &body); err != nil {
-			return nil, err
-		}
-
-		mimeInfo := mimetype.Detect(body).String()
-		detectedMimetype := strings.Split(mimeInfo, ";")[0]
-		v, err := ast.InterfaceToValue(detectedMimetype)
-		if err != nil {
-			return nil, err
-		}
-
-		return ast.NewTerm(v), nil
-	},
-)

services/policies/pkg/engine/opa/engine.go

@@ -0,0 +1,61 @@
+package opa
+
+import (
+	"context"
+	"time"
+
+	"github.com/open-policy-agent/opa/rego"
+	"github.com/open-policy-agent/opa/topdown/print"
+
+	"github.com/owncloud/ocis/v2/ocis-pkg/log"
+	"github.com/owncloud/ocis/v2/services/policies/pkg/config"
+	"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
+)
+
+// OPA wraps open policy agent makes it possible to ask if an action is granted.
+type OPA struct {
+	printHook print.Hook
+	policies  []string
+	timeout   time.Duration
+}
+
+// NewOPA returns a ready to use opa engine.
+func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) {
+	return OPA{
+			policies:  conf.Policies,
+			timeout:   timeout,
+			printHook: logPrinter{logger: logger},
+		},
+		nil
+}
+
+// Evaluate evaluates the opa policies and returns the result.
+func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (bool, error) {
+	ctx, cancel := context.WithTimeout(ctx, o.timeout)
+	defer cancel()
+
+	customFns := []func(r *rego.Rego){
+		RFResourceDownload,
+		RFMimetypeDetect,
+		RFMimetypeExtensions,
+	}
+
+	q, err := rego.New(
+		append([]func(r *rego.Rego){
+			rego.Query(qs),
+			rego.Load(o.policies, nil),
+			rego.EnablePrintStatements(true),
+			rego.PrintHook(o.printHook),
+		}, customFns...)...,
+	).PrepareForEval(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	result, err := q.Eval(ctx, rego.EvalInput(env))
+	if err != nil {
+		return false, err
+	}
+
+	return result.Allowed(), nil
+}

services/policies/pkg/engine/opa/opa.go

@@ -0,0 +1,16 @@
+package opa
+
+import (
+	"github.com/open-policy-agent/opa/topdown/print"
+
+	"github.com/owncloud/ocis/v2/ocis-pkg/log"
+)
+
+type logPrinter struct {
+	logger log.Logger
+}
+
+func (lp logPrinter) Print(_ print.Context, msg string) error {
+	lp.logger.Info().Msg(msg)
+	return nil
+}

services/policies/pkg/engine/opa/opa_suite_test.go

@@ -0,0 +1,13 @@
+package opa_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestOpa(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Opa Suite")
+}

services/policies/pkg/engine/opa/rf_mimetype.go

@@ -0,0 +1,59 @@
+package opa
+
+import (
+	"mime"
+	"strings"
+
+	"github.com/gabriel-vasile/mimetype"
+	"github.com/open-policy-agent/opa/ast"
+	"github.com/open-policy-agent/opa/rego"
+	"github.com/open-policy-agent/opa/types"
+)
+
+var RFMimetypeExtensions = rego.Function1(
+	&rego.Function{
+		Name:             "ocis.mimetype.extensions",
+		Decl:             types.NewFunction(types.Args(types.S), types.A),
+		Memoize:          true,
+		Nondeterministic: true,
+	},
+	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+		var mt string
+
+		if err := ast.As(a.Value, &mt); err != nil {
+			return nil, err
+		}
+
+		detectedExtensions, err := mime.ExtensionsByType(mt)
+		if err != nil {
+			return nil, err
+		}
+
+		var mimeTerms []*ast.Term
+		for _, extension := range detectedExtensions {
+			mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
+		}
+
+		return ast.ArrayTerm(mimeTerms...), nil
+	},
+)
+
+var RFMimetypeDetect = rego.Function1(
+	&rego.Function{
+		Name:             "ocis.mimetype.detect",
+		Decl:             types.NewFunction(types.Args(types.A), types.S),
+		Memoize:          true,
+		Nondeterministic: true,
+	},
+	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+		var body []byte
+
+		if err := ast.As(a.Value, &body); err != nil {
+			return nil, err
+		}
+
+		mimetype := mimetype.Detect(body).String()
+
+		return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil
+	},
+)

services/policies/pkg/engine/opa/rf_mimetype_test.go

@@ -0,0 +1,31 @@
+package opa_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/open-policy-agent/opa/rego"
+
+	"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
+)
+
+var _ = Describe("opa ocis mimetype functions", func() {
+	Describe("ocis.mimetype.detect", func() {
+		It("detects the mimetype", func() {
+			r := rego.New(rego.Query(`ocis.mimetype.detect("")`), opa.RFMimetypeDetect)
+			rs, err := r.Eval(context.Background())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
+		})
+	})
+
+	Describe("ocis.mimetype.extension_for_mimetype", func() {
+		It("provides matching extensions", func() {
+			r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions)
+			rs, err := r.Eval(context.Background())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]"))
+		})
+	})
+})

services/policies/pkg/engine/opa/rf_resource.go

@@ -0,0 +1,56 @@
+package opa
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+
+	"github.com/cs3org/reva/v2/pkg/rhttp"
+	"github.com/open-policy-agent/opa/ast"
+	"github.com/open-policy-agent/opa/rego"
+	"github.com/open-policy-agent/opa/types"
+)
+
+var RFResourceDownload = rego.Function1(
+	&rego.Function{
+		Name:             "ocis.resource.download",
+		Decl:             types.NewFunction(types.Args(types.S), types.A),
+		Memoize:          true,
+		Nondeterministic: true,
+	},
+	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+		var url string
+
+		if err := ast.As(a.Value, &url); err != nil {
+			return nil, err
+		}
+
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		client := rhttp.GetHTTPClient(rhttp.Insecure(true))
+		res, err := client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
+		}
+
+		buf := new(bytes.Buffer)
+		if _, err := buf.ReadFrom(res.Body); err != nil {
+			return nil, err
+		}
+
+		v, err := ast.InterfaceToValue(buf.Bytes())
+		if err != nil {
+			return nil, err
+		}
+
+		return ast.NewTerm(v), nil
+	},
+)

services/policies/pkg/engine/opa/rf_resource_test.go

@@ -0,0 +1,36 @@
+package opa_test
+
+import (
+	"context"
+	"encoding/base64"
+	"net/http"
+	"net/http/httptest"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/open-policy-agent/opa/rego"
+
+	"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
+)
+
+var _ = Describe("opa ocis resource functions", func() {
+	Describe("ocis.resource.download", func() {
+		It("downloads reva resources", func() {
+			ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Write(ts)
+			}))
+			defer srv.Close()
+
+			r := rego.New(rego.Query(`ocis.resource.download("`+srv.URL+`")`), opa.RFResourceDownload)
+			rs, err := r.Eval(context.Background())
+			Expect(err).ToNot(HaveOccurred())
+
+			data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(data).To(Equal(ts))
+
+		})
+	})
+})

services/policies/pkg/engine/opa_test.go

@@ -1,46 +0,0 @@
-package engine_test
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"net/http/httptest"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-	"github.com/open-policy-agent/opa/rego"
-	"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
-)
-
-var _ = Describe("Opa", func() {
-	Describe("Custom OPA function", func() {
-		Describe("GetResource", func() {
-			It("loads reva resources", func() {
-				ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
-				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-					w.Write(ts)
-				}))
-				defer srv.Close()
-
-				r := rego.New(rego.Query(`ocis_get_resource("`+srv.URL+`")`), engine.GetResource)
-				rs, err := r.Eval(context.Background())
-				Expect(err).ToNot(HaveOccurred())
-
-				data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
-				Expect(err).ToNot(HaveOccurred())
-
-				Expect(data).To(Equal(ts))
-
-			})
-		})
-
-		Describe("GetMimetype", func() {
-			It("is defined and returns a mimetype", func() {
-				r := rego.New(rego.Query(`ocis_get_mimetype("")`), engine.GetMimetype)
-				rs, err := r.Eval(context.Background())
-				Expect(err).ToNot(HaveOccurred())
-				Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
-			})
-		})
-	})
-})