sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhancement: add mimetype to file extension rego function (#6133)

Browse Source 
* enhancement: add mimetype to file extension rego function

add rego function to detect the resource extension by mimetype, at the same time this pr introduces a custom ocis namespace for the rego functions.

* enhancement: add custom logPrinter to opa policies service

* fix: imports and test

TypeByExtension which is used to resolve extension by mimetype relies on MIME-info database which differs at my local env (macos <-> drone). This is fixed by using one of the builtinTypes for testing

---------

Signed-off-by: Christian Richter <crichter@owncloud.com>
Co-authored-by: Christian Richter <crichter@owncloud.com>
This commit is contained in:
Florian Schade
2023-07-10 16:28:23 +02:00
committed by GitHub
co-authored by Christian Richter
parent ecfe2d9a7b
commit c09f82405f
14 changed files with 303 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/policies/pkg/command/server.go
+3 -3
View File
@@ -21,7 +21,7 @@ import (
svcProtogen "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/policies/v0"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
"github.com/owncloud/ocis/v2/services/policies/pkg/config/parser"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
svcEvent "github.com/owncloud/ocis/v2/services/policies/pkg/service/event"
svcGRPC "github.com/owncloud/ocis/v2/services/policies/pkg/service/grpc"
"github.com/urfave/cli/v2"
@@ -51,11 +51,11 @@ func Server(cfg *config.Config) *cli.Command {
log.Pretty(cfg.Log.Pretty),
log.Color(cfg.Log.Color),
log.File(cfg.Log.File),
)
).SubloggerWithRequestID(ctx)
)
defer cancel()
e, err := engine.NewOPA(cfg.Engine.Timeout, cfg.Engine)
e, err := opa.NewOPA(cfg.Engine.Timeout, logger, cfg.Engine)
if err != nil {
return err
}
services/policies/pkg/engine/opa.go
-124
View File
@@ -1,124 +0,0 @@
package engine
import (
"bytes"
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/gabriel-vasile/mimetype"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
)
// OPA wraps open policy agent makes it possible to ask if an action is granted.
type OPA struct {
policies []string
timeout time.Duration
}
// NewOPA returns a ready to use opa engine.
func NewOPA(timeout time.Duration, conf config.Engine) (OPA, error) {
return OPA{
policies: conf.Policies,
timeout: timeout,
},
nil
}
// Evaluate evaluates the opa policies and returns the result.
func (o OPA) Evaluate(ctx context.Context, qs string, env Environment) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
q, err := rego.New(
rego.Query(qs),
rego.Load(o.policies, nil),
GetMimetype,
GetResource,
).PrepareForEval(ctx)
if err != nil {
return false, err
}
result, err := q.Eval(ctx, rego.EvalInput(env))
if err != nil {
return false, err
}
return result.Allowed(), nil
}
var GetResource = rego.Function1(
&rego.Function{
Name: "ocis_get_resource",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var url string
if err := ast.As(a.Value, &url); err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
client := rhttp.GetHTTPClient(rhttp.Insecure(true))
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(res.Body); err != nil {
return nil, err
}
v, err := ast.InterfaceToValue(buf.Bytes())
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)
var GetMimetype = rego.Function1(
&rego.Function{
Name: "ocis_get_mimetype",
Decl: types.NewFunction(types.Args(types.A), types.S),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var body []byte
if err := ast.As(a.Value, &body); err != nil {
return nil, err
}
mimeInfo := mimetype.Detect(body).String()
detectedMimetype := strings.Split(mimeInfo, ";")[0]
v, err := ast.InterfaceToValue(detectedMimetype)
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)
services/policies/pkg/engine/opa/engine.go
+61
View File
@@ -0,0 +1,61 @@
package opa
import (
"context"
"time"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/topdown/print"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/policies/pkg/config"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
)
// OPA wraps open policy agent makes it possible to ask if an action is granted.
type OPA struct {
printHook print.Hook
policies []string
timeout time.Duration
}
// NewOPA returns a ready to use opa engine.
func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) {
return OPA{
policies: conf.Policies,
timeout: timeout,
printHook: logPrinter{logger: logger},
},
nil
}
// Evaluate evaluates the opa policies and returns the result.
func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
customFns := []func(r *rego.Rego){
RFResourceDownload,
RFMimetypeDetect,
RFMimetypeExtensions,
}
q, err := rego.New(
append([]func(r *rego.Rego){
rego.Query(qs),
rego.Load(o.policies, nil),
rego.EnablePrintStatements(true),
rego.PrintHook(o.printHook),
}, customFns...)...,
).PrepareForEval(ctx)
if err != nil {
return false, err
}
result, err := q.Eval(ctx, rego.EvalInput(env))
if err != nil {
return false, err
}
return result.Allowed(), nil
}
services/policies/pkg/engine/opa/opa.go
+16
View File
@@ -0,0 +1,16 @@
package opa
import (
"github.com/open-policy-agent/opa/topdown/print"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
)
type logPrinter struct {
logger log.Logger
}
func (lp logPrinter) Print(_ print.Context, msg string) error {
lp.logger.Info().Msg(msg)
return nil
}
services/policies/pkg/engine/opa/opa_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package opa_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestOpa(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Opa Suite")
}
services/policies/pkg/engine/opa/rf_mimetype.go
+59
View File
@@ -0,0 +1,59 @@
package opa
import (
"mime"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
)
var RFMimetypeExtensions = rego.Function1(
&rego.Function{
Name: "ocis.mimetype.extensions",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var mt string
if err := ast.As(a.Value, &mt); err != nil {
return nil, err
}
detectedExtensions, err := mime.ExtensionsByType(mt)
if err != nil {
return nil, err
}
var mimeTerms []*ast.Term
for _, extension := range detectedExtensions {
mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
}
return ast.ArrayTerm(mimeTerms...), nil
},
)
var RFMimetypeDetect = rego.Function1(
&rego.Function{
Name: "ocis.mimetype.detect",
Decl: types.NewFunction(types.Args(types.A), types.S),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var body []byte
if err := ast.As(a.Value, &body); err != nil {
return nil, err
}
mimetype := mimetype.Detect(body).String()
return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil
},
)
services/policies/pkg/engine/opa/rf_mimetype_test.go
+31
View File
@@ -0,0 +1,31 @@
package opa_test
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
)
var _ = Describe("opa ocis mimetype functions", func() {
Describe("ocis.mimetype.detect", func() {
It("detects the mimetype", func() {
r := rego.New(rego.Query(`ocis.mimetype.detect("")`), opa.RFMimetypeDetect)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
})
})
Describe("ocis.mimetype.extension_for_mimetype", func() {
It("provides matching extensions", func() {
r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]"))
})
})
})
services/policies/pkg/engine/opa/rf_resource.go
+56
View File
@@ -0,0 +1,56 @@
package opa
import (
"bytes"
"fmt"
"net/http"
"github.com/cs3org/reva/v2/pkg/rhttp"
"github.com/open-policy-agent/opa/ast"
"github.com/open-policy-agent/opa/rego"
"github.com/open-policy-agent/opa/types"
)
var RFResourceDownload = rego.Function1(
&rego.Function{
Name: "ocis.resource.download",
Decl: types.NewFunction(types.Args(types.S), types.A),
Memoize: true,
Nondeterministic: true,
},
func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
var url string
if err := ast.As(a.Value, &url); err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
client := rhttp.GetHTTPClient(rhttp.Insecure(true))
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code from Download %v", res.StatusCode)
}
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(res.Body); err != nil {
return nil, err
}
v, err := ast.InterfaceToValue(buf.Bytes())
if err != nil {
return nil, err
}
return ast.NewTerm(v), nil
},
)
services/policies/pkg/engine/opa/rf_resource_test.go
+36
View File
@@ -0,0 +1,36 @@
package opa_test
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine/opa"
)
var _ = Describe("opa ocis resource functions", func() {
Describe("ocis.resource.download", func() {
It("downloads reva resources", func() {
ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(ts)
}))
defer srv.Close()
r := rego.New(rego.Query(`ocis.resource.download("`+srv.URL+`")`), opa.RFResourceDownload)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(ts))
})
})
})
services/policies/pkg/engine/opa_test.go
-46
View File
@@ -1,46 +0,0 @@
package engine_test
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/open-policy-agent/opa/rego"
"github.com/owncloud/ocis/v2/services/policies/pkg/engine"
)
var _ = Describe("Opa", func() {
Describe("Custom OPA function", func() {
Describe("GetResource", func() {
It("loads reva resources", func() {
ts := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write(ts)
}))
defer srv.Close()
r := rego.New(rego.Query(`ocis_get_resource("`+srv.URL+`")`), engine.GetResource)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
data, err := base64.StdEncoding.DecodeString(rs[0].Expressions[0].String())
Expect(err).ToNot(HaveOccurred())
Expect(data).To(Equal(ts))
})
})
Describe("GetMimetype", func() {
It("is defined and returns a mimetype", func() {
r := rego.New(rego.Query(`ocis_get_mimetype("")`), engine.GetMimetype)
rs, err := r.Eval(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
})
})
})
})