add file type filter chip (#7602)

* add file type filter chip

* changed the keyword

* use reva issue-7432-replace

* changed the filter name. fix types

* rework mapBinary

---------

Co-authored-by: Roman Perekhod <rperekhod@owncloud.com>
This commit is contained in:
Roman Perekhod
2023-11-13 18:04:17 +01:00
committed by GitHub
co-authored by Roman Perekhod
parent 3653df5f96
commit 7ab553e23b
9 changed files with 399 additions and 60 deletions
@@ -0,0 +1,85 @@
Enhancement: Add search MediaType filter
Add filter MediaType filter shortcuts to search for specific document types.
For example, a search query mediatype:documents will search for files with the following mimetypes:
application/msword
MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document
MimeType:application/vnd.oasis.opendocument.text
MimeType:text/plain
MimeType:text/markdown
MimeType:application/rtf
MimeType:application/vnd.apple.pages
besides the document shorthand, it also contains following:
* file
* folder
* document
* spreadsheet
* presentation
* pdf
* image
* video
* audio
* archive
## File
## Folder
## Document:
application/msword
application/vnd.openxmlformats-officedocument.wordprocessingml.document
application/vnd.oasis.opendocument.text
text/plain
text/markdown
application/rtf
application/vnd.apple.pages
## Spreadsheet:
application/vnd.ms-excel
application/vnd.oasis.opendocument.spreadsheet
text/csv
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
application/vnd.oasis.opendocument.spreadsheet
application/vnd.apple.numbers
## Presentations:
application/vnd.ms-powerpoint
application/vnd.openxmlformats-officedocument.presentationml.presentation
application/vnd.oasis.opendocument.presentation
application/vnd.apple.keynote
## PDF
application/pdf
## Image:
image/*
## Video:
video/*
## Audio:
audio/*
## Archive (zip ...):
application/zip
application/x-tar
application/x-gzip
application/x-7z-compressed
application/x-rar-compressed
application/x-bzip2
application/x-bzip
application/x-tgz
https://github.com/owncloud/ocis/pull/7602
https://github.com/owncloud/ocis/issues/7432
+1 -1
View File
@@ -13,7 +13,7 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.7.0
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781
github.com/cs3org/reva/v2 v2.16.1-0.20231107141341-4b08a8b754e7
github.com/cs3org/reva/v2 v2.16.1-0.20231110061744-953e57a6a95c
github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25
github.com/disintegration/imaging v1.6.2
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
+2 -2
View File
@@ -1013,8 +1013,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c=
github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME=
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY=
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
github.com/cs3org/reva/v2 v2.16.1-0.20231107141341-4b08a8b754e7 h1:NmISK1T/c/1hDfOiGqPEVAOq3BTxWzIe2+bn3TtrzzU=
github.com/cs3org/reva/v2 v2.16.1-0.20231107141341-4b08a8b754e7/go.mod h1:utPCNSrWDdAwz2biLrKvzO6nDH9L7vRVGNzof13r8Kw=
github.com/cs3org/reva/v2 v2.16.1-0.20231110061744-953e57a6a95c h1:sxTOKm1ChsykAm8ITLHXNSic9RTUfpAq6Ujj94/irXQ=
github.com/cs3org/reva/v2 v2.16.1-0.20231110061744-953e57a6a95c/go.mod h1:utPCNSrWDdAwz2biLrKvzO6nDH9L7vRVGNzof13r8Kw=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+3 -2
View File
@@ -302,8 +302,9 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string
"size": map[string]interface{}{
"enabled": false,
},
"mimetype": map[string]interface{}{
"enabled": true,
"mediatype": map[string]interface{}{
"keywords": []string{"file", "folder", "document", "spreadsheet", "presentation", "pdf", "image", "video", "audio", "archive"},
"enabled": true,
},
"type": map[string]interface{}{
"enabled": true,
+124 -39
View File
@@ -6,24 +6,23 @@ import (
"github.com/blevesearch/bleve/v2"
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
"github.com/owncloud/ocis/v2/services/search/pkg/query/kql"
)
var _fields = map[string]string{
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mimetype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
}
// The following quoted string enumerates the characters which may be escaped: "+-=&|><!(){}[]^\"~*?:\\/ "
@@ -95,7 +94,18 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
v = strings.ToLower(v)
}
q := bleveQuery.NewQueryStringQuery(k + ":" + v)
var q bleveQuery.Query
var group bool
switch k {
case "MimeType":
q, group = mimeType(k, v)
if prev == nil {
isGroup = group
}
default:
q = bleveQuery.NewQueryStringQuery(k + ":" + v)
}
if prev == nil {
prev = q
} else {
@@ -210,35 +220,44 @@ func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
}
func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup bool) bleveQuery.Query {
if operator.Value == kql.BoolAND {
if left, ok := ln.(*bleveQuery.ConjunctionQuery); ok {
left.AddQuery(rn)
return left
}
if left, ok := ln.(*bleveQuery.DisjunctionQuery); ok && !leftIsGroup {
last := left.Disjuncts[len(left.Disjuncts)-1]
rn = bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
last,
rn,
})
dj := bleveQuery.NewDisjunctionQuery(left.Disjuncts[:len(left.Disjuncts)-1])
dj.AddQuery(rn)
return dj
}
return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
ln,
rn,
})
}
if operator.Value == kql.BoolOR {
if left, ok := ln.(*bleveQuery.DisjunctionQuery); ok {
right, ok := rn.(*bleveQuery.DisjunctionQuery)
switch left := ln.(type) {
case *bleveQuery.DisjunctionQuery:
if ok {
left.AddQuery(right.Disjuncts...)
} else {
left.AddQuery(rn)
}
return left
case *bleveQuery.ConjunctionQuery:
return bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{ln, rn})
default:
if ok {
left := bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{ln})
left.AddQuery(right.Disjuncts...)
return left
}
return bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{ln, rn})
}
}
if operator.Value == kql.BoolAND {
switch left := ln.(type) {
case *bleveQuery.ConjunctionQuery:
left.AddQuery(rn)
return left
case *bleveQuery.DisjunctionQuery:
if !leftIsGroup {
last := left.Disjuncts[len(left.Disjuncts)-1]
rn = bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
last,
rn,
})
dj := bleveQuery.NewDisjunctionQuery(left.Disjuncts[:len(left.Disjuncts)-1])
dj.AddQuery(rn)
return dj
}
}
return bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{
ln,
rn,
})
}
return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
ln,
@@ -264,3 +283,69 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
}
return group
}
func mimeType(k, v string) (bleveQuery.Query, bool) {
switch v {
case "file":
q := bleve.NewBooleanQuery()
q.AddMustNot(bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"))
return q, false
case "folder":
return bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"), false
case "document":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
)), true
case "spreadsheet":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.oasis.opendocument.spreadsheet",
"application/vnd.apple.numbers",
)), true
case "presentation":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.ms-powerpoint",
"application/vnd.apple.keynote",
)), true
case "pdf":
return bleveQuery.NewQueryStringQuery(k + ":application/pdf"), false
case "image":
return bleveQuery.NewQueryStringQuery(k + ":image/*"), false
case "video":
return bleveQuery.NewQueryStringQuery(k + ":video/*"), false
case "audio":
return bleveQuery.NewQueryStringQuery(k + ":audio/*"), false
case "archive":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/zip",
"application/x-tar",
"application/x-gzip",
"application/x-7z-compressed",
"application/x-rar-compressed",
"application/x-bzip2",
"application/x-bzip",
"application/x-tgz",
)), true
default:
return bleveQuery.NewQueryStringQuery(k + ":" + v), false
}
}
func newQueryStringQueryList(k string, v ...string) []bleveQuery.Query {
list := make([]bleveQuery.Query, len(v))
for i := 0; i < len(v); i++ {
list[i] = bleveQuery.NewQueryStringQuery(k + ":" + v[i])
}
return list
}
@@ -120,6 +120,51 @@ func Test_compile(t *testing.T) {
}),
wantErr: false,
},
{
name: `a OR b AND c`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Value: "c"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name:a`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name:b`),
query.NewQueryStringQuery(`Name:c`),
}),
}),
wantErr: false,
},
{
name: `(a OR b OR c) AND d`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "a"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Value: "c"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Value: "d"},
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name:a`),
query.NewQueryStringQuery(`Name:b`),
query.NewQueryStringQuery(`Name:c`),
}),
query.NewQueryStringQuery(`Name:d`),
}),
wantErr: false,
},
{
name: `(name:"moby di*" OR tag:bestseller) AND tag:book`,
args: &ast.Ast{
@@ -341,6 +386,129 @@ func Test_compile(t *testing.T) {
}),
wantErr: false,
},
{
name: `MimeType:document`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "document"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/msword`),
query.NewQueryStringQuery(`MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document`),
query.NewQueryStringQuery(`MimeType:application/vnd.oasis.opendocument.text`),
query.NewQueryStringQuery(`MimeType:text/plain`),
query.NewQueryStringQuery(`MimeType:text/markdown`),
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
wantErr: false,
},
{
name: `MimeType:document AND *tdd*`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "document"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "name", Value: "*tdd*"},
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/msword`),
query.NewQueryStringQuery(`MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document`),
query.NewQueryStringQuery(`MimeType:application/vnd.oasis.opendocument.text`),
query.NewQueryStringQuery(`MimeType:text/plain`),
query.NewQueryStringQuery(`MimeType:text/markdown`),
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
query.NewQueryStringQuery(`Name:*tdd*`),
}),
wantErr: false,
},
{
name: `MimeType:document OR MimeType:pdf AND *tdd*`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "mediatype", Value: "pdf"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "name", Value: "*tdd*"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/msword`),
query.NewQueryStringQuery(`MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document`),
query.NewQueryStringQuery(`MimeType:application/vnd.oasis.opendocument.text`),
query.NewQueryStringQuery(`MimeType:text/plain`),
query.NewQueryStringQuery(`MimeType:text/markdown`),
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/pdf`),
query.NewQueryStringQuery(`Name:*tdd*`),
}),
}),
wantErr: false,
},
{
name: `(MimeType:document OR MimeType:pdf) AND *tdd*`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "mediatype", Value: "pdf"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "name", Value: "*tdd*"},
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/msword`),
query.NewQueryStringQuery(`MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document`),
query.NewQueryStringQuery(`MimeType:application/vnd.oasis.opendocument.text`),
query.NewQueryStringQuery(`MimeType:text/plain`),
query.NewQueryStringQuery(`MimeType:text/markdown`),
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewQueryStringQuery(`MimeType:application/pdf`),
}),
query.NewQueryStringQuery(`Name:*tdd*`),
}),
wantErr: false,
},
{
name: `(MimeType:pdf OR MimeType:document) AND *tdd*`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "pdf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "mediatype", Value: "document"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "name", Value: "*tdd*"},
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/pdf`),
query.NewQueryStringQuery(`MimeType:application/msword`),
query.NewQueryStringQuery(`MimeType:application/vnd.openxmlformats-officedocument.wordprocessingml.document`),
query.NewQueryStringQuery(`MimeType:application/vnd.oasis.opendocument.text`),
query.NewQueryStringQuery(`MimeType:text/plain`),
query.NewQueryStringQuery(`MimeType:text/markdown`),
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
query.NewQueryStringQuery(`Name:*tdd*`),
}),
wantErr: false,
},
{
name: `John Smith`,
args: &ast.Ast{
@@ -69,26 +69,26 @@ type CapabilitiesSearch struct {
// CapabilitiesSearchProperties holds the search property capabilities
type CapabilitiesSearchProperties struct {
Name *CapabilitiesSearchProperty `json:"name" xml:"name" mapstructure:"name"`
Mtime *CapabilitiesSearchPropertyMtime `json:"mtime" xml:"mtime" mapstructure:"mtime"`
Size *CapabilitiesSearchProperty `json:"size" xml:"size" mapstructure:"size"`
Mimetype *CapabilitiesSearchProperty `json:"mimetype" xml:"mimetype" mapstructure:"mimetype"`
Type *CapabilitiesSearchProperty `json:"type" xml:"type" mapstructure:"type"`
Tag *CapabilitiesSearchProperty `json:"tag" xml:"tag" mapstructure:"tag"`
Tags *CapabilitiesSearchProperty `json:"tags" xml:"tags" mapstructure:"tags"`
Content *CapabilitiesSearchProperty `json:"content" xml:"content" mapstructure:"content"`
Scope *CapabilitiesSearchProperty `json:"scope" xml:"scope" mapstructure:"scope"`
Name *CapabilitiesSearchProperty `json:"name" xml:"name" mapstructure:"name"`
Mtime *CapabilitiesSearchPropertyExtended `json:"mtime" xml:"mtime" mapstructure:"mtime"`
Size *CapabilitiesSearchProperty `json:"size" xml:"size" mapstructure:"size"`
MediaType *CapabilitiesSearchPropertyExtended `json:"mediatype" xml:"mediatype" mapstructure:"mediatype"`
Type *CapabilitiesSearchProperty `json:"type" xml:"type" mapstructure:"type"`
Tag *CapabilitiesSearchProperty `json:"tag" xml:"tag" mapstructure:"tag"`
Tags *CapabilitiesSearchProperty `json:"tags" xml:"tags" mapstructure:"tags"`
Content *CapabilitiesSearchProperty `json:"content" xml:"content" mapstructure:"content"`
Scope *CapabilitiesSearchProperty `json:"scope" xml:"scope" mapstructure:"scope"`
}
// CapabilitiesSearchProperty represents the default search property
type CapabilitiesSearchProperty struct {
Enabled bool `json:"enabled" xml:"enabled" mapstructure:"enabled"`
Enabled ocsBool `json:"enabled" xml:"enabled" mapstructure:"enabled"`
}
// CapabilitiesSearchPropertyMtime represents the mtime search property
type CapabilitiesSearchPropertyMtime struct {
CapabilitiesSearchProperty
Keywords []string `json:"keywords" xml:"keywords" mapstructure:"keywords"`
// CapabilitiesSearchPropertyExtended represents the extended search property
type CapabilitiesSearchPropertyExtended struct {
CapabilitiesSearchProperty `mapstructure:",squash"`
Keywords []string `json:"keywords,omitempty" xml:"keywords,omitempty" mapstructure:"keywords"`
}
// Spaces lets a service configure its advertised options related to Storage Spaces.
@@ -368,7 +368,7 @@ func (upload *Upload) Finalize() (err error) {
err = upload.tp.WriteBlob(n, upload.binPath)
subspan.End()
if err != nil {
return errors.Wrap(err, "failed to upload file to blostore")
return errors.Wrap(err, "failed to upload file to blobstore")
}
return nil
+1 -1
View File
@@ -357,7 +357,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
# github.com/cs3org/reva/v2 v2.16.1-0.20231107141341-4b08a8b754e7
# github.com/cs3org/reva/v2 v2.16.1-0.20231110061744-953e57a6a95c
## explicit; go 1.20
github.com/cs3org/reva/v2/cmd/revad/internal/grace
github.com/cs3org/reva/v2/cmd/revad/runtime