enhancement: Keyword Query Language (KQL) search syntax support (#7043)
* feat(search): introduce search query package With the increasing complexity of how we organize our resources, the search must also be able to find them using entity properties. The query package provides the necessary functionality to do this. This makes it possible to search for resources via KQL, the microsoft spec is largely covered and can be used for this. In the current state, the legacy query language is still used, in a future update this will be deprecated and KQL will become the standard
This commit is contained in:
Florian Schade
@@ -0,0 +1,63 @@
|
||||
// Package ast provides available ast nodes.
|
||||
package ast
|
||||
|
||||
// Node represents abstract syntax tree node
|
||||
type Node interface {
|
||||
Location() *Location
|
||||
}
|
||||
|
||||
// Position represents a specific location in the source
|
||||
type Position struct {
|
||||
Line int
|
||||
Column int
|
||||
}
|
||||
|
||||
// Location represents the location of a node in the AST
|
||||
type Location struct {
|
||||
Start Position `json:"start"`
|
||||
End Position `json:"end"`
|
||||
Source *string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// Base contains shared node attributes
|
||||
// each node should inherit from this
|
||||
type Base struct {
|
||||
Loc *Location
|
||||
}
|
||||
|
||||
// Location is the source location of the Node
|
||||
func (b *Base) Location() *Location { return b.Loc }
|
||||
|
||||
// Ast represents the query - node structure as abstract syntax tree
|
||||
type Ast struct {
|
||||
*Base
|
||||
Nodes []Node `json:"body"`
|
||||
}
|
||||
|
||||
// StringNode represents a string value
|
||||
type StringNode struct {
|
||||
*Base
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
// BooleanNode represents a bool value
|
||||
type BooleanNode struct {
|
||||
*Base
|
||||
Key string
|
||||
Value bool
|
||||
}
|
||||
|
||||
// OperatorNode represents an operator value like
|
||||
// AND, OR, NOT, =, <= ... and so on
|
||||
type OperatorNode struct {
|
||||
*Base
|
||||
Value string
|
||||
}
|
||||
|
||||
// GroupNode represents a collection of many grouped nodes
|
||||
type GroupNode struct {
|
||||
*Base
|
||||
Key string
|
||||
Nodes []Node
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Package test provides shared test primitives for ast testing.
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
// DiffAst returns a human-readable report of the differences between two values
|
||||
// by default it ignores every ast node Base field.
|
||||
func DiffAst(x, y interface{}, opts ...cmp.Option) string {
|
||||
return cmp.Diff(
|
||||
x,
|
||||
y,
|
||||
append(
|
||||
opts,
|
||||
cmpopts.IgnoreFields(ast.Ast{}, "Base"),
|
||||
cmpopts.IgnoreFields(ast.StringNode{}, "Base"),
|
||||
cmpopts.IgnoreFields(ast.OperatorNode{}, "Base"),
|
||||
cmpopts.IgnoreFields(ast.GroupNode{}, "Base"),
|
||||
cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"),
|
||||
)...,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Package bleve provides the ability to work with bleve queries.
|
||||
package bleve
|
||||
|
||||
import (
|
||||
bQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query"
|
||||
)
|
||||
|
||||
// Creator is combines a Builder and a Compiler which is used to Create the query.
|
||||
type Creator[T any] struct {
|
||||
builder query.Builder
|
||||
compiler query.Compiler[T]
|
||||
}
|
||||
|
||||
// Create implements the Creator interface
|
||||
func (c Creator[T]) Create(qs string) (T, error) {
|
||||
var t T
|
||||
builderAst, err := c.builder.Build(qs)
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
|
||||
t, err = c.compiler.Compile(builderAst)
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// LegacyCreator exposes an ocis legacy bleve query creator.
|
||||
var LegacyCreator = Creator[bQuery.Query]{LegacyBuilder{}, LegacyCompiler{}}
|
||||
@@ -0,0 +1,170 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/kql"
|
||||
)
|
||||
|
||||
var _fields = map[string]string{
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mimetype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
}
|
||||
|
||||
// Compiler represents a KQL query search string to the bleve query formatter.
|
||||
type Compiler struct{}
|
||||
|
||||
// Compile implements the query formatter which converts the KQL query search string to the bleve query.
|
||||
func (c *Compiler) Compile(givenAst *ast.Ast) (bleveQuery.Query, error) {
|
||||
q, err := compile(givenAst)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func compile(a *ast.Ast) (bleveQuery.Query, error) {
|
||||
q, _ := walk(0, a.Nodes)
|
||||
switch q.(type) {
|
||||
case *bleveQuery.ConjunctionQuery, *bleveQuery.DisjunctionQuery:
|
||||
return q, nil
|
||||
}
|
||||
return bleve.NewConjunctionQuery(q), nil
|
||||
}
|
||||
|
||||
func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int) {
|
||||
var prev, next bleveQuery.Query
|
||||
var operator *ast.OperatorNode
|
||||
var isGroup bool
|
||||
for i := offset; i < len(nodes); i++ {
|
||||
switch n := nodes[i].(type) {
|
||||
case *ast.StringNode:
|
||||
q := bleveQuery.NewQueryStringQuery(getField(n.Key) + ":" + n.Value)
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
next = q
|
||||
}
|
||||
case *ast.BooleanNode:
|
||||
q := bleveQuery.NewQueryStringQuery(getField(n.Key) + fmt.Sprintf(":%v", n.Value))
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
next = q
|
||||
}
|
||||
case *ast.GroupNode:
|
||||
if n.Key != "" {
|
||||
n = normalizeGroupingProperty(n)
|
||||
}
|
||||
q, _ := walk(0, n.Nodes)
|
||||
if prev == nil {
|
||||
prev = q
|
||||
isGroup = true
|
||||
} else {
|
||||
next = q
|
||||
}
|
||||
case *ast.OperatorNode:
|
||||
if n.Value == kql.BoolAND || n.Value == kql.BoolOR {
|
||||
operator = n
|
||||
} else if n.Value == kql.BoolNOT {
|
||||
next, offset = nextNode(i+1, nodes)
|
||||
q := bleve.NewBooleanQuery()
|
||||
q.AddMustNot(next)
|
||||
next = q
|
||||
}
|
||||
}
|
||||
if prev != nil && next != nil && operator != nil {
|
||||
prev = mapBinary(operator, prev, next, isGroup)
|
||||
isGroup = false
|
||||
operator = nil
|
||||
next = nil
|
||||
}
|
||||
if i < offset {
|
||||
i = offset
|
||||
}
|
||||
}
|
||||
return prev, offset
|
||||
}
|
||||
|
||||
func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int) {
|
||||
if n, ok := nodes[offset].(*ast.GroupNode); ok {
|
||||
gq, _ := walk(0, n.Nodes)
|
||||
return gq, offset + 1
|
||||
}
|
||||
if n, ok := nodes[offset].(*ast.OperatorNode); ok {
|
||||
if n.Value == kql.BoolNOT {
|
||||
return walk(offset, nodes)
|
||||
}
|
||||
}
|
||||
one := nodes[:offset+1]
|
||||
return walk(offset, one)
|
||||
}
|
||||
|
||||
func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup bool) bleveQuery.Query {
|
||||
if operator.Value == kql.BoolAND {
|
||||
if left, ok := ln.(*bleveQuery.ConjunctionQuery); ok {
|
||||
left.AddQuery(rn)
|
||||
return left
|
||||
}
|
||||
if left, ok := ln.(*bleveQuery.DisjunctionQuery); ok && !leftIsGroup {
|
||||
last := left.Disjuncts[len(left.Disjuncts)-1]
|
||||
rn = bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
|
||||
last,
|
||||
rn,
|
||||
})
|
||||
dj := bleveQuery.NewDisjunctionQuery(left.Disjuncts[:len(left.Disjuncts)-1])
|
||||
dj.AddQuery(rn)
|
||||
return dj
|
||||
}
|
||||
return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
|
||||
ln,
|
||||
rn,
|
||||
})
|
||||
}
|
||||
if operator.Value == kql.BoolOR {
|
||||
if left, ok := ln.(*bleveQuery.DisjunctionQuery); ok {
|
||||
left.AddQuery(rn)
|
||||
return left
|
||||
}
|
||||
return bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{
|
||||
ln,
|
||||
rn,
|
||||
})
|
||||
}
|
||||
return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{
|
||||
ln,
|
||||
rn,
|
||||
})
|
||||
}
|
||||
|
||||
func getField(name string) string {
|
||||
if name == "" {
|
||||
return "Name"
|
||||
}
|
||||
if _, ok := _fields[strings.ToLower(name)]; ok {
|
||||
return _fields[strings.ToLower(name)]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
|
||||
for _, n := range group.Nodes {
|
||||
if onode, ok := n.(*ast.StringNode); ok {
|
||||
onode.Key = group.Key
|
||||
}
|
||||
}
|
||||
return group
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
func Test_compile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args *ast.Ast
|
||||
want query.Query
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: `federated`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "federated"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:federated`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `"John Smith"`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:John Smith`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `"John Smith" Jane`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "name", Value: "Jane"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:John Smith`),
|
||||
query.NewQueryStringQuery(`Name:Jane`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `tag:bestseller tag:book`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "book"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `name:"moby di*" OR tag:bestseller AND tag:book`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "moby di*"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "book"},
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:moby di*`),
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
}),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `a AND b OR c`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Value: "c"},
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:a`),
|
||||
query.NewQueryStringQuery(`Name:b`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Name:c`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `(name:"moby di*" OR tag:bestseller) AND tag:book`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "moby di*"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "book"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:moby di*`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `(name:"moby di*" OR tag:bestseller) AND tag:book AND NOT tag:read`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "moby di*"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "book"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "tag", Value: "read"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name:moby di*`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:read`)}),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `author:("John Smith" Jane)`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`author:John Smith`),
|
||||
query.NewQueryStringQuery(`author:Jane`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `author:("John Smith" Jane) AND tag:bestseller`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`author:John Smith`),
|
||||
query.NewQueryStringQuery(`author:Jane`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := compile(tt.args)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("compile() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
assert.Equal(tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
bQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
// LegacyBuilder implements the legacy Builder interface.
|
||||
type LegacyBuilder struct{}
|
||||
|
||||
// Build translates the ast to a valid bleve query.
|
||||
func (b LegacyBuilder) Build(qs string) (*ast.Ast, error) {
|
||||
return &ast.Ast{
|
||||
Base: &ast.Base{
|
||||
Loc: &ast.Location{
|
||||
Start: ast.Position{
|
||||
Line: 0,
|
||||
Column: 0,
|
||||
},
|
||||
End: ast.Position{
|
||||
Line: 0,
|
||||
Column: len(qs),
|
||||
},
|
||||
Source: &qs,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LegacyCompiler represents a default bleve query formatter.
|
||||
type LegacyCompiler struct{}
|
||||
|
||||
// Compile implements the default bleve query formatter which converts the bleve likes query search string to the bleve query.
|
||||
func (c LegacyCompiler) Compile(givenAst *ast.Ast) (bQuery.Query, error) {
|
||||
return &bQuery.QueryStringQuery{
|
||||
Query: c.formatQuery(*givenAst.Base.Loc.Source),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c LegacyCompiler) formatQuery(q string) string {
|
||||
cq := q
|
||||
fields := []string{"RootID", "Path", "ID", "Name", "Size", "Mtime", "MimeType", "Type"}
|
||||
for _, field := range fields {
|
||||
cq = strings.ReplaceAll(cq, strings.ToLower(field)+":", field+":")
|
||||
}
|
||||
|
||||
fieldRe := regexp.MustCompile(`\w+:[^ ]+`)
|
||||
if fieldRe.MatchString(cq) {
|
||||
nameTagesRe := regexp.MustCompile(`\+?(Name|Tags)`) // detect "Name", "+Name, "Tags" and "+Tags"
|
||||
parts := strings.Split(cq, " ")
|
||||
|
||||
cq = ""
|
||||
for _, part := range parts {
|
||||
fieldParts := strings.SplitN(part, ":", 2)
|
||||
if len(fieldParts) > 1 {
|
||||
key := fieldParts[0]
|
||||
value := fieldParts[1]
|
||||
if nameTagesRe.MatchString(key) {
|
||||
value = strings.ToLower(value) // do a lowercase query on the lowercased fields
|
||||
}
|
||||
cq += key + ":" + value + " "
|
||||
} else {
|
||||
cq += part + " "
|
||||
}
|
||||
}
|
||||
return cq // Sophisticated field based search
|
||||
}
|
||||
|
||||
// this is a basic filename search
|
||||
cq = strings.ReplaceAll(cq, ":", `\:`)
|
||||
return "Name:*" + strings.ReplaceAll(strings.ToLower(cq), " ", `\ `) + "*"
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package kql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
func toIfaceSlice(in interface{}) []interface{} {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
return in.([]interface{})
|
||||
}
|
||||
|
||||
func toNode(in interface{}) (ast.Node, error) {
|
||||
out, ok := in.(ast.Node)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("can't convert '%T' to ast.Node", in)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func toNodes(in interface{}) ([]ast.Node, error) {
|
||||
|
||||
switch v := in.(type) {
|
||||
case []interface{}:
|
||||
var nodes []ast.Node
|
||||
|
||||
for _, el := range toIfaceSlice(v) {
|
||||
node, err := toNode(el)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
case []ast.Node:
|
||||
return v, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("can't convert '%T' to []ast.Node", in)
|
||||
}
|
||||
}
|
||||
|
||||
func toString(in interface{}) (string, error) {
|
||||
switch v := in.(type) {
|
||||
case []byte:
|
||||
return string(v), nil
|
||||
case []interface{}:
|
||||
var str string
|
||||
|
||||
for _, i := range v {
|
||||
j := i.([]uint8)
|
||||
str += string(j[0])
|
||||
}
|
||||
|
||||
return str, nil
|
||||
case string:
|
||||
return v, nil
|
||||
default:
|
||||
return "", fmt.Errorf("can't convert '%T' to string", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package kql
|
||||
|
||||
// The operator node value definition
|
||||
const (
|
||||
BoolAND = "AND"
|
||||
BoolOR = "OR"
|
||||
BoolNOT = "NOT"
|
||||
)
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
package kql
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// ast
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
AST <-
|
||||
_ nodes:Nodes _ {
|
||||
return buildAST(nodes, c.text, c.pos)
|
||||
}
|
||||
|
||||
Nodes <-
|
||||
n:(
|
||||
_
|
||||
(
|
||||
GroupNode /
|
||||
PropertyRestrictionNodes /
|
||||
BooleanOperatorNode /
|
||||
FreeTextKeywordNodes
|
||||
)
|
||||
_
|
||||
)+ {
|
||||
return buildNodes(n)
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// nesting
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
GroupNode <-
|
||||
k:(Char+)? (ColonOperator / EqualOperator)? "(" v:Nodes ")" {
|
||||
return buildGroupNode(k, v, c.text, c.pos)
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// property restrictions
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
PropertyRestrictionNodes <-
|
||||
YesNoPropertyRestrictionNode /
|
||||
TextPropertyRestrictionNode
|
||||
|
||||
YesNoPropertyRestrictionNode <-
|
||||
k:Char+ (ColonOperator / EqualOperator) v:("true" / "false"){
|
||||
return buildBooleanNode(k, v, c.text, c.pos)
|
||||
}
|
||||
|
||||
TextPropertyRestrictionNode <-
|
||||
k:Char+ (ColonOperator / EqualOperator) v:(String / [^ ()]+){
|
||||
return buildStringNode(k, v, c.text, c.pos)
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// free text-keywords
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
FreeTextKeywordNodes <-
|
||||
PhraseNode /
|
||||
WordNode
|
||||
|
||||
PhraseNode <-
|
||||
ColonOperator? _ v:String _ ColonOperator? {
|
||||
return buildStringNode("", v, c.text, c.pos)
|
||||
}
|
||||
|
||||
WordNode <-
|
||||
ColonOperator? _ v:[^ :()]+ _ ColonOperator? {
|
||||
return buildStringNode("", v, c.text, c.pos)
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// operators
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
BooleanOperatorNode <-
|
||||
("AND" / "OR" / "NOT") {
|
||||
return buildOperatorNode(c.text, c.pos)
|
||||
}
|
||||
|
||||
ColonOperator <-
|
||||
":" {
|
||||
return c.text, nil
|
||||
}
|
||||
|
||||
EqualOperator <-
|
||||
"=" {
|
||||
return c.text, nil
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// misc
|
||||
////////////////////////////////////////////////////////
|
||||
|
||||
Char <-
|
||||
[A-Za-z] {
|
||||
return c.text, nil
|
||||
}
|
||||
|
||||
String <-
|
||||
'"' v:[^"]* '"' {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
_ <-
|
||||
[ \t]*
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,283 @@
|
||||
package kql_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast/test"
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/kql"
|
||||
)
|
||||
|
||||
var FullDictionary = []string{
|
||||
`federated search`,
|
||||
`federat* search`,
|
||||
`search fed*`,
|
||||
`author:"John Smith"`,
|
||||
`filetype:docx`,
|
||||
`filename:budget.xlsx`,
|
||||
`author: "John Smith"`,
|
||||
`author :"John Smith"`,
|
||||
`author : "John Smith"`,
|
||||
`author "John Smith"`,
|
||||
`author "John Smith"`,
|
||||
`author:Shakespear`,
|
||||
`author:Paul`,
|
||||
`author:Shakesp*`,
|
||||
`title:"Advanced Search"`,
|
||||
`title:"Advanced Sear*"`,
|
||||
`title:"Advan* Search"`,
|
||||
`title:"*anced Search"`,
|
||||
`author:"John Smith" OR author:"Jane Smith"`,
|
||||
`author:"John Smith" AND filetype:docx`,
|
||||
`author:("John Smith" "Jane Smith")`,
|
||||
`author:("John Smith" OR "Jane Smith")`,
|
||||
`(DepartmentId:* OR RelatedHubSites:*) AND contentclass:sts_site NOT IsHubSite:false`,
|
||||
`author:"John Smith" (filetype:docx title:"Advanced Search")`,
|
||||
}
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
givenQuery []string
|
||||
expectedAst *ast.Ast
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "FullDictionary",
|
||||
givenQuery: FullDictionary,
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "federated"},
|
||||
&ast.StringNode{Value: "search"},
|
||||
&ast.StringNode{Value: "federat*"},
|
||||
&ast.StringNode{Value: "search"},
|
||||
&ast.StringNode{Value: "search"},
|
||||
&ast.StringNode{Value: "fed*"},
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "filetype", Value: "docx"},
|
||||
&ast.StringNode{Key: "filename", Value: "budget.xlsx"},
|
||||
&ast.StringNode{Value: "author"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "author"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "author"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "author"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "author"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Key: "author", Value: "Shakespear"},
|
||||
&ast.StringNode{Key: "author", Value: "Paul"},
|
||||
&ast.StringNode{Key: "author", Value: "Shakesp*"},
|
||||
&ast.StringNode{Key: "title", Value: "Advanced Search"},
|
||||
&ast.StringNode{Key: "title", Value: "Advanced Sear*"},
|
||||
&ast.StringNode{Key: "title", Value: "Advan* Search"},
|
||||
&ast.StringNode{Key: "title", Value: "*anced Search"},
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "filetype", Value: "docx"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Value: "Jane Smith"},
|
||||
},
|
||||
},
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Value: "Jane Smith"},
|
||||
},
|
||||
},
|
||||
&ast.GroupNode{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "DepartmentId", Value: "*"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "RelatedHubSites", Value: "*"},
|
||||
},
|
||||
},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "contentclass", Value: "sts_site"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: "IsHubSite", Value: false},
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.GroupNode{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "filetype", Value: "docx"},
|
||||
&ast.StringNode{Key: "title", Value: "Advanced Search"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Group",
|
||||
givenQuery: []string{
|
||||
`(name:"moby di*" OR tag:bestseller) AND tag:book NOT tag:read`,
|
||||
`author:("John Smith" Jane)`,
|
||||
`author:("John Smith" OR Jane)`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "moby di*"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "tag", Value: "bestseller"},
|
||||
},
|
||||
},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "tag", Value: "book"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "tag", Value: "read"},
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "KeyGroup or key conjunction",
|
||||
givenQuery: []string{
|
||||
`author:("John Smith" Jane) author:"Jack" AND author:"Oggy"`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
&ast.StringNode{Key: "author", Value: "Jack"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "author", Value: "Oggy"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "KeyGroup",
|
||||
givenQuery: []string{
|
||||
`author:("John Smith" OR Jane)`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.GroupNode{
|
||||
Key: "author",
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "not and not",
|
||||
givenQuery: []string{
|
||||
`NOT "John Smith" NOT Jane`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Value: "Jane"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "not or not and not",
|
||||
givenQuery: []string{
|
||||
`NOT author:"John Smith" NOT author:"Jane Smith" NOT tag:sifi`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "tag", Value: "sifi"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "misc",
|
||||
givenQuery: []string{
|
||||
`scope:"<uuid>/new folder/subfolder" file`,
|
||||
},
|
||||
expectedAst: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{
|
||||
Key: "scope",
|
||||
Value: "<uuid>/new folder/subfolder",
|
||||
},
|
||||
&ast.StringNode{
|
||||
Value: "file",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
q := strings.Join(tt.givenQuery, " ")
|
||||
|
||||
parsedAST, err := kql.Parse("", []byte(q))
|
||||
|
||||
if tt.expectedError != nil {
|
||||
assert.Equal(err, tt.expectedError)
|
||||
assert.Nil(parsedAST)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
normalizedNodes, err := kql.NormalizeNodes(tt.expectedAst.Nodes)
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizeNodes() error = %v", err)
|
||||
}
|
||||
tt.expectedAst.Nodes = normalizedNodes
|
||||
|
||||
if diff := test.DiffAst(tt.expectedAst, parsedAST); diff != "" {
|
||||
t.Fatalf("AST mismatch \nquery: '%s' \n(-want +got): %s", q, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkParse(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for n := 0; n < b.N; n++ {
|
||||
if _, err := kql.Parse("", []byte(strings.Join(FullDictionary, " "))); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package kql
|
||||
|
||||
// StartsWithBinaryOperatorError records an error and the operation that caused it.
|
||||
type StartsWithBinaryOperatorError struct {
|
||||
Op string
|
||||
}
|
||||
|
||||
func (e *StartsWithBinaryOperatorError) Error() string {
|
||||
return "the expression can't begin from a binary operator: '" + e.Op + "'"
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package kql
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
func base(text []byte, pos position) (*ast.Base, error) {
|
||||
source, err := toString(text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.Base{
|
||||
Loc: &ast.Location{
|
||||
Start: ast.Position{
|
||||
Line: pos.line,
|
||||
Column: pos.col,
|
||||
},
|
||||
End: ast.Position{
|
||||
Line: pos.line,
|
||||
Column: pos.col + len(text),
|
||||
},
|
||||
Source: &source,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildAST(n interface{}, text []byte, pos position) (*ast.Ast, error) {
|
||||
b, err := base(text, pos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes, err := toNodes(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
normalizedNodes, err := NormalizeNodes(nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.Ast{
|
||||
Base: b,
|
||||
Nodes: normalizedNodes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildNodes(e interface{}) ([]ast.Node, error) {
|
||||
maybeNodesGroups := toIfaceSlice(e)
|
||||
|
||||
nodes := make([]ast.Node, len(maybeNodesGroups))
|
||||
for i, maybeNodesGroup := range maybeNodesGroups {
|
||||
node, err := toNode(toIfaceSlice(maybeNodesGroup)[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes[i] = node
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
func buildStringNode(k, v interface{}, text []byte, pos position) (*ast.StringNode, error) {
|
||||
b, err := base(text, pos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, err := toString(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
value, err := toString(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.StringNode{
|
||||
Base: b,
|
||||
Key: key,
|
||||
Value: value,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildBooleanNode(k, v interface{}, text []byte, pos position) (*ast.BooleanNode, error) {
|
||||
b, err := base(text, pos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, err := toString(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
value, err := toString(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.BooleanNode{
|
||||
Base: b,
|
||||
Key: key,
|
||||
Value: strings.ToLower(value) == "true",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildOperatorNode(text []byte, pos position) (*ast.OperatorNode, error) {
|
||||
b, err := base(text, pos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.OperatorNode{
|
||||
Base: b,
|
||||
Value: string(text),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildGroupNode(k, n interface{}, text []byte, pos position) (*ast.GroupNode, error) {
|
||||
b, err := base(text, pos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, _ := toString(k)
|
||||
|
||||
nodes, err := toNodes(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ast.GroupNode{
|
||||
Base: b,
|
||||
Key: key,
|
||||
Nodes: nodes,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package kql
|
||||
|
||||
//go:generate go run github.com/mna/pigeon -o dictionary_gen.go dictionary.peg
|
||||
@@ -0,0 +1,18 @@
|
||||
// Package kql provides the ability to work with kql queries.
|
||||
package kql
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
// Builder implements kql Builder interface
|
||||
type Builder struct{}
|
||||
|
||||
// Build creates an ast.Ast based on a kql query
|
||||
func (b Builder) Build(q string, opts ...Option) (*ast.Ast, error) {
|
||||
f, err := Parse("", []byte(q), opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.(*ast.Ast), nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package kql_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/kql"
|
||||
)
|
||||
|
||||
func TestNewAST(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
givenQuery string
|
||||
shouldError bool
|
||||
}{
|
||||
{
|
||||
name: "success",
|
||||
givenQuery: "foo:bar",
|
||||
},
|
||||
{
|
||||
name: "error",
|
||||
givenQuery: "AND",
|
||||
shouldError: true,
|
||||
},
|
||||
}
|
||||
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := kql.Builder{}.Build(tt.givenQuery)
|
||||
|
||||
if tt.shouldError {
|
||||
assert.NotNil(err)
|
||||
assert.Nil(got)
|
||||
} else {
|
||||
assert.Nil(err)
|
||||
assert.NotNil(got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package kql
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
var implicitOperatorNodeSource = "implicitly operator"
|
||||
var operatorNodeAnd = ast.OperatorNode{Base: &ast.Base{Loc: &ast.Location{Source: &implicitOperatorNodeSource}}, Value: BoolAND}
|
||||
var operatorNodeOr = ast.OperatorNode{Base: &ast.Base{Loc: &ast.Location{Source: &implicitOperatorNodeSource}}, Value: BoolOR}
|
||||
|
||||
// NormalizeNodes Populate the implicit logical operators in the ast
|
||||
//
|
||||
// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference#constructing-free-text-queries-using-kql
|
||||
// If there are multiple free-text expressions without any operators in between them, the query behavior is the same as using the AND operator.
|
||||
// "John Smith" "Jane Smith"
|
||||
// This functionally is the same as using the AND Boolean operator, as follows:
|
||||
// "John Smith" AND "Jane Smith"
|
||||
//
|
||||
// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference#using-multiple-property-restrictions-within-a-kql-query
|
||||
// When you use multiple instances of the same property restriction, matches are based on the union of the property restrictions in the KQL query.
|
||||
// author:"John Smith" author:"Jane Smith"
|
||||
// This functionally is the same as using the OR Boolean operator, as follows:
|
||||
// author:"John Smith" OR author:"Jane Smith"
|
||||
//
|
||||
// When you use different property restrictions, matches are based on an intersection of the property restrictions in the KQL query, as follows:
|
||||
// author:"John Smith" filetype:docx
|
||||
// This is the same as using the AND Boolean operator, as follows:
|
||||
// author:"John Smith" AND filetype:docx
|
||||
//
|
||||
// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference#grouping-property-restrictions-within-a-kql-query
|
||||
// author:("John Smith" "Jane Smith")
|
||||
// This is the same as using the AND Boolean operator, as follows:
|
||||
// author:"John Smith" AND author:"Jane Smith"
|
||||
func NormalizeNodes(nodes []ast.Node) ([]ast.Node, error) {
|
||||
res := make([]ast.Node, 0, len(nodes))
|
||||
var currentNode ast.Node
|
||||
var prevKey, currentKey *string
|
||||
var operator *ast.OperatorNode
|
||||
for _, node := range nodes {
|
||||
switch n := node.(type) {
|
||||
case *ast.StringNode:
|
||||
if prevKey == nil {
|
||||
prevKey = &n.Key
|
||||
res = append(res, node)
|
||||
continue
|
||||
}
|
||||
currentNode = n
|
||||
currentKey = &n.Key
|
||||
case *ast.BooleanNode:
|
||||
if prevKey == nil {
|
||||
prevKey = &n.Key
|
||||
res = append(res, node)
|
||||
continue
|
||||
}
|
||||
currentNode = n
|
||||
currentKey = &n.Key
|
||||
case *ast.GroupNode:
|
||||
var err error
|
||||
n.Nodes, err = NormalizeNodes(n.Nodes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if prevKey == nil {
|
||||
prevKey = &n.Key
|
||||
res = append(res, n)
|
||||
continue
|
||||
}
|
||||
currentNode = n
|
||||
currentKey = &n.Key
|
||||
case *ast.OperatorNode:
|
||||
if n.Value == BoolNOT {
|
||||
if prevKey == nil {
|
||||
res = append(res, n)
|
||||
} else {
|
||||
operator = n
|
||||
}
|
||||
} else {
|
||||
if prevKey == nil {
|
||||
return nil, &StartsWithBinaryOperatorError{Op: n.Value}
|
||||
}
|
||||
prevKey = nil
|
||||
res = append(res, node)
|
||||
}
|
||||
default:
|
||||
prevKey = nil
|
||||
res = append(res, node)
|
||||
}
|
||||
if prevKey != nil && currentKey != nil {
|
||||
if *prevKey == *currentKey && *prevKey != "" {
|
||||
res = append(res, &operatorNodeOr)
|
||||
} else {
|
||||
res = append(res, &operatorNodeAnd)
|
||||
}
|
||||
if operator != nil {
|
||||
res = append(res, operator)
|
||||
operator = nil
|
||||
}
|
||||
res = append(res, currentNode)
|
||||
|
||||
prevKey = currentKey
|
||||
currentNode = nil
|
||||
currentKey = nil
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return trimOrphan(res), nil
|
||||
}
|
||||
|
||||
func trimOrphan(nodes []ast.Node) []ast.Node {
|
||||
offset := len(nodes)
|
||||
for i := len(nodes) - 1; i >= 0; i-- {
|
||||
if _, ok := nodes[i].(*ast.OperatorNode); ok {
|
||||
offset--
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nodes[:offset]
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package kql_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
tAssert "github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast/test"
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/kql"
|
||||
)
|
||||
|
||||
func TestNormalizeNodes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
givenNodes []ast.Node
|
||||
expectedNodes []ast.Node
|
||||
fixme bool
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "start with binary operator",
|
||||
givenNodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
},
|
||||
expectedError: &kql.StartsWithBinaryOperatorError{Op: "OR"},
|
||||
},
|
||||
{
|
||||
name: "same key implicit OR",
|
||||
givenNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
},
|
||||
expectedNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no key implicit AND",
|
||||
givenNodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.StringNode{Value: "Jane Smith"},
|
||||
},
|
||||
expectedNodes: []ast.Node{
|
||||
&ast.StringNode{Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Value: "Jane Smith"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "same key explicit AND",
|
||||
givenNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
},
|
||||
expectedNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "key-group implicit AND",
|
||||
// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference#grouping-property-restrictions-within-a-kql-query
|
||||
fixme: true,
|
||||
givenNodes: []ast.Node{
|
||||
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
}},
|
||||
},
|
||||
expectedNodes: []ast.Node{
|
||||
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "author", Value: "Jane Smith"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "different key implicit AND",
|
||||
givenNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.StringNode{Key: "filetype", Value: "docx"},
|
||||
},
|
||||
expectedNodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "John Smith"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "filetype", Value: "docx"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert := tAssert.New(t)
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.fixme {
|
||||
t.Skip("not implemented")
|
||||
}
|
||||
|
||||
normalizedNodes, err := kql.NormalizeNodes(tt.givenNodes)
|
||||
|
||||
if tt.expectedError != nil {
|
||||
assert.Equal(err, tt.expectedError)
|
||||
assert.Nil(normalizedNodes)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if diff := test.DiffAst(tt.expectedNodes, normalizedNodes); diff != "" {
|
||||
t.Fatalf("Nodes mismatch (-want +got): %s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Package query provides functions to work with the different search query flavours.
|
||||
package query
|
||||
|
||||
import (
|
||||
"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
|
||||
)
|
||||
|
||||
// Builder is the interface that wraps the basic Build method.
|
||||
type Builder interface {
|
||||
Build(qs string) (*ast.Ast, error)
|
||||
}
|
||||
|
||||
// Compiler is the interface that wraps the basic Compile method.
|
||||
type Compiler[T any] interface {
|
||||
Compile(ast *ast.Ast) (T, error)
|
||||
}
|
||||
|
||||
// Creator is the interface that wraps the basic Create method.
|
||||
type Creator[T any] interface {
|
||||
Create(qs string) (T, error)
|
||||
}
|
||||
Reference in New Issue
Block a user