Initial QSfera import

Server/pkg/kql/cast.go

@@ -0,0 +1,138 @@
+package kql
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/jinzhu/now"
+	"github.com/qsfera/server/pkg/ast"
+	"github.com/qsfera/server/services/search/pkg/query"
+)
+
+func toNode[T ast.Node](in any) (T, error) {
+	var t T
+	out, ok := in.(T)
+	if !ok {
+		return t, fmt.Errorf("can't convert '%T' to '%T'", in, t)
+	}
+
+	return out, nil
+}
+
+func toNodes[T ast.Node](in any) ([]T, error) {
+	switch v := in.(type) {
+	case T:
+		return []T{v}, nil
+	case []T:
+		return v, nil
+	case []*ast.OperatorNode, []*ast.DateTimeNode:
+		return toNodes[T](v)
+	case []any:
+		var nodes []T
+		for _, el := range v {
+			node, err := toNodes[T](el)
+			if err != nil {
+				return nil, err
+			}
+
+			nodes = append(nodes, node...)
+		}
+		return nodes, nil
+	case nil:
+		return nil, nil
+	default:
+		var t T
+		return nil, fmt.Errorf("can't convert '%T' to '%T'", in, t)
+	}
+}
+
+func toString(in any) (string, error) {
+	switch v := in.(type) {
+	case []byte:
+		return string(v), nil
+	case []any:
+		var str string
+
+		for i := range v {
+			sv, err := toString(v[i])
+			if err != nil {
+				return "", err
+			}
+
+			str += sv
+		}
+
+		return str, nil
+	case string:
+		return v, nil
+	default:
+		return "", fmt.Errorf("can't convert '%T' to string", v)
+	}
+}
+
+func toTime(in any) (time.Time, error) {
+	ts, err := toString(in)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	return now.Parse(ts)
+}
+
+func toTimeRange(in any) (*time.Time, *time.Time, error) {
+	var from, to time.Time
+
+	value, err := toString(in)
+	if err != nil {
+		return &from, &to, &query.UnsupportedTimeRangeError{}
+	}
+
+	c := &now.Config{
+		WeekStartDay: time.Monday,
+	}
+
+	n := c.With(timeNow())
+
+	switch value {
+	case "today":
+		from = n.BeginningOfDay()
+		to = n.EndOfDay()
+	case "yesterday":
+		yesterday := n.With(n.AddDate(0, 0, -1))
+		from = yesterday.BeginningOfDay()
+		to = yesterday.EndOfDay()
+	case "this week":
+		from = n.BeginningOfWeek()
+		to = n.EndOfWeek()
+	case "last week":
+		lastWeek := n.With(n.AddDate(0, 0, -7))
+		from = lastWeek.BeginningOfWeek()
+		to = lastWeek.EndOfWeek()
+	case "last 7 days":
+		from = n.With(n.AddDate(0, 0, -6)).BeginningOfDay()
+		to = n.EndOfDay()
+	case "this month":
+		from = n.BeginningOfMonth()
+		to = n.EndOfMonth()
+	case "last month":
+		lastMonth := n.With(n.BeginningOfMonth().AddDate(0, 0, -1))
+		from = lastMonth.BeginningOfMonth()
+		to = lastMonth.EndOfMonth()
+	case "last 30 days":
+		from = n.With(n.AddDate(0, 0, -29)).BeginningOfDay()
+		to = n.EndOfDay()
+	case "this year":
+		from = n.BeginningOfYear()
+		to = n.EndOfYear()
+	case "last year":
+		lastYear := n.With(n.AddDate(-1, 0, 0))
+		from = lastYear.BeginningOfYear()
+		to = lastYear.EndOfYear()
+	}
+
+	if from.IsZero() || to.IsZero() {
+		return nil, nil, &query.UnsupportedTimeRangeError{}
+	}
+
+	return &from, &to, nil
+}

Server/pkg/kql/connect.go

@@ -0,0 +1,129 @@
+package kql
+
+import (
+	"strings"
+
+	"github.com/qsfera/server/pkg/ast"
+)
+
+// connectNodes connects given nodes
+func connectNodes(c Connector, nodes ...ast.Node) []ast.Node {
+	var connectedNodes []ast.Node
+
+	for i := range nodes {
+		ri := len(nodes) - 1 - i
+		head := nodes[ri]
+
+		if connectionNodes := connectNode(c, head, connectedNodes...); len(connectionNodes) > 0 {
+			connectedNodes = append(connectionNodes, connectedNodes...)
+		}
+
+		connectedNodes = append([]ast.Node{head}, connectedNodes...)
+	}
+
+	return connectedNodes
+}
+
+// connectNode connects a tip node with the rest
+func connectNode(c Connector, headNode ast.Node, tailNodes ...ast.Node) []ast.Node {
+	var nearestNeighborNode ast.Node
+	var nearestNeighborOperators []*ast.OperatorNode
+
+l:
+	for _, tailNode := range tailNodes {
+		switch node := tailNode.(type) {
+		case *ast.OperatorNode:
+			nearestNeighborOperators = append(nearestNeighborOperators, node)
+		default:
+			nearestNeighborNode = node
+			break l
+		}
+	}
+
+	if nearestNeighborNode == nil {
+		return nil
+	}
+
+	return c.Connect(headNode, nearestNeighborNode, nearestNeighborOperators)
+}
+
+// Connector is responsible to decide what node connections are needed
+type Connector interface {
+	Connect(head ast.Node, neighbor ast.Node, connections []*ast.OperatorNode) []ast.Node
+}
+
+// DefaultConnector is the default node connector
+type DefaultConnector struct {
+	sameKeyOPValue string
+}
+
+// Connect implements the Connector interface and is used to connect the nodes using
+// the default logic defined by the kql spec.
+func (c DefaultConnector) Connect(head ast.Node, neighbor ast.Node, connections []*ast.OperatorNode) []ast.Node {
+	switch head.(type) {
+	case *ast.OperatorNode:
+		return nil
+	}
+
+	headKey := strings.ToLower(ast.NodeKey(head))
+	neighborKey := strings.ToLower(ast.NodeKey(neighbor))
+
+	connection := &ast.OperatorNode{
+		Base:  &ast.Base{Loc: &ast.Location{Source: &[]string{"implicitly operator"}[0]}},
+		Value: BoolAND,
+	}
+
+	// if the current node and the neighbor node have the same key
+	// the connection is of type OR
+	//
+	// spec: same
+	//		author:"John Smith" author:"Jane Smith"
+	//		author:"John Smith" OR author:"Jane Smith"
+	//
+	// if the nodes have NO key, the edge is a AND connection
+	//
+	// spec: same
+	//		cat dog
+	//		cat AND dog
+	// from the spec:
+	// 		To construct complex queries, you can combine multiple
+	// 		free-text expressions with KQL query operators.
+	// 		If there are multiple free-text expressions without any
+	// 		operators in between them, the query behavior is the same
+	// 		as using the AND operator.
+	//
+	// nodes inside of group node are handled differently,
+	// if no explicit operator given, it uses AND
+	//
+	// spec: same
+	// 		author:"John Smith" AND author:"Jane Smith"
+	// 		author:("John Smith" "Jane Smith")
+	if headKey == neighborKey && headKey != "" && neighborKey != "" {
+		connection.Value = c.sameKeyOPValue
+	}
+
+	// decisions based on nearest neighbor operators
+	for i, node := range connections {
+		// consider direct neighbor operator only
+		if i == 0 {
+			// no connection is necessary here because an `AND` or `OR` edge is already present
+			// exit
+			for _, skipValue := range []string{BoolOR, BoolAND} {
+				if node.Value == skipValue {
+					return nil
+				}
+			}
+
+			// if neighbor node negotiates, an AND edge is needed
+			//
+			// spec: same
+			// 		cat -dog
+			// 		cat AND NOT dog
+			if node.Value == BoolNOT {
+				connection.Value = BoolAND
+			}
+		}
+	}
+
+	return []ast.Node{connection}
+}

Server/pkg/kql/dictionary.peg

@@ -0,0 +1,240 @@
+{
+    package kql
+}
+
+////////////////////////////////////////////////////////
+// ast
+////////////////////////////////////////////////////////
+
+AST <-
+    n:Nodes {
+        return buildAST(n, c.text, c.pos)
+    }
+
+////////////////////////////////////////////////////////
+// nodes
+////////////////////////////////////////////////////////
+
+Nodes <-
+    (_ Node)+
+
+Node <-
+    GroupNode /
+    PropertyRestrictionNodes /
+    OperatorBooleanNodes /
+    FreeTextKeywordNodes
+
+////////////////////////////////////////////////////////
+// nesting
+////////////////////////////////////////////////////////
+
+GroupNode <-
+    k:Key? (OperatorColonNode / OperatorEqualNode)? "(" v:Nodes ")" {
+        return buildGroupNode(k, v, c.text, c.pos)
+    }
+
+////////////////////////////////////////////////////////
+// property restrictions
+////////////////////////////////////////////////////////
+
+PropertyRestrictionNodes <-
+    YesNoPropertyRestrictionNode /
+    DateTimeRestrictionNode /
+    TextPropertyRestrictionNode
+
+YesNoPropertyRestrictionNode <-
+    k:Key (OperatorColonNode / OperatorEqualNode) v:("true" / "false"){
+        return buildBooleanNode(k, v, c.text, c.pos)
+    }
+
+DateTimeRestrictionNode <-
+    k:Key o:(
+        OperatorGreaterOrEqualNode /
+        OperatorLessOrEqualNode /
+        OperatorGreaterNode /
+        OperatorLessNode /
+        OperatorEqualNode /
+        OperatorColonNode
+    ) '"'? v:(
+        DateTime /
+        FullDate /
+        FullTime
+    ) '"'? {
+        return buildDateTimeNode(k, o, v, c.text, c.pos)
+    } /
+    k:Key (
+        OperatorEqualNode /
+        OperatorColonNode
+    ) '"'? v:NaturalLanguageDateTime '"'? {
+        return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos)
+    }
+
+TextPropertyRestrictionNode <-
+    k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){
+        return buildStringNode(k, v, c.text, c.pos)
+    }
+
+////////////////////////////////////////////////////////
+// free text-keywords
+////////////////////////////////////////////////////////
+
+FreeTextKeywordNodes <-
+    PhraseNode /
+    WordNode
+
+PhraseNode <-
+     OperatorColonNode? _ v:String _ OperatorColonNode? {
+        return buildStringNode("", v, c.text, c.pos)
+    }
+
+WordNode <-
+     OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? {
+        return buildStringNode("", v, c.text, c.pos)
+    }
+
+////////////////////////////////////////////////////////
+// operators
+////////////////////////////////////////////////////////
+
+OperatorBooleanNodes <-
+    OperatorBooleanAndNode /
+    OperatorBooleanNotNode /
+    OperatorBooleanOrNode
+
+OperatorBooleanAndNode <-
+    ("AND" / "+") {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorBooleanNotNode <-
+    ("NOT" / "-") {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorBooleanOrNode <-
+    ("OR") {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorColonNode <-
+    ":" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorEqualNode <-
+    "=" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorLessNode <-
+    "<" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorLessOrEqualNode <-
+    "<=" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorGreaterNode <-
+    ">" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+OperatorGreaterOrEqualNode <-
+    ">=" {
+        return buildOperatorNode(c.text, c.pos)
+    }
+
+
+////////////////////////////////////////////////////////
+// time
+////////////////////////////////////////////////////////
+
+TimeYear <-
+    Digit Digit Digit Digit {
+        return c.text, nil
+    }
+
+TimeMonth <-
+    Digit Digit {
+        return c.text, nil
+    }
+
+TimeDay <-
+    Digit Digit {
+        return c.text, nil
+    }
+
+TimeHour <-
+    Digit Digit {
+        return c.text, nil
+    }
+
+TimeMinute <-
+    Digit Digit {
+        return c.text, nil
+    }
+
+TimeSecond <-
+    Digit Digit {
+        return c.text, nil
+    }
+
+FullDate <-
+    TimeYear "-" TimeMonth "-" TimeDay {
+        return c.text, nil
+    }
+
+FullTime <-
+    TimeHour ":" TimeMinute ":" TimeSecond ("." Digit+)? ("Z" / ("+" / "-") TimeHour ":" TimeMinute) {
+        return c.text, nil
+    }
+
+DateTime  <-
+    FullDate "T" FullTime {
+      return c.text, nil
+    }
+
+NaturalLanguageDateTime  <-
+    "today" /
+    "yesterday" /
+    "this week" /
+    "last week" /
+    "last 7 days" /
+    "this month" /
+    "last month" /
+    "last 30 days" /
+    "this year" /
+    "last year"  {
+        return c.text, nil
+    }
+
+////////////////////////////////////////////////////////
+// misc
+////////////////////////////////////////////////////////
+
+Char <-
+    [A-Za-z] {
+        return c.text, nil
+    }
+
+Key <-
+    Char+ ("." Char+)* {
+        return c.text, nil
+    }
+
+String <-
+    '"' v:[^"]* '"' {
+        return v, nil
+    }
+
+Digit <-
+    [0-9] {
+        return c.text, nil
+    }
+
+_ <-
+    [ \t]* {
+       return nil, nil
+    }

Server/pkg/kql/doc.go

@@ -0,0 +1,27 @@
+/*
+Package kql provides the ability to work with kql queries.
+
+Not every aspect of the spec is implemented yet.
+The language support will grow over time if needed.
+
+The following spec parts are supported and tested:
+  - 2.1.2 AND Operator
+  - 2.1.6 NOT Operator
+  - 2.1.8 OR Operator
+  - 2.1.12 Parentheses
+  - 2.3.5 Date Tokens
+  - 3.1.11 Implicit Operator
+  - 3.1.12 Parentheses
+  - 3.1.2 AND Operator
+  - 3.1.6 NOT Operator
+  - 3.1.8 OR Operator
+  - 3.2.3 Implicit Operator for Property Restriction
+  - 3.3.1.1.1 Implicit AND Operator
+  - 3.3.5 Date Tokens
+
+References:
+  - https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference
+  - https://learn.microsoft.com/en-us/openspecs/sharepoint_protocols/ms-kql/3bbf06cd-8fc1-4277-bd92-8661ccd3c9b0
+  - https://msopenspecs.azureedge.net/files/MS-KQL/%5bMS-KQL%5d.pdf
+*/
+package kql

Server/pkg/kql/engine_suite_test.go

@@ -0,0 +1,11 @@
+package kql
+
+import (
+	"time"
+)
+
+// PatchTimeNow is here to patch the package time now func,
+// which is used in the test suite
+func PatchTimeNow(t func() time.Time) {
+	timeNow = t
+}

Server/pkg/kql/factory.go

@@ -0,0 +1,209 @@
+package kql
+
+import (
+	"strings"
+
+	"github.com/qsfera/server/pkg/ast"
+)
+
+func base(text []byte, pos position) (*ast.Base, error) {
+	source, err := toString(text)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ast.Base{
+		Loc: &ast.Location{
+			Start: ast.Position{
+				Line:   pos.line,
+				Column: pos.col,
+			},
+			End: ast.Position{
+				Line:   pos.line,
+				Column: pos.col + len(text),
+			},
+			Source: &source,
+		},
+	}, nil
+}
+
+func buildAST(n any, text []byte, pos position) (*ast.Ast, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	nodes, err := toNodes[ast.Node](n)
+	if err != nil {
+		return nil, err
+	}
+
+	a := &ast.Ast{
+		Base:  b,
+		Nodes: connectNodes(DefaultConnector{sameKeyOPValue: BoolOR}, nodes...),
+	}
+
+	if err := validateAst(a); err != nil {
+		return nil, err
+	}
+
+	return a, nil
+}
+
+func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := toString(k)
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := toString(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ast.StringNode{
+		Base:  b,
+		Key:   key,
+		Value: value,
+	}, nil
+}
+
+func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNode, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	operator, err := toNode[*ast.OperatorNode](o)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := toString(k)
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := toTime(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ast.DateTimeNode{
+		Base:     b,
+		Key:      key,
+		Operator: operator,
+		Value:    value,
+	}, nil
+}
+func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := toString(k)
+	if err != nil {
+		return nil, err
+	}
+
+	from, to, err := toTimeRange(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return []ast.Node{
+		&ast.DateTimeNode{
+			Base:     b,
+			Value:    *from,
+			Key:      key,
+			Operator: &ast.OperatorNode{Value: ">="},
+		},
+		&ast.OperatorNode{Value: BoolAND},
+		&ast.DateTimeNode{
+			Base:     b,
+			Value:    *to,
+			Key:      key,
+			Operator: &ast.OperatorNode{Value: "<="},
+		},
+	}, nil
+
+}
+
+func buildBooleanNode(k, v any, text []byte, pos position) (*ast.BooleanNode, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := toString(k)
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := toString(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ast.BooleanNode{
+		Base:  b,
+		Key:   key,
+		Value: strings.ToLower(value) == "true",
+	}, nil
+}
+
+func buildOperatorNode(text []byte, pos position) (*ast.OperatorNode, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	value, err := toString(text)
+	if err != nil {
+		return nil, err
+	}
+
+	switch value {
+	case "+":
+		value = BoolAND
+	case "-":
+		value = BoolNOT
+	}
+
+	return &ast.OperatorNode{
+		Base:  b,
+		Value: value,
+	}, nil
+}
+
+func buildGroupNode(k, n any, text []byte, pos position) (*ast.GroupNode, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	key, _ := toString(k)
+
+	nodes, err := toNodes[ast.Node](n)
+	if err != nil {
+		return nil, err
+	}
+
+	gn := &ast.GroupNode{
+		Base:  b,
+		Key:   key,
+		Nodes: connectNodes(DefaultConnector{sameKeyOPValue: BoolOR}, nodes...),
+	}
+
+	if err := validateGroupNode(gn); err != nil {
+		return nil, err
+	}
+
+	return gn, nil
+}

Server/pkg/kql/gen.go

@@ -0,0 +1,3 @@
+package kql
+
+//go:generate go run github.com/mna/pigeon -optimize-grammar -optimize-parser -o dictionary_gen.go dictionary.peg

Server/pkg/kql/kql.go

@@ -0,0 +1,49 @@
+// Package kql provides the ability to work with kql queries.
+package kql
+
+import (
+	"errors"
+	"time"
+
+	"github.com/qsfera/server/pkg/ast"
+)
+
+// The operator node value definition
+const (
+	// BoolAND connect two nodes with "AND"
+	BoolAND = "AND"
+	// BoolOR connect two nodes with "OR"
+	BoolOR = "OR"
+	// BoolNOT connect two nodes with "NOT"
+	BoolNOT = "NOT"
+)
+
+// Builder implements kql Builder interface
+type Builder struct{}
+
+// Build creates an ast.Ast based on a kql query
+func (b Builder) Build(q string) (*ast.Ast, error) {
+	f, err := Parse("", []byte(q))
+	if err != nil {
+		var list errList
+		errors.As(err, &list)
+
+		for _, listError := range list {
+			var parserError *parserError
+			switch {
+			case errors.As(listError, &parserError):
+				if parserError.Inner != nil {
+					return nil, parserError.Inner
+				}
+
+				return nil, listError
+			}
+		}
+	}
+
+	return f.(*ast.Ast), nil
+}
+
+// timeNow mirrors time.Now by default, the only reason why this exists
+// is to monkey patch it from the tests. See PatchTimeNow
+var timeNow = time.Now

Server/pkg/kql/kql_test.go

@@ -0,0 +1,53 @@
+package kql_test
+
+import (
+	"testing"
+
+	"github.com/qsfera/server/pkg/ast"
+	"github.com/qsfera/server/pkg/kql"
+	"github.com/qsfera/server/services/search/pkg/query"
+	tAssert "github.com/stretchr/testify/assert"
+)
+
+func TestNewAST(t *testing.T) {
+	tests := []struct {
+		name          string
+		givenQuery    string
+		expectedError error
+	}{
+		{
+			name:       "success",
+			givenQuery: "foo:bar",
+		},
+		{
+			name:       "error",
+			givenQuery: kql.BoolAND,
+			expectedError: query.StartsWithBinaryOperatorError{
+				Node: &ast.OperatorNode{Value: kql.BoolAND},
+			},
+		},
+	}
+
+	assert := tAssert.New(t)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := kql.Builder{}.Build(tt.givenQuery)
+
+			if tt.expectedError != nil {
+				if tt.expectedError.Error() != "" {
+					assert.Equal(err.Error(), tt.expectedError.Error())
+				} else {
+					assert.NotNil(err)
+				}
+
+				assert.Nil(got)
+
+				return
+			}
+
+			assert.Nil(err)
+			assert.NotNil(got)
+		})
+	}
+}

Server/pkg/kql/validate.go

@@ -0,0 +1,37 @@
+package kql
+
+import (
+	"github.com/qsfera/server/pkg/ast"
+	"github.com/qsfera/server/services/search/pkg/query"
+)
+
+func validateAst(a *ast.Ast) error {
+	switch node := a.Nodes[0].(type) {
+	case *ast.OperatorNode:
+		switch node.Value {
+		case BoolAND, BoolOR:
+			return &query.StartsWithBinaryOperatorError{Node: node}
+		}
+	}
+	return nil
+}
+
+func validateGroupNode(n *ast.GroupNode) error {
+	switch node := n.Nodes[0].(type) {
+	case *ast.OperatorNode:
+		switch node.Value {
+		case BoolAND, BoolOR:
+			return &query.StartsWithBinaryOperatorError{Node: node}
+		}
+	}
+
+	if n.Key != "" {
+		for _, node := range n.Nodes {
+			if ast.NodeKey(node) != "" {
+				return &query.NamedGroupInvalidNodesError{Node: node}
+			}
+		}
+	}
+
+	return nil
+}