Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+306
View File
@@ -0,0 +1,306 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"math/big"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinCount(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
return iter(ast.InternedTerm(a.Len()))
case ast.Object:
return iter(ast.InternedTerm(a.Len()))
case ast.Set:
return iter(ast.InternedTerm(a.Len()))
case ast.String:
return iter(ast.InternedTerm(len([]rune(a))))
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "array", "object", "set", "string")
}
func builtinSum(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
// Fast path for arrays of integers
is := 0
nonInts := a.Until(func(x *ast.Term) bool {
if n, ok := x.Value.(ast.Number); ok {
if i, ok := n.Int(); ok {
is += i
return false
}
}
return true
})
if !nonInts {
return iter(ast.InternedTerm(is))
}
// Non-integer values found, so we need to sum as floats.
sum := big.NewFloat(0)
tmp := new(big.Float)
err := a.Iter(func(x *ast.Term) error {
n, ok := x.Value.(ast.Number)
if !ok {
return builtins.NewOperandElementErr(1, a, x.Value, "number")
}
sum = new(big.Float).Add(sum, builtins.NumberToFloatInto(tmp, n))
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(sum)))
case ast.Set:
// Fast path for sets of integers
is := 0
nonInts := a.Until(func(x *ast.Term) bool {
if n, ok := x.Value.(ast.Number); ok {
if i, ok := n.Int(); ok {
is += i
return false
}
}
return true
})
if !nonInts {
return iter(ast.InternedTerm(is))
}
sum := big.NewFloat(0)
tmp := new(big.Float)
err := a.Iter(func(x *ast.Term) error {
n, ok := x.Value.(ast.Number)
if !ok {
return builtins.NewOperandElementErr(1, a, x.Value, "number")
}
sum = new(big.Float).Add(sum, builtins.NumberToFloatInto(tmp, n))
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(sum)))
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "set", "array")
}
func builtinProduct(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
product := big.NewFloat(1)
tmp := new(big.Float)
err := a.Iter(func(x *ast.Term) error {
n, ok := x.Value.(ast.Number)
if !ok {
return builtins.NewOperandElementErr(1, a, x.Value, "number")
}
product = new(big.Float).Mul(product, builtins.NumberToFloatInto(tmp, n))
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(product)))
case ast.Set:
product := big.NewFloat(1)
tmp := new(big.Float)
err := a.Iter(func(x *ast.Term) error {
n, ok := x.Value.(ast.Number)
if !ok {
return builtins.NewOperandElementErr(1, a, x.Value, "number")
}
product = new(big.Float).Mul(product, builtins.NumberToFloatInto(tmp, n))
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(product)))
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "set", "array")
}
func builtinMax(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
if a.Len() == 0 {
return nil
}
max := ast.InternedNullTerm.Value
a.Foreach(func(x *ast.Term) {
if ast.Compare(max, x.Value) <= 0 {
max = x.Value
}
})
return iter(ast.NewTerm(max))
case ast.Set:
if a.Len() == 0 {
return nil
}
max, err := a.Reduce(ast.InternedNullTerm, func(max *ast.Term, elem *ast.Term) (*ast.Term, error) {
if ast.Compare(max, elem) <= 0 {
return elem, nil
}
return max, nil
})
if err != nil {
return err
}
return iter(max)
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "set", "array")
}
func builtinMin(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
if a.Len() == 0 {
return nil
}
min := a.Elem(0).Value
a.Foreach(func(x *ast.Term) {
if ast.Compare(min, x.Value) >= 0 {
min = x.Value
}
})
return iter(ast.NewTerm(min))
case ast.Set:
if a.Len() == 0 {
return nil
}
min, err := a.Reduce(ast.InternedNullTerm, func(min *ast.Term, elem *ast.Term) (*ast.Term, error) {
// The null term is considered to be less than any other term,
// so in order for min of a set to make sense, we need to check
// for it.
if min.Value.Compare(ast.InternedNullValue) == 0 {
return elem, nil
}
if ast.Compare(min, elem) >= 0 {
return elem, nil
}
return min, nil
})
if err != nil {
return err
}
return iter(min)
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "set", "array")
}
func builtinSort(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case *ast.Array:
return iter(ast.NewTerm(a.Sorted()))
case ast.Set:
return iter(ast.NewTerm(a.Sorted()))
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "set", "array")
}
func builtinAll(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.Set:
res := true
match := ast.InternedTerm(true)
val.Until(func(term *ast.Term) bool {
if !match.Equal(term) {
res = false
return true
}
return false
})
return iter(ast.InternedTerm(res))
case *ast.Array:
res := true
match := ast.InternedTerm(true)
val.Until(func(term *ast.Term) bool {
if !match.Equal(term) {
res = false
return true
}
return false
})
return iter(ast.InternedTerm(res))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "array", "set")
}
}
func builtinAny(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.Set:
res := val.Len() > 0 && val.Contains(ast.InternedTerm(true))
return iter(ast.InternedTerm(res))
case *ast.Array:
res := false
match := ast.InternedTerm(true)
val.Until(func(term *ast.Term) bool {
if match.Equal(term) {
res = true
return true
}
return false
})
return iter(ast.InternedTerm(res))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "array", "set")
}
}
func builtinMember(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
containee := operands[0]
switch c := operands[1].Value.(type) {
case ast.Set:
return iter(ast.InternedTerm(c.Contains(containee)))
case *ast.Array:
for i := range c.Len() {
if c.Elem(i).Value.Compare(containee.Value) == 0 {
return iter(ast.InternedTerm(true))
}
}
return iter(ast.InternedTerm(false))
case ast.Object:
return iter(ast.InternedTerm(c.Until(func(_, v *ast.Term) bool {
return v.Value.Compare(containee.Value) == 0
})))
}
return iter(ast.InternedTerm(false))
}
func builtinMemberWithKey(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
key, val := operands[0], operands[1]
switch c := operands[2].Value.(type) {
case interface{ Get(*ast.Term) *ast.Term }:
ret := false
if act := c.Get(key); act != nil {
ret = act.Value.Compare(val.Value) == 0
}
return iter(ast.InternedTerm(ret))
}
return iter(ast.InternedTerm(false))
}
func init() {
RegisterBuiltinFunc(ast.Count.Name, builtinCount)
RegisterBuiltinFunc(ast.Sum.Name, builtinSum)
RegisterBuiltinFunc(ast.Product.Name, builtinProduct)
RegisterBuiltinFunc(ast.Max.Name, builtinMax)
RegisterBuiltinFunc(ast.Min.Name, builtinMin)
RegisterBuiltinFunc(ast.Sort.Name, builtinSort)
RegisterBuiltinFunc(ast.Any.Name, builtinAny)
RegisterBuiltinFunc(ast.All.Name, builtinAll)
RegisterBuiltinFunc(ast.Member.Name, builtinMember)
RegisterBuiltinFunc(ast.MemberWithKey.Name, builtinMemberWithKey)
}
+240
View File
@@ -0,0 +1,240 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"math/big"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
type arithArity1 func(a *big.Float) (*big.Float, error)
type arithArity2 func(a, b *big.Float) (*big.Float, error)
func arithAbs(a *big.Float) (*big.Float, error) {
return a.Abs(a), nil
}
var halfAwayFromZero = big.NewFloat(0.5)
func arithRound(a *big.Float) (*big.Float, error) {
var i *big.Int
if a.Signbit() {
i, _ = new(big.Float).Sub(a, halfAwayFromZero).Int(nil)
} else {
i, _ = new(big.Float).Add(a, halfAwayFromZero).Int(nil)
}
return new(big.Float).SetInt(i), nil
}
func arithCeil(a *big.Float) (*big.Float, error) {
i, _ := a.Int(nil)
f := new(big.Float).SetInt(i)
if f.Signbit() || a.Cmp(f) == 0 {
return f, nil
}
return new(big.Float).Add(f, big.NewFloat(1.0)), nil
}
func arithFloor(a *big.Float) (*big.Float, error) {
i, _ := a.Int(nil)
f := new(big.Float).SetInt(i)
if !f.Signbit() || a.Cmp(f) == 0 {
return f, nil
}
return new(big.Float).Sub(f, big.NewFloat(1.0)), nil
}
func builtinPlus(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n1, err := builtins.NumberOperand(operands[0].Value, 1)
if err != nil {
return err
}
n2, err := builtins.NumberOperand(operands[1].Value, 2)
if err != nil {
return err
}
x, ok1 := n1.Int()
y, ok2 := n2.Int()
if ok1 && ok2 && inSmallIntRange(x) && inSmallIntRange(y) {
return iter(ast.InternedTerm(x + y))
}
f := new(big.Float).Add(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2))
return iter(ast.NewTerm(builtins.FloatToNumber(f)))
}
func builtinMultiply(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n1, err := builtins.NumberOperand(operands[0].Value, 1)
if err != nil {
return err
}
n2, err := builtins.NumberOperand(operands[1].Value, 2)
if err != nil {
return err
}
x, ok1 := n1.Int()
y, ok2 := n2.Int()
if ok1 && ok2 && inSmallIntRange(x) && inSmallIntRange(y) {
return iter(ast.InternedTerm(x * y))
}
f := new(big.Float).Mul(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2))
return iter(ast.NewTerm(builtins.FloatToNumber(f)))
}
func arithDivide(a, b *big.Float) (*big.Float, error) {
i, acc := b.Int64()
if acc == big.Exact && i == 0 {
return nil, errors.New("divide by zero")
}
return new(big.Float).Quo(a, b), nil
}
func arithRem(a, b *big.Int) (*big.Int, error) {
if b.Int64() == 0 {
return nil, errors.New("modulo by zero")
}
return new(big.Int).Rem(a, b), nil
}
func builtinArithArity1(fn arithArity1) BuiltinFunc {
return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n, err := builtins.NumberOperand(operands[0].Value, 1)
if err != nil {
return err
}
f, err := fn(builtins.NumberToFloat(n))
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(f)))
}
}
func builtinArithArity2(fn arithArity2) BuiltinFunc {
return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n1, err := builtins.NumberOperand(operands[0].Value, 1)
if err != nil {
return err
}
n2, err := builtins.NumberOperand(operands[1].Value, 2)
if err != nil {
return err
}
f, err := fn(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2))
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.FloatToNumber(f)))
}
}
func builtinMinus(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n1, ok1 := operands[0].Value.(ast.Number)
n2, ok2 := operands[1].Value.(ast.Number)
if ok1 && ok2 {
x, okx := n1.Int()
y, oky := n2.Int()
if okx && oky && inSmallIntRange(x) && inSmallIntRange(y) {
return iter(ast.InternedTerm(x - y))
}
f := new(big.Float).Sub(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2))
return iter(ast.NewTerm(builtins.FloatToNumber(f)))
}
s1, ok3 := operands[0].Value.(ast.Set)
s2, ok4 := operands[1].Value.(ast.Set)
if ok3 && ok4 {
diff := s1.Diff(s2)
if diff.Len() == 0 {
return iter(ast.InternedEmptySet)
}
return iter(ast.NewTerm(diff))
}
if !ok1 && !ok3 {
return builtins.NewOperandTypeErr(1, operands[0].Value, "number", "set")
}
if ok2 {
return builtins.NewOperandTypeErr(2, operands[1].Value, "set")
}
return builtins.NewOperandTypeErr(2, operands[1].Value, "number")
}
func builtinRem(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
n1, ok1 := operands[0].Value.(ast.Number)
n2, ok2 := operands[1].Value.(ast.Number)
if ok1 && ok2 {
x, okx := n1.Int()
y, oky := n2.Int()
if okx && oky && inSmallIntRange(x) && inSmallIntRange(y) {
if y == 0 {
return errors.New("modulo by zero")
}
return iter(ast.InternedTerm(x % y))
}
op1, err1 := builtins.NumberToInt(n1)
op2, err2 := builtins.NumberToInt(n2)
if err1 != nil || err2 != nil {
return errors.New("modulo on floating-point number")
}
i, err := arithRem(op1, op2)
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.IntToNumber(i)))
}
if !ok1 {
return builtins.NewOperandTypeErr(1, operands[0].Value, "number")
}
return builtins.NewOperandTypeErr(2, operands[1].Value, "number")
}
func inSmallIntRange(num int) bool {
return -1000 < num && num < 1000
}
func init() {
RegisterBuiltinFunc(ast.Abs.Name, builtinArithArity1(arithAbs))
RegisterBuiltinFunc(ast.Round.Name, builtinArithArity1(arithRound))
RegisterBuiltinFunc(ast.Ceil.Name, builtinArithArity1(arithCeil))
RegisterBuiltinFunc(ast.Floor.Name, builtinArithArity1(arithFloor))
RegisterBuiltinFunc(ast.Plus.Name, builtinPlus)
RegisterBuiltinFunc(ast.Minus.Name, builtinMinus)
RegisterBuiltinFunc(ast.Multiply.Name, builtinMultiply)
RegisterBuiltinFunc(ast.Divide.Name, builtinArithArity2(arithDivide))
RegisterBuiltinFunc(ast.Rem.Name, builtinRem)
}
+152
View File
@@ -0,0 +1,152 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinArrayConcat(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arrA, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
arrB, err := builtins.ArrayOperand(operands[1].Value, 2)
if err != nil {
return err
}
if arrA.Len() == 0 {
return iter(operands[1])
}
if arrB.Len() == 0 {
return iter(operands[0])
}
arrC := make([]*ast.Term, arrA.Len()+arrB.Len())
i := 0
arrA.Foreach(func(elemA *ast.Term) {
arrC[i] = elemA
i++
})
arrB.Foreach(func(elemB *ast.Term) {
arrC[i] = elemB
i++
})
return iter(ast.ArrayTerm(arrC...))
}
func builtinArrayFlatten(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
size := arr.Len()
preAlloc := size
containsArray := false
for i := range size {
if nested, ok := arr.Elem(i).Value.(*ast.Array); ok {
containsArray = true
preAlloc += nested.Len() - 1
}
}
if !containsArray && size == preAlloc {
return iter(operands[0]) // Empty array, or no nested arrays -> nothing to flatten.
}
flattened := make([]*ast.Term, 0, preAlloc)
for i := range size {
elem := arr.Elem(i)
if nested, ok := elem.Value.(*ast.Array); ok {
for j := range nested.Len() {
flattened = append(flattened, nested.Elem(j))
}
} else {
flattened = append(flattened, elem)
}
}
return iter(ast.ArrayTerm(flattened...))
}
func builtinArraySlice(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
startIndex, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil {
return err
}
stopIndex, err := builtins.IntOperand(operands[2].Value, 3)
if err != nil {
return err
}
l := arr.Len()
// Clamp stopIndex to avoid out-of-range errors. If negative, clamp to zero.
// Otherwise, clamp to length of array.
if stopIndex < 0 {
stopIndex = 0
} else if stopIndex > l {
stopIndex = l
}
// Clamp startIndex to avoid out-of-range errors. If negative, clamp to zero.
// Otherwise, clamp to stopIndex to avoid to avoid cases like arr[1:0].
if startIndex < 0 {
startIndex = 0
} else if startIndex > stopIndex {
startIndex = stopIndex
}
if startIndex == 0 && stopIndex >= l {
return iter(operands[0])
}
return iter(ast.NewTerm(arr.Slice(startIndex, stopIndex)))
}
func builtinArrayReverse(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
length := arr.Len()
if length == 0 {
return iter(ast.InternedEmptyArray)
}
if length == 1 {
return iter(operands[0])
}
reversedArr := make([]*ast.Term, length)
for index := range length {
reversedArr[index] = arr.Elem(length - index - 1)
}
return iter(ast.ArrayTerm(reversedArr...))
}
func init() {
RegisterBuiltinFunc(ast.ArrayConcat.Name, builtinArrayConcat)
RegisterBuiltinFunc(ast.ArrayFlatten.Name, builtinArrayFlatten)
RegisterBuiltinFunc(ast.ArraySlice.Name, builtinArraySlice)
RegisterBuiltinFunc(ast.ArrayReverse.Name, builtinArrayReverse)
}
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinBinaryAnd(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.SetOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.SetOperand(operands[1].Value, 2)
if err != nil {
return err
}
i := s1.Intersect(s2)
if i.Len() == 0 {
return iter(ast.InternedEmptySet)
}
return iter(ast.NewTerm(i))
}
func builtinBinaryOr(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.SetOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.SetOperand(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.NewTerm(s1.Union(s2)))
}
func init() {
RegisterBuiltinFunc(ast.And.Name, builtinBinaryAnd)
RegisterBuiltinFunc(ast.Or.Name, builtinBinaryOr)
}
+473
View File
@@ -0,0 +1,473 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"strconv"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
)
type undo struct {
k *ast.Term
u *bindings
}
func (u *undo) Undo() {
if u == nil {
// Allow call on zero value of Undo for ease-of-use.
return
}
if u.u == nil {
// Call on empty unifier undos a no-op unify operation.
return
}
u.u.delete(u.k)
}
type bindings struct {
id uint64
values bindingsArrayHashmap
instr *Instrumentation
}
func newBindings(id uint64, instr *Instrumentation) *bindings {
values := newBindingsArrayHashmap()
return &bindings{id, values, instr}
}
// newBindingsWithSize creates bindings pre-sized for the expected number of entries.
// This avoids over-allocation when the binding count is known in advance (e.g., function arguments).
// For sizeHint <= maxLinearScan, it uses array mode; for larger hints, it pre-allocates a map.
func newBindingsWithSize(id uint64, instr *Instrumentation, sizeHint int) *bindings {
values := newBindingsArrayHashmapWithSize(sizeHint)
return &bindings{id, values, instr}
}
func (u *bindings) Iter(caller *bindings, iter func(*ast.Term, *ast.Term) error) error {
var err error
u.values.Iter(func(k *ast.Term, _ value) bool {
if err != nil {
return true
}
err = iter(k, u.PlugNamespaced(k, caller))
return false
})
return err
}
func (u *bindings) Namespace(x ast.Node, caller *bindings) {
vis := namespacingVisitor{
b: u,
caller: caller,
}
ast.NewGenericVisitor(vis.Visit).Walk(x)
}
func (u *bindings) Plug(a *ast.Term) *ast.Term {
return u.PlugNamespaced(a, nil)
}
func (u *bindings) PlugNamespaced(a *ast.Term, caller *bindings) *ast.Term {
if u != nil && u.instr != nil {
u.instr.startTimer(evalOpPlug)
t := u.plugNamespaced(a, caller)
u.instr.stopTimer(evalOpPlug)
return t
}
return u.plugNamespaced(a, caller)
}
func (u *bindings) plugNamespaced(a *ast.Term, caller *bindings) *ast.Term {
switch v := a.Value.(type) {
case ast.Var:
b, next := u.apply(a)
if a != b || u != next {
return next.plugNamespaced(b, caller)
}
return u.namespaceVar(b, caller)
case *ast.Array:
if a.IsGround() {
return a
}
cpy := *a
arr := make([]*ast.Term, v.Len())
for i := range arr {
arr[i] = u.plugNamespaced(v.Elem(i), caller)
}
cpy.Value = ast.NewArray(arr...)
return &cpy
case ast.Object:
if a.IsGround() {
return a
}
cpy := *a
cpy.Value, _ = v.Map(func(k, v *ast.Term) (*ast.Term, *ast.Term, error) {
return u.plugNamespaced(k, caller), u.plugNamespaced(v, caller), nil
})
return &cpy
case ast.Set:
if a.IsGround() {
return a
}
cpy := *a
cpy.Value, _ = v.Map(func(x *ast.Term) (*ast.Term, error) {
return u.plugNamespaced(x, caller), nil
})
return &cpy
case ast.Ref:
cpy := *a
ref := make(ast.Ref, len(v))
for i := range ref {
ref[i] = u.plugNamespaced(v[i], caller)
}
cpy.Value = ref
return &cpy
}
return a
}
func (u *bindings) bind(a *ast.Term, b *ast.Term, other *bindings, und *undo) {
u.values.Put(a, value{
u: other,
v: b,
})
und.k = a
und.u = u
}
func (u *bindings) apply(a *ast.Term) (*ast.Term, *bindings) {
// Early exit for non-var terms. Only vars are bound in the binding list,
// so the lookup below will always fail for non-var terms. In some cases,
// the lookup may be expensive as it has to hash the term (which for large
// inputs can be costly).
_, ok := a.Value.(ast.Var)
if !ok {
return a, u
}
val, ok := u.get(a)
if !ok {
return a, u
}
return val.u.apply(val.v)
}
func (u *bindings) delete(v *ast.Term) {
u.values.Delete(v)
}
func (u *bindings) get(v *ast.Term) (value, bool) {
if u == nil {
return value{}, false
}
return u.values.Get(v)
}
func (u *bindings) String() string {
if u == nil {
return "()"
}
var buf []string
u.values.Iter(func(a *ast.Term, b value) bool {
buf = append(buf, fmt.Sprintf("%v: %v", a, b))
return false
})
return fmt.Sprintf("({%v}, %v)", strings.Join(buf, ", "), u.id)
}
func (u *bindings) namespaceVar(v *ast.Term, caller *bindings) *ast.Term {
name, ok := v.Value.(ast.Var)
if !ok {
panic("illegal value")
}
if caller != nil && caller != u {
// Root documents (i.e., data, input) should never be namespaced because they
// are globally unique.
if !ast.RootDocumentNames.Contains(v) {
return ast.VarTerm(string(name) + strconv.FormatUint(u.id, 10))
}
}
return v
}
type value struct {
u *bindings
v *ast.Term
}
func (v value) String() string {
return fmt.Sprintf("(%v, %d)", v.v, v.u.id)
}
func (v value) equal(other *value) bool {
if v.u == other.u {
return v.v.Equal(other.v)
}
return false
}
type namespacingVisitor struct {
b *bindings
caller *bindings
}
func (vis namespacingVisitor) Visit(x any) bool {
switch x := x.(type) {
case *ast.ArrayComprehension:
x.Term = vis.namespaceTerm(x.Term)
vis := ast.NewGenericVisitor(vis.Visit)
for _, expr := range x.Body {
vis.Walk(expr)
}
return true
case *ast.SetComprehension:
x.Term = vis.namespaceTerm(x.Term)
vis := ast.NewGenericVisitor(vis.Visit)
for _, expr := range x.Body {
vis.Walk(expr)
}
return true
case *ast.ObjectComprehension:
x.Key = vis.namespaceTerm(x.Key)
x.Value = vis.namespaceTerm(x.Value)
vis := ast.NewGenericVisitor(vis.Visit)
for _, expr := range x.Body {
vis.Walk(expr)
}
return true
case *ast.Expr:
switch terms := x.Terms.(type) {
case []*ast.Term:
for i := 1; i < len(terms); i++ {
terms[i] = vis.namespaceTerm(terms[i])
}
case *ast.Term:
x.Terms = vis.namespaceTerm(terms)
}
for _, w := range x.With {
w.Target = vis.namespaceTerm(w.Target)
w.Value = vis.namespaceTerm(w.Value)
}
}
return false
}
func (vis namespacingVisitor) namespaceTerm(a *ast.Term) *ast.Term {
switch v := a.Value.(type) {
case ast.Var:
return vis.b.namespaceVar(a, vis.caller)
case *ast.Array:
if a.IsGround() {
return a
}
cpy := *a
arr := make([]*ast.Term, v.Len())
for i := range arr {
arr[i] = vis.namespaceTerm(v.Elem(i))
}
cpy.Value = ast.NewArray(arr...)
return &cpy
case ast.Object:
if a.IsGround() {
return a
}
cpy := *a
cpy.Value, _ = v.Map(func(k, v *ast.Term) (*ast.Term, *ast.Term, error) {
return vis.namespaceTerm(k), vis.namespaceTerm(v), nil
})
return &cpy
case ast.Set:
if a.IsGround() {
return a
}
cpy := *a
cpy.Value, _ = v.Map(func(x *ast.Term) (*ast.Term, error) {
return vis.namespaceTerm(x), nil
})
return &cpy
case ast.Ref:
cpy := *a
ref := make(ast.Ref, len(v))
for i := range ref {
ref[i] = vis.namespaceTerm(v[i])
}
cpy.Value = ref
return &cpy
}
return a
}
const maxLinearScan = 16
// bindingsArrayHashMap uses a dynamically growing slice with linear scan instead
// of a hash map for smaller # of entries. Hash maps start to
// show off their performance advantage only after 16 keys.
//
// Memory optimization: The slice grows incrementally (2 -> 4 -> 8 -> 16) to avoid
// wasting memory when only a few bindings are used. This is critical for scenarios
// like comprehensions and functions with few arguments that are called thousands of times.
type bindingsArrayHashmap struct {
n int // Entries in the slice.
a []bindingArrayKeyValue
m map[ast.Var]bindingArrayKeyValue
}
type bindingArrayKeyValue struct {
key *ast.Term
value value
}
func newBindingsArrayHashmap() bindingsArrayHashmap {
return bindingsArrayHashmap{}
}
// newBindingsArrayHashmapWithSize creates a bindingsArrayHashmap pre-sized for the expected number of entries.
// This optimization reduces memory waste when the binding count is known in advance.
//
// Size selection strategy:
// - sizeHint == 0: lazy allocation (no pre-allocation)
// - sizeHint <= maxLinearScan: pre-allocate slice with exact capacity to avoid reallocation
// - sizeHint > maxLinearScan: pre-allocate map with exact capacity
//
// Memory impact example:
// - Without hint: dynamic growth 0 -> 2 -> 4 -> 8 -> 16 (saves memory for small counts)
// - With hint=2: pre-allocates slice with capacity 2 (exact fit, no waste)
// - With hint=20: pre-allocates map with capacity 20 (saves array allocation + reallocation)
func newBindingsArrayHashmapWithSize(sizeHint int) bindingsArrayHashmap {
if sizeHint <= 0 {
// For unknown sizes, use default lazy allocation with dynamic growth.
return bindingsArrayHashmap{}
}
if sizeHint <= maxLinearScan {
// For small known sizes, pre-allocate slice with exact capacity to avoid growth overhead.
return bindingsArrayHashmap{
a: make([]bindingArrayKeyValue, 0, sizeHint),
}
}
// For larger sizes, pre-allocate map to avoid array allocation + transition cost.
return bindingsArrayHashmap{
m: make(map[ast.Var]bindingArrayKeyValue, sizeHint),
}
}
func (b *bindingsArrayHashmap) Put(key *ast.Term, value value) {
if b.m == nil {
// Check if key already exists and update value
if i := b.find(key); i >= 0 {
b.a[i].value = value
return
}
// Still room in slice mode (< maxLinearScan)
if b.n < maxLinearScan {
// Grow slice if needed using exponential growth strategy
if b.n == cap(b.a) {
newCap := cap(b.a) * 2
if newCap == 0 {
newCap = 2 // Start with 2 elements
}
if newCap > maxLinearScan {
newCap = maxLinearScan
}
newA := make([]bindingArrayKeyValue, b.n, newCap)
copy(newA, b.a)
b.a = newA
}
b.a = append(b.a, bindingArrayKeyValue{key, value})
b.n++
return
}
// Slice is full (reached maxLinearScan), transition to map mode.
b.m = make(map[ast.Var]bindingArrayKeyValue, maxLinearScan+1)
for _, kv := range b.a {
b.m[kv.key.Value.(ast.Var)] = bindingArrayKeyValue{kv.key, kv.value}
}
b.m[key.Value.(ast.Var)] = bindingArrayKeyValue{key, value}
// Clear slice to allow GC
b.a = nil
b.n = 0
return
}
b.m[key.Value.(ast.Var)] = bindingArrayKeyValue{key, value}
}
func (b *bindingsArrayHashmap) Get(key *ast.Term) (value, bool) {
if b.m == nil {
if i := b.find(key); i >= 0 {
return b.a[i].value, true
}
return value{}, false
}
v, ok := b.m[key.Value.(ast.Var)]
if ok {
return v.value, true
}
return value{}, false
}
func (b *bindingsArrayHashmap) Delete(key *ast.Term) {
if b.m == nil {
if i := b.find(key); i >= 0 {
n := b.n - 1
if i < n {
b.a[i] = b.a[n]
}
// Shrink slice to reflect deletion
b.a = b.a[:n]
b.n = n
}
return
}
delete(b.m, key.Value.(ast.Var))
}
func (b *bindingsArrayHashmap) Iter(f func(k *ast.Term, v value) bool) {
if b.m == nil {
if b.a != nil {
for i := range b.n {
if f(b.a[i].key, b.a[i].value) {
return
}
}
}
return
}
for _, v := range b.m {
if f(v.key, v.value) {
return
}
}
}
func (b *bindingsArrayHashmap) find(key *ast.Term) int {
if b.a == nil || b.n == 0 {
return -1
}
v := key.Value.(ast.Var)
for i := range b.n {
if b.a[i].key.Value.(ast.Var) == v {
return i
}
}
return -1
}
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"math/big"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
type bitsArity1 func(a *big.Int) (*big.Int, error)
type bitsArity2 func(a, b *big.Int) (*big.Int, error)
func bitsOr(a, b *big.Int) (*big.Int, error) {
return new(big.Int).Or(a, b), nil
}
func bitsAnd(a, b *big.Int) (*big.Int, error) {
return new(big.Int).And(a, b), nil
}
func bitsNegate(a *big.Int) (*big.Int, error) {
return new(big.Int).Not(a), nil
}
func bitsXOr(a, b *big.Int) (*big.Int, error) {
return new(big.Int).Xor(a, b), nil
}
func bitsShiftLeft(a, b *big.Int) (*big.Int, error) {
if b.Sign() == -1 {
return nil, builtins.NewOperandErr(2, "must be an unsigned integer number but got a negative integer")
}
shift := uint(b.Uint64())
return new(big.Int).Lsh(a, shift), nil
}
func bitsShiftRight(a, b *big.Int) (*big.Int, error) {
if b.Sign() == -1 {
return nil, builtins.NewOperandErr(2, "must be an unsigned integer number but got a negative integer")
}
shift := uint(b.Uint64())
return new(big.Int).Rsh(a, shift), nil
}
func builtinBitsArity1(fn bitsArity1) BuiltinFunc {
return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
i, err := builtins.BigIntOperand(operands[0].Value, 1)
if err != nil {
return err
}
iOut, err := fn(i)
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.IntToNumber(iOut)))
}
}
func builtinBitsArity2(fn bitsArity2) BuiltinFunc {
return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
i1, err := builtins.BigIntOperand(operands[0].Value, 1)
if err != nil {
return err
}
i2, err := builtins.BigIntOperand(operands[1].Value, 2)
if err != nil {
return err
}
iOut, err := fn(i1, i2)
if err != nil {
return err
}
return iter(ast.NewTerm(builtins.IntToNumber(iOut)))
}
}
func init() {
RegisterBuiltinFunc(ast.BitsOr.Name, builtinBitsArity2(bitsOr))
RegisterBuiltinFunc(ast.BitsAnd.Name, builtinBitsArity2(bitsAnd))
RegisterBuiltinFunc(ast.BitsNegate.Name, builtinBitsArity1(bitsNegate))
RegisterBuiltinFunc(ast.BitsXOr.Name, builtinBitsArity2(bitsXOr))
RegisterBuiltinFunc(ast.BitsShiftLeft.Name, builtinBitsArity2(bitsShiftLeft))
RegisterBuiltinFunc(ast.BitsShiftRight.Name, builtinBitsArity2(bitsShiftRight))
}
+224
View File
@@ -0,0 +1,224 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"context"
"encoding/binary"
"fmt"
"io"
"math/rand"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/metrics"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/topdown/cache"
"github.com/open-policy-agent/opa/v1/topdown/print"
"github.com/open-policy-agent/opa/v1/tracing"
)
type (
// Deprecated: Functional-style builtins are deprecated. Use BuiltinFunc instead.
FunctionalBuiltin1 func(op1 ast.Value) (output ast.Value, err error)
// Deprecated: Functional-style builtins are deprecated. Use BuiltinFunc instead.
FunctionalBuiltin2 func(op1, op2 ast.Value) (output ast.Value, err error)
// Deprecated: Functional-style builtins are deprecated. Use BuiltinFunc instead.
FunctionalBuiltin3 func(op1, op2, op3 ast.Value) (output ast.Value, err error)
// Deprecated: Functional-style builtins are deprecated. Use BuiltinFunc instead.
FunctionalBuiltin4 func(op1, op2, op3, op4 ast.Value) (output ast.Value, err error)
// BuiltinContext contains context from the evaluator that may be used by
// built-in functions.
BuiltinContext struct {
Context context.Context // request context that was passed when query started
Metrics metrics.Metrics // metrics registry for recording built-in specific metrics
Seed io.Reader // randomization source
Time *ast.Term // wall clock time
Cancel Cancel // atomic value that signals evaluation to halt
Runtime *ast.Term // runtime information on the OPA instance
Cache builtins.Cache // built-in function state cache
InterQueryBuiltinCache cache.InterQueryCache // cross-query built-in function state cache
InterQueryBuiltinValueCache cache.InterQueryValueCache // cross-query built-in function state value cache. this cache is useful for scenarios where the entry size cannot be calculated
NDBuiltinCache builtins.NDBCache // cache for non-deterministic built-in state
Location *ast.Location // location of built-in call
Tracers []Tracer // Deprecated: Use QueryTracers instead
QueryTracers []QueryTracer // tracer objects for trace() built-in function
TraceEnabled bool // indicates whether tracing is enabled for the evaluation
QueryID uint64 // identifies query being evaluated
ParentID uint64 // identifies parent of query being evaluated
PrintHook print.Hook // provides callback function to use for printing
RoundTripper CustomizeRoundTripper // customize transport to use for HTTP requests
DistributedTracingOpts tracing.Options // options to be used by distributed tracing.
rand *rand.Rand // randomization source for non-security-sensitive operations
Capabilities *ast.Capabilities
}
// BuiltinFunc defines an interface for implementing built-in functions.
// The built-in function is called with the plugged operands from the call
// (including the output operands.) The implementation should evaluate the
// operands and invoke the iterator for each successful/defined output
// value.
BuiltinFunc func(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error
)
// Rand returns a random number generator based on the Seed for this built-in
// context. The random number will be re-used across multiple calls to this
// function. If a random number generator cannot be created, an error is
// returned.
func (bctx *BuiltinContext) Rand() (*rand.Rand, error) {
if bctx.rand != nil {
return bctx.rand, nil
}
seed, err := readInt64(bctx.Seed)
if err != nil {
return nil, err
}
bctx.rand = rand.New(rand.NewSource(seed))
return bctx.rand, nil
}
// RegisterBuiltinFunc adds a new built-in function to the evaluation engine.
func RegisterBuiltinFunc(name string, f BuiltinFunc) {
builtinFunctions[name] = builtinErrorWrapper(name, f)
}
// Deprecated: Functional-style builtins are deprecated. Use RegisterBuiltinFunc instead.
func RegisterFunctionalBuiltin1(name string, fun FunctionalBuiltin1) {
builtinFunctions[name] = functionalWrapper1(name, fun)
}
// Deprecated: Functional-style builtins are deprecated. Use RegisterBuiltinFunc instead.
func RegisterFunctionalBuiltin2(name string, fun FunctionalBuiltin2) {
builtinFunctions[name] = functionalWrapper2(name, fun)
}
// Deprecated: Functional-style builtins are deprecated. Use RegisterBuiltinFunc instead.
func RegisterFunctionalBuiltin3(name string, fun FunctionalBuiltin3) {
builtinFunctions[name] = functionalWrapper3(name, fun)
}
// Deprecated: Functional-style builtins are deprecated. Use RegisterBuiltinFunc instead.
func RegisterFunctionalBuiltin4(name string, fun FunctionalBuiltin4) {
builtinFunctions[name] = functionalWrapper4(name, fun)
}
// GetBuiltin returns a built-in function implementation, nil if no built-in found.
func GetBuiltin(name string) BuiltinFunc {
return builtinFunctions[name]
}
// Deprecated: The BuiltinEmpty type is no longer needed. Use nil return values instead.
type BuiltinEmpty struct{}
func (BuiltinEmpty) Error() string {
return "<empty>"
}
var builtinFunctions = map[string]BuiltinFunc{}
func builtinErrorWrapper(name string, fn BuiltinFunc) BuiltinFunc {
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
err := fn(bctx, args, iter)
if err == nil {
return nil
}
return handleBuiltinErr(name, bctx.Location, err)
}
}
func functionalWrapper1(name string, fn FunctionalBuiltin1) BuiltinFunc {
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
return handleBuiltinErr(name, bctx.Location, err)
}
}
func functionalWrapper2(name string, fn FunctionalBuiltin2) BuiltinFunc {
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value, args[1].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
return handleBuiltinErr(name, bctx.Location, err)
}
}
func functionalWrapper3(name string, fn FunctionalBuiltin3) BuiltinFunc {
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value, args[1].Value, args[2].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
return handleBuiltinErr(name, bctx.Location, err)
}
}
func functionalWrapper4(name string, fn FunctionalBuiltin4) BuiltinFunc {
return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
result, err := fn(args[0].Value, args[1].Value, args[2].Value, args[3].Value)
if err == nil {
return iter(ast.NewTerm(result))
}
if _, empty := err.(BuiltinEmpty); empty {
return nil
}
return handleBuiltinErr(name, bctx.Location, err)
}
}
func handleBuiltinErr(name string, loc *ast.Location, err error) error {
switch err := err.(type) {
case BuiltinEmpty:
return nil
case *Error, Halt:
return err
case builtins.ErrOperand:
e := &Error{
Code: TypeErr,
Message: fmt.Sprintf("%v: %v", name, err.Error()),
Location: loc,
}
return e.Wrap(err)
default:
e := &Error{
Code: BuiltinErr,
Message: fmt.Sprintf("%v: %v", name, err.Error()),
Location: loc,
}
return e.Wrap(err)
}
}
func readInt64(r io.Reader) (int64, error) {
bs := make([]byte, 8)
n, err := io.ReadFull(r, bs)
if n != len(bs) || err != nil {
return 0, err
}
return int64(binary.BigEndian.Uint64(bs)), nil
}
// Used to get older-style (ast.Term, error) tuples out of newer functions.
func getResult(fn BuiltinFunc, operands ...*ast.Term) (*ast.Term, error) {
var result *ast.Term
extractionFn := func(r *ast.Term) error {
result = r
return nil
}
err := fn(BuiltinContext{}, operands, extractionFn)
if err != nil {
return nil, err
}
return result, nil
}
@@ -0,0 +1,346 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package builtins contains utilities for implementing built-in functions.
package builtins
import (
"encoding/json"
"errors"
"fmt"
"math/big"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
// Cache defines the built-in cache used by the top-down evaluation. The keys
// must be comparable and should not be of type string.
type Cache map[any]any
// Put updates the cache for the named built-in.
func (c Cache) Put(k, v any) {
c[k] = v
}
// Get returns the cached value for k.
func (c Cache) Get(k any) (any, bool) {
v, ok := c[k]
return v, ok
}
// We use an ast.Object for the cached keys/values because a naive
// map[ast.Value]ast.Value will not correctly detect value equality of
// the member keys.
type NDBCache map[string]ast.Object
func (c NDBCache) AsValue() ast.Value {
out := ast.NewObject()
for bname, obj := range c {
out.Insert(ast.InternedTerm(bname), ast.NewTerm(obj))
}
return out
}
// Put updates the cache for the named built-in.
// Automatically creates the 2-level hierarchy as needed.
func (c NDBCache) Put(name string, k, v ast.Value) {
if _, ok := c[name]; !ok {
c[name] = ast.NewObject()
}
c[name].Insert(ast.NewTerm(k), ast.NewTerm(v))
}
// Get returns the cached value for k for the named builtin.
func (c NDBCache) Get(name string, k ast.Value) (ast.Value, bool) {
if m, ok := c[name]; ok {
v := m.Get(ast.NewTerm(k))
if v != nil {
return v.Value, true
}
return nil, false
}
return nil, false
}
// Convenience functions for serializing the data structure.
func (c NDBCache) MarshalJSON() ([]byte, error) {
v, err := ast.JSON(c.AsValue())
if err != nil {
return nil, err
}
return json.Marshal(v)
}
func (c *NDBCache) UnmarshalJSON(data []byte) error {
out := map[string]ast.Object{}
var incoming any
// Note: We use util.Unmarshal instead of json.Unmarshal to get
// correct deserialization of number types.
err := util.Unmarshal(data, &incoming)
if err != nil {
return err
}
// Convert interface types back into ast.Value types.
nestedObject, err := ast.InterfaceToValue(incoming)
if err != nil {
return err
}
// Reconstruct NDBCache from nested ast.Object structure.
if source, ok := nestedObject.(ast.Object); ok {
err = source.Iter(func(k, v *ast.Term) error {
if obj, ok := v.Value.(ast.Object); ok {
out[string(k.Value.(ast.String))] = obj
return nil
}
return errors.New("expected Object, got other Value type in conversion")
})
if err != nil {
return err
}
}
*c = out
return nil
}
// ErrOperand represents an invalid operand has been passed to a built-in
// function. Built-ins should return ErrOperand to indicate a type error has
// occurred.
type ErrOperand string
func (err ErrOperand) Error() string {
return string(err)
}
// NewOperandErr returns a generic operand error.
func NewOperandErr(pos int, f string, a ...any) error {
f = fmt.Sprintf("operand %v ", pos) + f
return ErrOperand(fmt.Sprintf(f, a...))
}
// NewOperandTypeErr returns an operand error indicating the operand's type was wrong.
func NewOperandTypeErr(pos int, got ast.Value, expected ...string) error {
if len(expected) == 1 {
return NewOperandErr(pos, "must be %v but got %v", expected[0], ast.ValueName(got))
}
return NewOperandErr(pos, "must be one of {%v} but got %v", strings.Join(expected, ", "), ast.ValueName(got))
}
// NewOperandElementErr returns an operand error indicating an element in the
// composite operand was wrong.
func NewOperandElementErr(pos int, composite ast.Value, got ast.Value, expected ...string) error {
tpe := ast.ValueName(composite)
if len(expected) == 1 {
return NewOperandErr(pos, "must be %v of %vs but got %v containing %v", tpe, expected[0], tpe, ast.ValueName(got))
}
return NewOperandErr(pos, "must be %v of (any of) {%v} but got %v containing %v", tpe, strings.Join(expected, ", "), tpe, ast.ValueName(got))
}
// NewOperandEnumErr returns an operand error indicating a value was wrong.
func NewOperandEnumErr(pos int, expected ...string) error {
if len(expected) == 1 {
return NewOperandErr(pos, "must be %v", expected[0])
}
return NewOperandErr(pos, "must be one of {%v}", strings.Join(expected, ", "))
}
// IntOperand converts x to an int. If the cast fails, a descriptive error is
// returned.
func IntOperand(x ast.Value, pos int) (int, error) {
n, ok := x.(ast.Number)
if !ok {
return 0, NewOperandTypeErr(pos, x, "number")
}
i, ok := n.Int()
if !ok {
return 0, NewOperandErr(pos, "must be integer number but got floating-point number")
}
return i, nil
}
// BigIntOperand converts x to a big int. If the cast fails, a descriptive error
// is returned.
func BigIntOperand(x ast.Value, pos int) (*big.Int, error) {
n, err := NumberOperand(x, 1)
if err != nil {
return nil, NewOperandTypeErr(pos, x, "integer")
}
bi, err := NumberToInt(n)
if err != nil {
return nil, NewOperandErr(pos, "must be integer number but got floating-point number")
}
return bi, nil
}
// NumberOperand converts x to a number. If the cast fails, a descriptive error is
// returned.
func NumberOperand(x ast.Value, pos int) (ast.Number, error) {
n, ok := x.(ast.Number)
if !ok {
return ast.Number(""), NewOperandTypeErr(pos, x, "number")
}
return n, nil
}
// SetOperand converts x to a set. If the cast fails, a descriptive error is
// returned.
func SetOperand(x ast.Value, pos int) (ast.Set, error) {
s, ok := x.(ast.Set)
if !ok {
return nil, NewOperandTypeErr(pos, x, "set")
}
return s, nil
}
// StringOperand returns x as [ast.String], or a descriptive error if the conversion fails.
func StringOperand(x ast.Value, pos int) (ast.String, error) {
s, ok := x.(ast.String)
if !ok {
return ast.String(""), NewOperandTypeErr(pos, x, "string")
}
return s, nil
}
// StringOperandByteSlice returns x a []byte, assuming x is [ast.String], or a descriptive error
// if that is not the case. The returned byte slice points directly at the underlying array backing
// the string, and should not be modified.
func StringOperandByteSlice(x ast.Value, pos int) ([]byte, error) {
s, err := StringOperand(x, pos)
if err != nil {
return nil, err
}
return util.StringToByteSlice(string(s)), nil
}
// ObjectOperand converts x to an object. If the cast fails, a descriptive
// error is returned.
func ObjectOperand(x ast.Value, pos int) (ast.Object, error) {
o, ok := x.(ast.Object)
if !ok {
return nil, NewOperandTypeErr(pos, x, "object")
}
return o, nil
}
// ArrayOperand converts x to an array. If the cast fails, a descriptive
// error is returned.
func ArrayOperand(x ast.Value, pos int) (*ast.Array, error) {
a, ok := x.(*ast.Array)
if !ok {
return nil, NewOperandTypeErr(pos, x, "array")
}
return a, nil
}
// NumberToFloat converts n to a big float.
func NumberToFloat(n ast.Number) *big.Float {
return NumberToFloatInto(nil, n)
}
// NumberToFloatInto converts n to a big float, storing it in dst when provided.
func NumberToFloatInto(dst *big.Float, n ast.Number) *big.Float {
if dst == nil {
dst = new(big.Float)
}
if _, ok := dst.SetString(string(n)); !ok {
panic("illegal value")
}
return dst
}
// FloatToNumber converts f to a number.
func FloatToNumber(f *big.Float) ast.Number {
var format byte = 'g'
if f.IsInt() {
format = 'f'
}
return ast.Number(f.Text(format, -1))
}
// NumberToInt converts n to a big int.
// If n cannot be converted to an big int, an error is returned.
func NumberToInt(n ast.Number) (*big.Int, error) {
f := NumberToFloat(n)
r, accuracy := f.Int(nil)
if accuracy != big.Exact {
return nil, errors.New("illegal value")
}
return r, nil
}
// IntToNumber converts i to a number.
func IntToNumber(i *big.Int) ast.Number {
return ast.Number(i.String())
}
// StringSliceOperand converts x to a []string. If the cast fails, a descriptive error is
// returned.
func StringSliceOperand(a ast.Value, pos int) ([]string, error) {
type iterable interface {
Iter(func(*ast.Term) error) error
Len() int
}
strs, ok := a.(iterable)
if !ok {
return nil, NewOperandTypeErr(pos, a, "array", "set")
}
var outStrs = make([]string, 0, strs.Len())
if err := strs.Iter(func(x *ast.Term) error {
s, ok := x.Value.(ast.String)
if !ok {
return NewOperandElementErr(pos, a, x.Value, "string")
}
outStrs = append(outStrs, string(s))
return nil
}); err != nil {
return nil, err
}
return outStrs, nil
}
// RuneSliceOperand converts x to a []rune. If the cast fails, a descriptive error is
// returned.
func RuneSliceOperand(x ast.Value, pos int) ([]rune, error) {
a, err := ArrayOperand(x, pos)
if err != nil {
return nil, err
}
var f = make([]rune, a.Len())
for k := range a.Len() {
b := a.Elem(k)
c, ok := b.Value.(ast.String)
if !ok {
return nil, NewOperandElementErr(pos, x, b.Value, "string")
}
d := []rune(string(c))
if len(d) != 1 {
return nil, NewOperandElementErr(pos, x, b.Value, "rune")
}
f[k] = d[0]
}
return f, nil
}
+363
View File
@@ -0,0 +1,363 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"slices"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
// VirtualCache defines the interface for a cache that stores the results of
// evaluated virtual documents (rules).
// The cache is a stack of frames, where each frame is a mapping from references
// to values.
type VirtualCache interface {
// Push pushes a new, empty frame of value mappings onto the stack.
Push()
// Pop pops the top frame of value mappings from the stack, removing all associated entries.
Pop()
// Get returns the value associated with the given reference. The second return value
// indicates whether the reference has a recorded 'undefined' result.
Get(ref ast.Ref) (*ast.Term, bool)
// Put associates the given reference with the given value. If the value is nil, the reference
// is marked as having an 'undefined' result.
Put(ref ast.Ref, value *ast.Term)
// Keys returns the set of keys that have been cached for the active frame.
Keys() []ast.Ref
}
// BaseCache defines the interface for a cache that stores cached base documents, i.e. data.
type BaseCache interface {
Get(ast.Ref) ast.Value
Put(ast.Ref, ast.Value)
}
type virtualCache struct {
stack []*virtualCacheElem
}
type virtualCacheElem struct {
value *ast.Term
children *util.HasherMap[*ast.Term, *virtualCacheElem]
undefined bool
}
func NewVirtualCache() VirtualCache {
cache := &virtualCache{}
cache.Push()
return cache
}
func (c *virtualCache) Push() {
c.stack = append(c.stack, newVirtualCacheElem())
}
func (c *virtualCache) Pop() {
c.stack = c.stack[:len(c.stack)-1]
}
// Returns the resolved value of the AST term and a flag indicating if the value
// should be interpretted as undefined:
//
// nil, true indicates the ref is undefined
// ast.Term, false indicates the ref is defined
// nil, false indicates the ref has not been cached
// ast.Term, true is impossible
func (c *virtualCache) Get(ref ast.Ref) (*ast.Term, bool) {
node := c.stack[len(c.stack)-1]
for i := range ref {
x, ok := node.children.Get(ref[i])
if !ok {
return nil, false
}
node = x
}
if node.undefined {
return nil, true
}
return node.value, false
}
// If value is a nil pointer, set the 'undefined' flag on the cache element to
// indicate that the Ref has resolved to undefined.
func (c *virtualCache) Put(ref ast.Ref, value *ast.Term) {
node := c.stack[len(c.stack)-1]
for i := range ref {
x, ok := node.children.Get(ref[i])
if ok {
node = x
} else {
next := newVirtualCacheElem()
node.children.Put(ref[i], next)
node = next
}
}
if value != nil {
node.value = value
} else {
node.undefined = true
}
}
func (c *virtualCache) Keys() []ast.Ref {
node := c.stack[len(c.stack)-1]
return keysRecursive(nil, node)
}
func keysRecursive(root ast.Ref, node *virtualCacheElem) []ast.Ref {
var keys []ast.Ref
node.children.Iter(func(k *ast.Term, v *virtualCacheElem) bool {
ref := root.Append(k)
if v.value != nil {
keys = append(keys, ref)
}
if v.children.Len() > 0 {
keys = append(keys, keysRecursive(ref, v)...)
}
return false
})
return keys
}
func newVirtualCacheElem() *virtualCacheElem {
return &virtualCacheElem{children: newVirtualCacheHashMap()}
}
func newVirtualCacheHashMap() *util.HasherMap[*ast.Term, *virtualCacheElem] {
return util.NewHasherMap[*ast.Term, *virtualCacheElem](ast.TermValueEqual)
}
// baseCache implements a trie structure to cache base documents read out of
// storage. Values inserted into the cache may contain other values that were
// previously inserted. In this case, the previous values are erased from the
// structure.
type baseCache struct {
root *baseCacheElem
}
func newBaseCache() *baseCache {
return &baseCache{
root: newBaseCacheElem(),
}
}
func (c *baseCache) Get(ref ast.Ref) ast.Value {
node := c.root
for i := range ref {
node = node.children[ref[i].Value]
if node == nil {
return nil
} else if node.value != nil {
if len(ref) == 1 && ast.IsScalar(node.value) {
// If the node is a scalar, return the value directly
// and avoid an allocation when calling Find.
return node.value
}
result, err := node.value.Find(ref[i+1:])
if err != nil {
return nil
}
return result
}
}
return nil
}
func (c *baseCache) Put(ref ast.Ref, value ast.Value) {
node := c.root
for i := range ref {
if child, ok := node.children[ref[i].Value]; ok {
node = child
} else {
child := newBaseCacheElem()
node.children[ref[i].Value] = child
node = child
}
}
node.set(value)
}
type baseCacheElem struct {
value ast.Value
children map[ast.Value]*baseCacheElem
}
func newBaseCacheElem() *baseCacheElem {
return &baseCacheElem{
children: map[ast.Value]*baseCacheElem{},
}
}
func (e *baseCacheElem) set(value ast.Value) {
e.value = value
e.children = map[ast.Value]*baseCacheElem{}
}
type refStack struct {
sl []refStackElem
}
type refStackElem struct {
refs []ast.Ref
}
func newRefStack() *refStack {
return &refStack{}
}
func (s *refStack) Push(refs []ast.Ref) {
s.sl = append(s.sl, refStackElem{refs: refs})
}
func (s *refStack) Pop() {
if s == nil {
return
}
s.sl = s.sl[:len(s.sl)-1]
}
func (s *refStack) Prefixed(ref ast.Ref) bool {
if s != nil {
for i := len(s.sl) - 1; i >= 0; i-- {
if slices.ContainsFunc(s.sl[i].refs, ref.HasPrefix) {
return true
}
}
}
return false
}
type comprehensionCache struct {
stack []map[*ast.Term]*comprehensionCacheElem
}
type comprehensionCacheElem struct {
value *ast.Term
children *util.HasherMap[*ast.Term, *comprehensionCacheElem]
}
func newComprehensionCache() *comprehensionCache {
cache := &comprehensionCache{}
cache.Push()
return cache
}
func (c *comprehensionCache) Push() {
c.stack = append(c.stack, map[*ast.Term]*comprehensionCacheElem{})
}
func (c *comprehensionCache) Pop() {
c.stack = c.stack[:len(c.stack)-1]
}
func (c *comprehensionCache) Elem(t *ast.Term) (*comprehensionCacheElem, bool) {
elem, ok := c.stack[len(c.stack)-1][t]
return elem, ok
}
func (c *comprehensionCache) Set(t *ast.Term, elem *comprehensionCacheElem) {
c.stack[len(c.stack)-1][t] = elem
}
func newComprehensionCacheElem() *comprehensionCacheElem {
return &comprehensionCacheElem{children: newComprehensionCacheHashMap()}
}
func (c *comprehensionCacheElem) Get(key []*ast.Term) *ast.Term {
node := c
for i := range key {
x, ok := node.children.Get(key[i])
if !ok {
return nil
}
node = x
}
return node.value
}
func (c *comprehensionCacheElem) Put(key []*ast.Term, value *ast.Term) {
node := c
for i := range key {
x, ok := node.children.Get(key[i])
if ok {
node = x
} else {
next := newComprehensionCacheElem()
node.children.Put(key[i], next)
node = next
}
}
node.value = value
}
func newComprehensionCacheHashMap() *util.HasherMap[*ast.Term, *comprehensionCacheElem] {
return util.NewHasherMap[*ast.Term, *comprehensionCacheElem](ast.TermValueEqual)
}
type functionMocksStack struct {
stack []*functionMocksElem
}
type functionMocksElem []frame
type frame map[string]*ast.Term
func newFunctionMocksStack() *functionMocksStack {
stack := &functionMocksStack{}
stack.Push()
return stack
}
func newFunctionMocksElem() *functionMocksElem {
return &functionMocksElem{}
}
func (s *functionMocksStack) Push() {
s.stack = append(s.stack, newFunctionMocksElem())
}
func (s *functionMocksStack) Pop() {
s.stack = s.stack[:len(s.stack)-1]
}
func (s *functionMocksStack) PopPairs() {
current := s.stack[len(s.stack)-1]
*current = (*current)[:len(*current)-1]
}
func (s *functionMocksStack) PutPairs(mocks [][2]*ast.Term) {
el := frame{}
for i := range mocks {
el[mocks[i][0].Value.String()] = mocks[i][1]
}
s.Put(el)
}
func (s *functionMocksStack) Put(el frame) {
current := s.stack[len(s.stack)-1]
*current = append(*current, el)
}
func (s *functionMocksStack) Get(f ast.Ref) (*ast.Term, bool) {
if s == nil {
return nil, false
}
current := *s.stack[len(s.stack)-1]
for i := len(current) - 1; i >= 0; i-- {
if r, ok := current[i][f.String()]; ok {
return r, true
}
}
return nil, false
}
@@ -0,0 +1,666 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package cache defines the inter-query cache interface that can cache data across queries
package cache
import (
"container/list"
"context"
"fmt"
"math"
"sync"
"time"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
const (
defaultInterQueryBuiltinValueCacheSize = int(0) // unlimited
defaultMaxSizeBytes = int64(0) // unlimited
defaultForcedEvictionThresholdPercentage = int64(100) // trigger at max_size_bytes
defaultStaleEntryEvictionPeriodSeconds = int64(0) // never
)
var interQueryBuiltinValueCacheDefaultConfigs = map[string]*NamedValueCacheConfig{}
func getDefaultInterQueryBuiltinValueCacheConfig(name string) *NamedValueCacheConfig {
return interQueryBuiltinValueCacheDefaultConfigs[name]
}
// RegisterDefaultInterQueryBuiltinValueCacheConfig registers a default configuration for the inter-query value cache;
// used when none has been explicitly configured.
// To disable a named cache when not configured, pass a config with the disabled value set to true.
func RegisterDefaultInterQueryBuiltinValueCacheConfig(name string, config *NamedValueCacheConfig) {
interQueryBuiltinValueCacheDefaultConfigs[name] = config
}
// Config represents the configuration for the inter-query builtin cache.
type Config struct {
InterQueryBuiltinCache InterQueryBuiltinCacheConfig `json:"inter_query_builtin_cache"`
InterQueryBuiltinValueCache InterQueryBuiltinValueCacheConfig `json:"inter_query_builtin_value_cache"`
}
// Clone creates a deep copy of Config.
func (c *Config) Clone() *Config {
if c == nil {
return nil
}
return &Config{
InterQueryBuiltinCache: *c.InterQueryBuiltinCache.Clone(),
InterQueryBuiltinValueCache: *c.InterQueryBuiltinValueCache.Clone(),
}
}
// NamedValueCacheConfig represents the configuration of a named cache that built-in functions can utilize.
// A default configuration to be used if not explicitly configured can be registered using RegisterDefaultInterQueryBuiltinValueCacheConfig.
type NamedValueCacheConfig struct {
MaxNumEntries *int `json:"max_num_entries,omitempty"`
Disabled *bool `json:"disabled,omitempty"`
}
// Clone creates a deep copy of NamedValueCacheConfig.
func (n *NamedValueCacheConfig) Clone() *NamedValueCacheConfig {
if n == nil {
return nil
}
clone := &NamedValueCacheConfig{}
if n.MaxNumEntries != nil {
maxEntries := *n.MaxNumEntries
clone.MaxNumEntries = &maxEntries
}
if n.Disabled != nil {
disabled := *n.Disabled
clone.Disabled = &disabled
}
return clone
}
// InterQueryBuiltinValueCacheConfig represents the configuration of the inter-query value cache that built-in functions can utilize.
// MaxNumEntries - max number of cache entries
type InterQueryBuiltinValueCacheConfig struct {
MaxNumEntries *int `json:"max_num_entries,omitempty"`
NamedCacheConfigs map[string]*NamedValueCacheConfig `json:"named,omitempty"`
}
// Clone creates a deep copy of InterQueryBuiltinValueCacheConfig.
func (i *InterQueryBuiltinValueCacheConfig) Clone() *InterQueryBuiltinValueCacheConfig {
if i == nil {
return nil
}
clone := &InterQueryBuiltinValueCacheConfig{}
if i.MaxNumEntries != nil {
maxEntries := *i.MaxNumEntries
clone.MaxNumEntries = &maxEntries
}
if i.NamedCacheConfigs != nil {
clone.NamedCacheConfigs = make(map[string]*NamedValueCacheConfig, len(i.NamedCacheConfigs))
for k, v := range i.NamedCacheConfigs {
clone.NamedCacheConfigs[k] = v.Clone()
}
}
return clone
}
// InterQueryBuiltinCacheConfig represents the configuration of the inter-query cache that built-in functions can utilize.
// MaxSizeBytes - max capacity of cache in bytes
// ForcedEvictionThresholdPercentage - capacity usage in percentage after which forced FIFO eviction starts
// StaleEntryEvictionPeriodSeconds - time period between end of previous and start of new stale entry eviction routine
type InterQueryBuiltinCacheConfig struct {
MaxSizeBytes *int64 `json:"max_size_bytes,omitempty"`
ForcedEvictionThresholdPercentage *int64 `json:"forced_eviction_threshold_percentage,omitempty"`
StaleEntryEvictionPeriodSeconds *int64 `json:"stale_entry_eviction_period_seconds,omitempty"`
}
// Clone creates a deep copy of InterQueryBuiltinCacheConfig.
func (i *InterQueryBuiltinCacheConfig) Clone() *InterQueryBuiltinCacheConfig {
if i == nil {
return nil
}
clone := &InterQueryBuiltinCacheConfig{}
if i.MaxSizeBytes != nil {
maxSize := *i.MaxSizeBytes
clone.MaxSizeBytes = &maxSize
}
if i.ForcedEvictionThresholdPercentage != nil {
threshold := *i.ForcedEvictionThresholdPercentage
clone.ForcedEvictionThresholdPercentage = &threshold
}
if i.StaleEntryEvictionPeriodSeconds != nil {
period := *i.StaleEntryEvictionPeriodSeconds
clone.StaleEntryEvictionPeriodSeconds = &period
}
return clone
}
// ParseCachingConfig returns the config for the inter-query cache.
func ParseCachingConfig(raw []byte) (*Config, error) {
if raw == nil {
maxSize := new(int64)
*maxSize = defaultMaxSizeBytes
threshold := new(int64)
*threshold = defaultForcedEvictionThresholdPercentage
period := new(int64)
*period = defaultStaleEntryEvictionPeriodSeconds
maxInterQueryBuiltinValueCacheSize := new(int)
*maxInterQueryBuiltinValueCacheSize = defaultInterQueryBuiltinValueCacheSize
return &Config{
InterQueryBuiltinCache: InterQueryBuiltinCacheConfig{
MaxSizeBytes: maxSize,
ForcedEvictionThresholdPercentage: threshold,
StaleEntryEvictionPeriodSeconds: period,
},
InterQueryBuiltinValueCache: InterQueryBuiltinValueCacheConfig{
MaxNumEntries: maxInterQueryBuiltinValueCacheSize,
},
}, nil
}
var config Config
if err := util.Unmarshal(raw, &config); err == nil {
if err = config.validateAndInjectDefaults(); err != nil {
return nil, err
}
} else {
return nil, err
}
return &config, nil
}
func (c *Config) validateAndInjectDefaults() error {
if c.InterQueryBuiltinCache.MaxSizeBytes == nil {
maxSize := new(int64)
*maxSize = defaultMaxSizeBytes
c.InterQueryBuiltinCache.MaxSizeBytes = maxSize
}
if c.InterQueryBuiltinCache.ForcedEvictionThresholdPercentage == nil {
threshold := new(int64)
*threshold = defaultForcedEvictionThresholdPercentage
c.InterQueryBuiltinCache.ForcedEvictionThresholdPercentage = threshold
} else {
threshold := *c.InterQueryBuiltinCache.ForcedEvictionThresholdPercentage
if threshold < 0 || threshold > 100 {
return fmt.Errorf("invalid forced_eviction_threshold_percentage %v", threshold)
}
}
if c.InterQueryBuiltinCache.StaleEntryEvictionPeriodSeconds == nil {
period := new(int64)
*period = defaultStaleEntryEvictionPeriodSeconds
c.InterQueryBuiltinCache.StaleEntryEvictionPeriodSeconds = period
} else {
period := *c.InterQueryBuiltinCache.StaleEntryEvictionPeriodSeconds
if period < 0 {
return fmt.Errorf("invalid stale_entry_eviction_period_seconds %v", period)
}
}
if c.InterQueryBuiltinValueCache.MaxNumEntries == nil {
maxSize := new(int)
*maxSize = defaultInterQueryBuiltinValueCacheSize
c.InterQueryBuiltinValueCache.MaxNumEntries = maxSize
} else {
numEntries := *c.InterQueryBuiltinValueCache.MaxNumEntries
if numEntries < 0 {
return fmt.Errorf("invalid max_num_entries %v", numEntries)
}
}
for name, namedConfig := range c.InterQueryBuiltinValueCache.NamedCacheConfigs {
if namedConfig == nil || (namedConfig.MaxNumEntries == nil && namedConfig.Disabled == nil) {
return fmt.Errorf("missing configuration for named cache %v", name)
}
if namedConfig.MaxNumEntries != nil {
numEntries := *namedConfig.MaxNumEntries
if numEntries < 0 {
return fmt.Errorf("invalid max_num_entries %v for named cache %v", numEntries, name)
}
}
}
return nil
}
// InterQueryCacheValue defines the interface for the data that the inter-query cache holds.
type InterQueryCacheValue interface {
SizeInBytes() int64
Clone() (InterQueryCacheValue, error)
}
// InterQueryCache defines the interface for the inter-query cache.
type InterQueryCache interface {
Get(key ast.Value) (value InterQueryCacheValue, found bool)
Insert(key ast.Value, value InterQueryCacheValue) int
InsertWithExpiry(key ast.Value, value InterQueryCacheValue, expiresAt time.Time) int
Delete(key ast.Value)
UpdateConfig(config *Config)
Clone(value InterQueryCacheValue) (InterQueryCacheValue, error)
}
// NewInterQueryCache returns a new inter-query cache.
// The cache uses a FIFO eviction policy when it reaches the forced eviction threshold.
// Parameters:
//
// config - to configure the InterQueryCache
func NewInterQueryCache(config *Config) InterQueryCache {
return newCache(config)
}
// NewInterQueryCacheWithContext returns a new inter-query cache with context.
// The cache uses a combination of FIFO eviction policy when it reaches the forced eviction threshold
// and a periodic cleanup routine to remove stale entries that exceed their expiration time, if specified.
// If configured with a zero stale_entry_eviction_period_seconds value, the stale entry cleanup routine is disabled.
//
// Parameters:
//
// ctx - used to control lifecycle of the stale entry cleanup routine
// config - to configure the InterQueryCache
func NewInterQueryCacheWithContext(ctx context.Context, config *Config) InterQueryCache {
iqCache := newCache(config)
if iqCache.staleEntryEvictionTimePeriodSeconds() > 0 {
go func() {
cleanupTicker := time.NewTicker(time.Duration(iqCache.staleEntryEvictionTimePeriodSeconds()) * time.Second)
for {
select {
case <-cleanupTicker.C:
// NOTE: We stop the ticker and create a new one here to ensure that applications
// get _at least_ staleEntryEvictionTimePeriodSeconds with the cache unlocked;
// see https://github.com/open-policy-agent/opa/pull/7188/files#r1855342998
cleanupTicker.Stop()
iqCache.cleanStaleValues()
cleanupTicker = time.NewTicker(time.Duration(iqCache.staleEntryEvictionTimePeriodSeconds()) * time.Second)
case <-ctx.Done():
cleanupTicker.Stop()
return
}
}
}()
}
return iqCache
}
type cacheItem struct {
value InterQueryCacheValue
expiresAt time.Time
keyElement *list.Element
}
type cache struct {
items map[string]cacheItem
usage int64
config *Config
l *list.List
mtx sync.Mutex
}
func newCache(config *Config) *cache {
return &cache{
items: map[string]cacheItem{},
usage: 0,
config: config,
l: list.New(),
}
}
// InsertWithExpiry inserts a key k into the cache with value v with an expiration time expiresAt.
// A zero time value for expiresAt indicates no expiry
func (c *cache) InsertWithExpiry(k ast.Value, v InterQueryCacheValue, expiresAt time.Time) (dropped int) {
c.mtx.Lock()
defer c.mtx.Unlock()
return c.unsafeInsert(k, v, expiresAt)
}
// Insert inserts a key k into the cache with value v with no expiration time.
func (c *cache) Insert(k ast.Value, v InterQueryCacheValue) (dropped int) {
return c.InsertWithExpiry(k, v, time.Time{})
}
// Get returns the value in the cache for k.
func (c *cache) Get(k ast.Value) (InterQueryCacheValue, bool) {
c.mtx.Lock()
defer c.mtx.Unlock()
cacheItem, ok := c.unsafeGet(k)
if ok {
return cacheItem.value, true
}
return nil, false
}
// Delete deletes the value in the cache for k.
func (c *cache) Delete(k ast.Value) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.unsafeDelete(k)
}
func (c *cache) UpdateConfig(config *Config) {
if config == nil {
return
}
c.mtx.Lock()
defer c.mtx.Unlock()
c.config = config
}
func (c *cache) Clone(value InterQueryCacheValue) (InterQueryCacheValue, error) {
c.mtx.Lock()
defer c.mtx.Unlock()
return c.unsafeClone(value)
}
func (c *cache) unsafeInsert(k ast.Value, v InterQueryCacheValue, expiresAt time.Time) (dropped int) {
size := v.SizeInBytes()
limit := int64(math.Ceil(float64(c.forcedEvictionThresholdPercentage())/100.0) * (float64(c.maxSizeBytes())))
if limit > 0 {
if size > limit {
dropped++
return dropped
}
for key := c.l.Front(); key != nil && (c.usage+size > limit); key = c.l.Front() {
dropKey := key.Value.(ast.Value)
c.unsafeDelete(dropKey)
dropped++
}
}
// By deleting the old value, if it exists, we ensure the usage variable stays correct
c.unsafeDelete(k)
c.items[k.String()] = cacheItem{
value: v,
expiresAt: expiresAt,
keyElement: c.l.PushBack(k),
}
c.usage += size
return dropped
}
func (c *cache) unsafeGet(k ast.Value) (cacheItem, bool) {
value, ok := c.items[k.String()]
return value, ok
}
func (c *cache) unsafeDelete(k ast.Value) {
cacheItem, ok := c.unsafeGet(k)
if !ok {
return
}
c.usage -= cacheItem.value.SizeInBytes()
delete(c.items, k.String())
c.l.Remove(cacheItem.keyElement)
}
func (*cache) unsafeClone(value InterQueryCacheValue) (InterQueryCacheValue, error) {
return value.Clone()
}
func (c *cache) maxSizeBytes() int64 {
if c.config == nil {
return defaultMaxSizeBytes
}
return *c.config.InterQueryBuiltinCache.MaxSizeBytes
}
func (c *cache) forcedEvictionThresholdPercentage() int64 {
if c.config == nil {
return defaultForcedEvictionThresholdPercentage
}
return *c.config.InterQueryBuiltinCache.ForcedEvictionThresholdPercentage
}
func (c *cache) staleEntryEvictionTimePeriodSeconds() int64 {
if c.config == nil {
return defaultStaleEntryEvictionPeriodSeconds
}
return *c.config.InterQueryBuiltinCache.StaleEntryEvictionPeriodSeconds
}
func (c *cache) cleanStaleValues() (dropped int) {
c.mtx.Lock()
defer c.mtx.Unlock()
for key := c.l.Front(); key != nil; {
nextKey := key.Next()
// if expiresAt is zero, the item doesn't have an expiry
if ea := c.items[(key.Value.(ast.Value)).String()].expiresAt; !ea.IsZero() && ea.Before(time.Now()) {
c.unsafeDelete(key.Value.(ast.Value))
dropped++
}
key = nextKey
}
return dropped
}
type InterQueryValueCacheBucket interface {
Get(key ast.Value) (value any, found bool)
Insert(key ast.Value, value any) int
Delete(key ast.Value)
}
type interQueryValueCacheBucket struct {
items util.HasherMap[ast.Value, any]
config *NamedValueCacheConfig
mtx sync.RWMutex
}
func newItemsMap() *util.HasherMap[ast.Value, any] {
return util.NewHasherMap[ast.Value, any](ast.ValueEqual)
}
func (c *interQueryValueCacheBucket) Get(k ast.Value) (any, bool) {
c.mtx.RLock()
defer c.mtx.RUnlock()
return c.items.Get(k)
}
func (c *interQueryValueCacheBucket) Insert(k ast.Value, v any) (dropped int) {
c.mtx.Lock()
defer c.mtx.Unlock()
maxEntries := c.maxNumEntries()
if maxEntries > 0 {
l := c.items.Len()
if l >= maxEntries {
itemsToRemove := l - maxEntries + 1
// Delete a (semi-)random key to make room for the new one.
c.items.Iter(func(k ast.Value, _ any) bool {
c.items.Delete(k)
dropped++
return itemsToRemove == dropped
})
}
}
c.items.Put(k, v)
return dropped
}
func (c *interQueryValueCacheBucket) Delete(k ast.Value) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.items.Delete(k)
}
func (c *interQueryValueCacheBucket) updateConfig(config *NamedValueCacheConfig) {
if config == nil {
return
}
c.mtx.Lock()
defer c.mtx.Unlock()
c.config = config
}
func (c *interQueryValueCacheBucket) maxNumEntries() int {
if c.config == nil {
return defaultInterQueryBuiltinValueCacheSize
}
return *c.config.MaxNumEntries
}
type InterQueryValueCache interface {
InterQueryValueCacheBucket
GetCache(name string) InterQueryValueCacheBucket
UpdateConfig(config *Config)
}
func NewInterQueryValueCache(_ context.Context, config *Config) InterQueryValueCache {
var c *InterQueryBuiltinValueCacheConfig
var nc *NamedValueCacheConfig
if config != nil {
c = &config.InterQueryBuiltinValueCache
// NOTE: This is a side-effect of reusing the interQueryValueCacheBucket as the global cache.
// It's a hidden implementation detail that we can clean up in the future when revisiting the named caches
// to automatically apply them to any built-in instead of the global cache.
nc = &NamedValueCacheConfig{
MaxNumEntries: c.MaxNumEntries,
}
}
return &interQueryBuiltinValueCache{
globalCache: interQueryValueCacheBucket{
items: *newItemsMap(),
config: nc,
},
namedCaches: map[string]*interQueryValueCacheBucket{},
config: c,
}
}
type interQueryBuiltinValueCache struct {
globalCache interQueryValueCacheBucket
namedCachesLock sync.RWMutex
namedCaches map[string]*interQueryValueCacheBucket
config *InterQueryBuiltinValueCacheConfig
}
func (c *interQueryBuiltinValueCache) Get(k ast.Value) (any, bool) {
if c == nil {
return nil, false
}
return c.globalCache.Get(k)
}
func (c *interQueryBuiltinValueCache) Insert(k ast.Value, v any) int {
if c == nil {
return 0
}
return c.globalCache.Insert(k, v)
}
func (c *interQueryBuiltinValueCache) Delete(k ast.Value) {
if c == nil {
return
}
c.globalCache.Delete(k)
}
func (c *interQueryBuiltinValueCache) GetCache(name string) InterQueryValueCacheBucket {
if c == nil {
return nil
}
if c.namedCaches == nil {
return nil
}
c.namedCachesLock.RLock()
nc, ok := c.namedCaches[name]
c.namedCachesLock.RUnlock()
if !ok {
c.namedCachesLock.Lock()
defer c.namedCachesLock.Unlock()
if nc, ok := c.namedCaches[name]; ok {
// Some other goroutine has created the cache while we were waiting for the lock.
return nc
}
var config *NamedValueCacheConfig
if c.config != nil {
config = c.config.NamedCacheConfigs[name]
if config == nil {
config = getDefaultInterQueryBuiltinValueCacheConfig(name)
}
}
if config == nil {
// No config, cache disabled.
return nil
}
if config.Disabled != nil && *config.Disabled {
return nil
}
nc = &interQueryValueCacheBucket{
items: *newItemsMap(),
config: config,
}
c.namedCaches[name] = nc
}
return nc
}
func (c *interQueryBuiltinValueCache) UpdateConfig(config *Config) {
if c == nil {
return
}
if config == nil {
c.globalCache.updateConfig(nil)
} else {
c.globalCache.updateConfig(&NamedValueCacheConfig{
MaxNumEntries: config.InterQueryBuiltinValueCache.MaxNumEntries,
})
}
c.namedCachesLock.Lock()
defer c.namedCachesLock.Unlock()
c.config = &config.InterQueryBuiltinValueCache
for name, nc := range c.namedCaches {
// For each named cache: if it has a config, update it; if no config, remove it.
namedConfig := c.config.NamedCacheConfigs[name]
if namedConfig == nil {
namedConfig = getDefaultInterQueryBuiltinValueCacheConfig(name)
}
if namedConfig == nil {
delete(c.namedCaches, name)
} else {
nc.updateConfig(namedConfig)
}
}
}
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"sync/atomic"
)
// Cancel defines the interface for cancelling topdown queries. Cancel
// operations are thread-safe and idempotent.
type Cancel interface {
Cancel()
Cancelled() bool
}
type cancel struct {
flag int32
}
// NewCancel returns a new Cancel object.
func NewCancel() Cancel {
return &cancel{}
}
func (c *cancel) Cancel() {
atomic.StoreInt32(&c.flag, 1)
}
func (c *cancel) Cancelled() bool {
return atomic.LoadInt32(&c.flag) != 0
}
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"strconv"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinToNumber(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch a := operands[0].Value.(type) {
case ast.Null:
return iter(ast.InternedTerm(0))
case ast.Boolean:
if a {
return iter(ast.InternedTerm(1))
}
return iter(ast.InternedTerm(0))
case ast.Number:
return iter(operands[0])
case ast.String:
strValue := string(a)
if it := ast.InternedIntNumberTermFromString(strValue); it != nil {
return iter(it)
}
trimmedVal := strings.TrimLeft(strValue, "+-")
lowerCaseVal := strings.ToLower(trimmedVal)
if lowerCaseVal == "inf" || lowerCaseVal == "infinity" || lowerCaseVal == "nan" {
return builtins.NewOperandTypeErr(1, operands[0].Value, "valid number string")
}
_, err := strconv.ParseFloat(strValue, 64)
if err != nil {
return err
}
return iter(ast.NewTerm(ast.Number(a)))
}
return builtins.NewOperandTypeErr(1, operands[0].Value, "null", "boolean", "number", "string")
}
// Deprecated: deprecated in v0.13.0.
func builtinToArray(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case *ast.Array:
return iter(ast.NewTerm(val))
case ast.Set:
arr := make([]*ast.Term, val.Len())
i := 0
val.Foreach(func(term *ast.Term) {
arr[i] = term
i++
})
return iter(ast.NewTerm(ast.NewArray(arr...)))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "array", "set")
}
}
// Deprecated: deprecated in v0.13.0.
func builtinToSet(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case *ast.Array:
s := ast.NewSet()
val.Foreach(func(v *ast.Term) {
s.Add(v)
})
return iter(ast.NewTerm(s))
case ast.Set:
return iter(ast.NewTerm(val))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "array", "set")
}
}
// Deprecated: deprecated in v0.13.0.
func builtinToString(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.String:
return iter(ast.NewTerm(val))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "string")
}
}
// Deprecated: deprecated in v0.13.0.
func builtinToBoolean(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.Boolean:
return iter(ast.NewTerm(val))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "boolean")
}
}
// Deprecated: deprecated in v0.13.0.
func builtinToNull(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.Null:
return iter(ast.NewTerm(val))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "null")
}
}
// Deprecated: deprecated in v0.13.0.
func builtinToObject(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch val := operands[0].Value.(type) {
case ast.Object:
return iter(ast.NewTerm(val))
default:
return builtins.NewOperandTypeErr(1, operands[0].Value, "object")
}
}
func init() {
RegisterBuiltinFunc(ast.ToNumber.Name, builtinToNumber)
RegisterBuiltinFunc(ast.CastArray.Name, builtinToArray)
RegisterBuiltinFunc(ast.CastSet.Name, builtinToSet)
RegisterBuiltinFunc(ast.CastString.Name, builtinToString)
RegisterBuiltinFunc(ast.CastBoolean.Name, builtinToBoolean)
RegisterBuiltinFunc(ast.CastNull.Name, builtinToNull)
RegisterBuiltinFunc(ast.CastObject.Name, builtinToObject)
}
+419
View File
@@ -0,0 +1,419 @@
package topdown
import (
"bytes"
"errors"
"fmt"
"math/big"
"net"
"slices"
"sort"
cidrMerge "github.com/open-policy-agent/opa/internal/cidr/merge"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func getNetFromOperand(v ast.Value) (*net.IPNet, error) {
subnetStringA, err := builtins.StringOperand(v, 1)
if err != nil {
return nil, err
}
_, cidrnet, err := net.ParseCIDR(string(subnetStringA))
if err != nil {
return nil, err
}
return cidrnet, nil
}
func getLastIP(cidr *net.IPNet) (net.IP, error) {
prefixLen, bits := cidr.Mask.Size()
if prefixLen == 0 && bits == 0 {
// non-standard mask, see https://golang.org/pkg/net/#IPMask.Size
return nil, errors.New("CIDR mask is in non-standard format")
}
var lastIP []byte
if prefixLen == bits {
// Special case for single ip address ranges ex: 192.168.1.1/32
// We can just use the starting IP as the last IP
lastIP = cidr.IP
} else {
// Use big.Int's so we can handle ipv6 addresses
firstIPInt := new(big.Int)
firstIPInt.SetBytes(cidr.IP)
hostLen := uint(bits) - uint(prefixLen)
lastIPInt := big.NewInt(1)
lastIPInt.Lsh(lastIPInt, hostLen)
lastIPInt.Sub(lastIPInt, big.NewInt(1))
lastIPInt.Or(lastIPInt, firstIPInt)
ipBytes := lastIPInt.Bytes()
lastIP = make([]byte, bits/8)
// Pack our IP bytes into the end of the return array,
// since big.Int.Bytes() removes front zero padding.
for i := 1; i <= len(lastIPInt.Bytes()); i++ {
lastIP[len(lastIP)-i] = ipBytes[len(ipBytes)-i]
}
}
return lastIP, nil
}
func builtinNetCIDRIntersects(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
cidrnetA, err := getNetFromOperand(operands[0].Value)
if err != nil {
return err
}
cidrnetB, err := getNetFromOperand(operands[1].Value)
if err != nil {
return err
}
// If either net contains the others starting IP they are overlapping
cidrsOverlap := cidrnetA.Contains(cidrnetB.IP) || cidrnetB.Contains(cidrnetA.IP)
return iter(ast.InternedTerm(cidrsOverlap))
}
func builtinNetCIDRContains(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
cidrnetA, err := getNetFromOperand(operands[0].Value)
if err != nil {
return err
}
// b could be either an IP addressor CIDR string, try to parse it as an IP first, fall back to CIDR
bStr, err := builtins.StringOperand(operands[1].Value, 1)
if err != nil {
return err
}
ip := net.ParseIP(string(bStr))
if ip != nil {
return iter(ast.InternedTerm(cidrnetA.Contains(ip)))
}
// It wasn't an IP, try and parse it as a CIDR
cidrnetB, err := getNetFromOperand(operands[1].Value)
if err != nil {
return fmt.Errorf("not a valid textual representation of an IP address or CIDR: %s", string(bStr))
}
// We can determine if cidr A contains cidr B if and only if A contains
// the starting address of B and the last address in B.
cidrContained := false
if cidrnetA.Contains(cidrnetB.IP) {
// Only spend time calculating the last IP if the starting IP is already verified to be in cidr A
lastIP, err := getLastIP(cidrnetB)
if err != nil {
return err
}
cidrContained = cidrnetA.Contains(lastIP)
}
return iter(ast.InternedTerm(cidrContained))
}
var errNetCIDRContainsMatchElementType = errors.New("element must be string or non-empty array")
func getCIDRMatchTerm(a *ast.Term) (*ast.Term, error) {
switch v := a.Value.(type) {
case ast.String:
return a, nil
case *ast.Array:
if v.Len() == 0 {
return nil, errNetCIDRContainsMatchElementType
}
return v.Elem(0), nil
default:
return nil, errNetCIDRContainsMatchElementType
}
}
func evalNetCIDRContainsMatchesOperand(operand int, a *ast.Term, iter func(cidr, index *ast.Term) error) error {
switch v := a.Value.(type) {
case ast.String:
return iter(a, a)
case *ast.Array:
for i := range v.Len() {
cidr, err := getCIDRMatchTerm(v.Elem(i))
if err != nil {
return fmt.Errorf("operand %v: %v", operand, err)
}
if err := iter(cidr, ast.InternedTerm(i)); err != nil {
return err
}
}
return nil
case ast.Set:
return v.Iter(func(x *ast.Term) error {
cidr, err := getCIDRMatchTerm(x)
if err != nil {
return fmt.Errorf("operand %v: %v", operand, err)
}
return iter(cidr, x)
})
case ast.Object:
return v.Iter(func(k, v *ast.Term) error {
cidr, err := getCIDRMatchTerm(v)
if err != nil {
return fmt.Errorf("operand %v: %v", operand, err)
}
return iter(cidr, k)
})
}
return nil
}
func builtinNetCIDRContainsMatches(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
result := ast.NewSet()
err := evalNetCIDRContainsMatchesOperand(1, operands[0], func(cidr1 *ast.Term, index1 *ast.Term) error {
return evalNetCIDRContainsMatchesOperand(2, operands[1], func(cidr2 *ast.Term, index2 *ast.Term) error {
if v, err := getResult(builtinNetCIDRContains, cidr1, cidr2); err != nil {
return err
} else if vb, ok := v.Value.(ast.Boolean); ok && bool(vb) {
result.Add(ast.ArrayTerm(index1, index2))
}
return nil
})
})
if err == nil {
return iter(ast.NewTerm(result))
}
return err
}
func builtinNetCIDRExpand(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
ip, ipNet, err := net.ParseCIDR(string(s))
if err != nil {
return err
}
result := ast.NewSet()
for ip := ip.Mask(ipNet.Mask); ipNet.Contains(ip); incIP(ip) {
if bctx.Cancel != nil && bctx.Cancel.Cancelled() {
return Halt{
Err: &Error{
Code: CancelErr,
Message: "net.cidr_expand: timed out before generating all IP addresses",
},
}
}
result.Add(ast.StringTerm(ip.String()))
}
return iter(ast.NewTerm(result))
}
func builtinNetCIDRIsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
cidr, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return iter(ast.InternedTerm(false))
}
if _, _, err := net.ParseCIDR(string(cidr)); err != nil {
return iter(ast.InternedTerm(false))
}
return iter(ast.InternedTerm(true))
}
type cidrBlockRange struct {
First *net.IP
Last *net.IP
Network *net.IPNet
}
type cidrBlockRanges []*cidrBlockRange
// Implement Sort interface
func (c cidrBlockRanges) Len() int {
return len(c)
}
func (c cidrBlockRanges) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
func (c cidrBlockRanges) Less(i, j int) bool {
// Compare last IP.
cmp := bytes.Compare(*c[i].Last, *c[j].Last)
if cmp < 0 {
return true
} else if cmp > 0 {
return false
}
// Then compare first IP.
cmp = bytes.Compare(*c[i].First, *c[j].First)
if cmp < 0 {
return true
} else if cmp > 0 {
return false
}
// Ranges are Equal.
return false
}
// builtinNetCIDRMerge merges the provided list of IP addresses and subnets into the smallest possible list of CIDRs.
// It merges adjacent subnets where possible, those contained within others and also removes any duplicates.
// Original Algorithm: https://github.com/netaddr/netaddr.
func builtinNetCIDRMerge(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
networks := []*net.IPNet{}
switch v := operands[0].Value.(type) {
case *ast.Array:
for i := range v.Len() {
network, err := generateIPNet(v.Elem(i))
if err != nil {
return err
}
networks = append(networks, network)
}
case ast.Set:
err := v.Iter(func(x *ast.Term) error {
network, err := generateIPNet(x)
if err != nil {
return err
}
networks = append(networks, network)
return nil
})
if err != nil {
return err
}
default:
return errors.New("operand must be an array")
}
merged := evalNetCIDRMerge(networks)
result := ast.NewSetWithCapacity(len(merged))
for _, network := range merged {
result.Add(ast.StringTerm(network.String()))
}
return iter(ast.NewTerm(result))
}
func evalNetCIDRMerge(networks []*net.IPNet) []*net.IPNet {
if len(networks) == 0 {
return nil
}
ranges := make(cidrBlockRanges, 0, len(networks))
// For each CIDR, create an IP range. Sort them and merge when possible.
for _, network := range networks {
firstIP, lastIP := cidrMerge.GetAddressRange(*network)
ranges = append(ranges, &cidrBlockRange{
First: &firstIP,
Last: &lastIP,
Network: network,
})
}
// merge CIDRs.
merged := mergeCIDRs(ranges)
// convert ranges into an equivalent list of net.IPNet.
result := []*net.IPNet{}
for _, r := range merged {
// Not merged with any other CIDR.
if r.Network != nil {
result = append(result, r.Network)
} else {
// Find new network that represents the merged range.
rangeCIDRs := cidrMerge.RangeToCIDRs(*r.First, *r.Last)
result = append(result, rangeCIDRs...)
}
}
return result
}
func generateIPNet(term *ast.Term) (*net.IPNet, error) {
e, ok := term.Value.(ast.String)
if !ok {
return nil, errors.New("element must be string")
}
// try to parse element as an IP first, fall back to CIDR
ip := net.ParseIP(string(e))
if ip == nil {
_, network, err := net.ParseCIDR(string(e))
return network, err
}
if ip.To4() != nil {
return &net.IPNet{
IP: ip,
Mask: ip.DefaultMask(),
}, nil
}
return nil, errors.New("IPv6 invalid: needs prefix length")
}
func mergeCIDRs(ranges cidrBlockRanges) cidrBlockRanges {
sort.Sort(ranges)
// Merge adjacent CIDRs if possible.
for i := len(ranges) - 1; i > 0; i-- {
previousIP := cidrMerge.GetPreviousIP(*ranges[i].First)
// If the previous IP of the current network overlaps
// with the last IP of the previous network in the
// list, then merge the two ranges together.
if bytes.Compare(previousIP, *ranges[i-1].Last) <= 0 {
var firstIP *net.IP
if bytes.Compare(*ranges[i-1].First, *ranges[i].First) < 0 {
firstIP = ranges[i-1].First
} else {
firstIP = ranges[i].First
}
lastIPRange := make(net.IP, len(*ranges[i].Last))
copy(lastIPRange, *ranges[i].Last)
firstIPRange := make(net.IP, len(*firstIP))
copy(firstIPRange, *firstIP)
ranges[i-1] = &cidrBlockRange{First: &firstIPRange, Last: &lastIPRange, Network: nil}
// Delete ranges[i] since merged with the previous.
ranges = slices.Delete(ranges, i, i+1)
}
}
return ranges
}
func incIP(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
func init() {
RegisterBuiltinFunc(ast.NetCIDROverlap.Name, builtinNetCIDRContains)
RegisterBuiltinFunc(ast.NetCIDRIntersects.Name, builtinNetCIDRIntersects)
RegisterBuiltinFunc(ast.NetCIDRContains.Name, builtinNetCIDRContains)
RegisterBuiltinFunc(ast.NetCIDRContainsMatches.Name, builtinNetCIDRContainsMatches)
RegisterBuiltinFunc(ast.NetCIDRExpand.Name, builtinNetCIDRExpand)
RegisterBuiltinFunc(ast.NetCIDRMerge.Name, builtinNetCIDRMerge)
RegisterBuiltinFunc(ast.NetCIDRIsValid.Name, builtinNetCIDRIsValid)
}
@@ -0,0 +1,48 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import "github.com/open-policy-agent/opa/v1/ast"
type compareFunc func(a, b ast.Value) bool
func compareGreaterThan(a, b ast.Value) bool {
return ast.Compare(a, b) > 0
}
func compareGreaterThanEq(a, b ast.Value) bool {
return ast.Compare(a, b) >= 0
}
func compareLessThan(a, b ast.Value) bool {
return ast.Compare(a, b) < 0
}
func compareLessThanEq(a, b ast.Value) bool {
return ast.Compare(a, b) <= 0
}
func compareNotEq(a, b ast.Value) bool {
return ast.Compare(a, b) != 0
}
func compareEq(a, b ast.Value) bool {
return ast.Compare(a, b) == 0
}
func builtinCompare(cmp compareFunc) BuiltinFunc {
return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
return iter(ast.InternedTerm(cmp(operands[0].Value, operands[1].Value)))
}
}
func init() {
RegisterBuiltinFunc(ast.GreaterThan.Name, builtinCompare(compareGreaterThan))
RegisterBuiltinFunc(ast.GreaterThanEq.Name, builtinCompare(compareGreaterThanEq))
RegisterBuiltinFunc(ast.LessThan.Name, builtinCompare(compareLessThan))
RegisterBuiltinFunc(ast.LessThanEq.Name, builtinCompare(compareLessThanEq))
RegisterBuiltinFunc(ast.NotEqual.Name, builtinCompare(compareNotEq))
RegisterBuiltinFunc(ast.Equal.Name, builtinCompare(compareEq))
}
@@ -0,0 +1,488 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package copypropagation
import (
"fmt"
"sort"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
// CopyPropagator implements a simple copy propagation optimization to remove
// intermediate variables in partial evaluation results.
//
// For example, given the query: input.x > 1 where 'input' is unknown, the
// compiled query would become input.x = a; a > 1 which would remain in the
// partial evaluation result. The CopyPropagator will remove the variable
// assignment so that partial evaluation simply outputs input.x > 1.
//
// In many cases, copy propagation can remove all variables from the result of
// partial evaluation which simplifies evaluation for non-OPA consumers.
//
// In some cases, copy propagation cannot remove all variables. If the output of
// a built-in call is subsequently used as a ref head, the output variable must
// be kept. For example. sort(input, x); x[0] == 1. In this case, copy
// propagation cannot replace x[0] == 1 with sort(input, x)[0] == 1 as this is
// not legal.
type CopyPropagator struct {
livevars ast.VarSet // vars that must be preserved in the resulting query
sorted []ast.Var // sorted copy of vars to ensure deterministic result
ensureNonEmptyBody bool
compiler *ast.Compiler
localvargen *localVarGenerator
}
type localVarGenerator struct {
next int
}
func (l *localVarGenerator) Generate() ast.Var {
result := ast.Var(fmt.Sprintf("__localcp%d__", l.next))
l.next++
return result
}
// New returns a new CopyPropagator that optimizes queries while preserving vars
// in the livevars set.
func New(livevars ast.VarSet) *CopyPropagator {
return &CopyPropagator{livevars: livevars, sorted: util.KeysSorted(livevars), localvargen: &localVarGenerator{}}
}
// WithEnsureNonEmptyBody configures p to ensure that results are always non-empty.
func (p *CopyPropagator) WithEnsureNonEmptyBody(yes bool) *CopyPropagator {
p.ensureNonEmptyBody = yes
return p
}
// WithCompiler configures the compiler to read from while processing the query. This
// should be the same compiler used to compile the original policy.
func (p *CopyPropagator) WithCompiler(c *ast.Compiler) *CopyPropagator {
p.compiler = c
return p
}
// Apply executes the copy propagation optimization and returns a new query.
func (p *CopyPropagator) Apply(query ast.Body) ast.Body {
result := ast.NewBody()
uf, ok := makeDisjointSets(p.livevars, query)
if !ok {
return query
}
// Compute set of vars that appear in the head of refs in the query. If a var
// is dereferenced, we can plug it with a constant value, but it is not always
// optimal to do so.
// TODO: Improve the algorithm for when we should plug constants/calls/etc
headvars := ast.NewVarSet()
ast.WalkRefs(query, func(x ast.Ref) bool {
if v, ok := x[0].Value.(ast.Var); ok {
if root, ok := uf.Find(v); ok {
root.constant = nil
headvars.Add(root.key.(ast.Var))
} else {
headvars.Add(v)
}
}
return false
})
removedEqs := ast.NewValueMap()
for _, expr := range query {
pctx := &plugContext{
removedEqs: removedEqs,
uf: uf,
negated: expr.Negated,
headvars: headvars,
}
expr = p.plugBindings(pctx, expr)
if p.updateBindings(pctx, expr) {
result.Append(expr)
}
}
// Run post-processing step on the query to ensure that all live vars are bound
// in the result. The plugging that happens above substitutes all vars in the
// same set with the root.
//
// This step should run before the next step to prevent unnecessary bindings
// from being added to the result. For example:
//
// - Given the following result: <empty>
// - Given the following removed equalities: "x = input.x" and "y = input"
// - Given the following liveset: {x}
//
// If this step were to run AFTER the following step, the output would be:
//
// x = input.x; y = input
//
// Even though y = input is not required.
for _, v := range p.sorted {
if root, ok := uf.Find(v); ok {
if root.constant != nil {
result.Append(ast.Equality.Expr(ast.NewTerm(v), root.constant))
} else if b := removedEqs.Get(root.key); b != nil {
result.Append(ast.Equality.Expr(ast.NewTerm(v), ast.NewTerm(b)))
} else if root.key != v {
result.Append(ast.Equality.Expr(ast.NewTerm(v), ast.NewTerm(root.key)))
}
}
}
// Run post-processing step on query to ensure that all killed exprs are
// accounted for. There are several cases we look for:
//
// * If an expr is killed but the binding is never used, the query
// must still include the expr. For example, given the query 'input.x = a' and
// an empty livevar set, the result must include the ref input.x otherwise the
// query could be satisfied without input.x being defined.
//
// * If an expr is killed that provided safety to vars which are not
// otherwise being made safe by the current result.
//
// For any of these cases we re-add the removed equality expression
// to the current result.
// Invariant: Live vars are bound (above) and reserved vars are implicitly ground.
safe := ast.NewVarSetOfSize(len(p.livevars) + len(ast.ReservedVars) + 6)
safe.Update(ast.ReservedVars)
safe.Update(p.livevars)
safe.Update(ast.OutputVarsFromBody(p.compiler, result, safe))
unsafe := result.Vars(ast.SafetyCheckVisitorParams).Diff(safe)
for _, b := range sortbindings(removedEqs) {
removedEq := ast.Equality.Expr(ast.NewTerm(b.k), ast.NewTerm(b.v))
providesSafety := false
outputVars := ast.OutputVarsFromExpr(p.compiler, removedEq, safe)
if unsafe.DiffCount(outputVars) < len(unsafe) {
unsafe = unsafe.Diff(outputVars)
providesSafety = true
}
safevarRef := false // don't add something like `_ = input`
if r, ok := b.v.(ast.Ref); ok {
if len(r) == 1 {
if v, ok := r[0].Value.(ast.Var); ok {
safevarRef = safe.Contains(v)
}
}
}
if providesSafety || (!safevarRef && !containedIn(b.v, result)) {
result.Append(removedEq)
safe.Update(outputVars)
}
}
if len(unsafe) > 0 {
// NOTE(tsandall): This should be impossible but if it does occur, throw
// away the result rather than generating unsafe output.
return query
}
if p.ensureNonEmptyBody && len(result) == 0 {
result = append(result, ast.NewExpr(ast.BooleanTerm(true)))
}
return result
}
// plugBindings applies the binding list and union-find to x. This process
// removes as many variables as possible.
func (*CopyPropagator) plugBindings(pctx *plugContext, expr *ast.Expr) *ast.Expr {
xform := bindingPlugTransform{
pctx: pctx,
}
// Deep copy the expression as it may be mutated during the transform and
// the caller running copy propagation may have references to the
// expression. Note, the transform does not contain any error paths and
// should never return a non-expression value for the root so consider
// errors unreachable.
x, err := ast.Transform(xform, expr.Copy())
expr, ok := x.(*ast.Expr)
if !ok || err != nil {
panic("unreachable")
}
return expr
}
type bindingPlugTransform struct {
pctx *plugContext
}
func (t bindingPlugTransform) Transform(x any) (any, error) {
switch x := x.(type) {
case ast.Var:
return t.plugBindingsVar(t.pctx, x), nil
case ast.Ref:
return t.plugBindingsRef(t.pctx, x), nil
default:
return x, nil
}
}
func (bindingPlugTransform) plugBindingsVar(pctx *plugContext, v ast.Var) ast.Value {
var result ast.Value = v
// Apply union-find to remove redundant variables from input.
root, ok := pctx.uf.Find(v)
if ok {
result = root.Value()
}
// Apply binding list to substitute remaining vars.
v, ok = result.(ast.Var)
if !ok {
return result
}
b := pctx.removedEqs.Get(v)
if b == nil {
return result
}
if pctx.negated && !b.IsGround() {
return result
}
if r, ok := b.(ast.Ref); ok && r.OutputVars().Contains(v) {
return result
}
return b
}
func (bindingPlugTransform) plugBindingsRef(pctx *plugContext, v ast.Ref) ast.Ref {
// Apply union-find to remove redundant variables from input.
if root, ok := pctx.uf.Find(v[0].Value); ok {
v[0].Value = root.Value()
}
result := v
// Refs require special handling. If the head of the ref was killed, then
// the rest of the ref must be concatenated with the new base.
if b := pctx.removedEqs.Get(v[0].Value); b != nil {
if !pctx.negated || b.IsGround() {
var base ast.Ref
switch x := b.(type) {
case ast.Ref:
base = x
default:
base = ast.Ref{ast.NewTerm(x)}
}
result = base.Concat(v[1:])
}
}
return result
}
// updateBindings returns false if the expression can be killed. If the
// expression is killed, the binding list is updated to map a var to value.
func (p *CopyPropagator) updateBindings(pctx *plugContext, expr *ast.Expr) bool {
switch {
case pctx.negated || len(expr.With) > 0:
return true
case expr.IsEquality():
a, b := expr.Operand(0), expr.Operand(1)
if a.Equal(b) {
if p.livevarRef(a) {
pctx.removedEqs.Put(p.localvargen.Generate(), a.Value)
}
return false
}
k, v, keep := p.updateBindingsEq(a, b)
if !keep {
if v != nil {
pctx.removedEqs.Put(k, v)
}
return false
}
case expr.IsCall():
terms := expr.Terms.([]*ast.Term)
if p.compiler.GetArity(expr.Operator()) == len(terms)-2 { // with captured output
output := terms[len(terms)-1]
if k, ok := output.Value.(ast.Var); ok && !p.livevars.Contains(k) && !pctx.headvars.Contains(k) {
pctx.removedEqs.Put(k, ast.CallTerm(terms[:len(terms)-1]...).Value)
return false
}
}
}
return !isNoop(expr)
}
func (p *CopyPropagator) livevarRef(a *ast.Term) bool {
ref, ok := a.Value.(ast.Ref)
if !ok {
return false
}
for _, v := range p.sorted {
if v.Equal(ref[0].Value) {
return true
}
}
return false
}
func (p *CopyPropagator) updateBindingsEq(a, b *ast.Term) (ast.Var, ast.Value, bool) {
k, v, keep := p.updateBindingsEqAsymmetric(a, b)
if !keep {
return k, v, keep
}
return p.updateBindingsEqAsymmetric(b, a)
}
func (p *CopyPropagator) updateBindingsEqAsymmetric(a, b *ast.Term) (ast.Var, ast.Value, bool) {
k, ok := a.Value.(ast.Var)
if !ok || p.livevars.Contains(k) {
return "", nil, true
}
switch b.Value.(type) {
case ast.Ref, ast.Call:
return k, b.Value, false
}
return "", nil, true
}
type plugContext struct {
removedEqs *ast.ValueMap
uf *unionFind
headvars ast.VarSet
negated bool
}
type binding struct {
k, v ast.Value
}
func containedIn(value ast.Value, x any) bool {
var stop bool
var vis *ast.GenericVisitor
vis = ast.NewGenericVisitor(func(x any) bool {
switch x := x.(type) {
case *ast.Every: // skip body
vis.Walk(x.Key)
vis.Walk(x.Value)
vis.Walk(x.Domain)
return true
case *ast.ArrayComprehension, *ast.ObjectComprehension, *ast.SetComprehension: // skip
return true
case ast.Ref:
var match bool
if v, ok := value.(ast.Ref); ok {
match = x.HasPrefix(v)
} else {
match = x.Equal(value)
}
if stop || match {
stop = true
return stop
}
case ast.Value:
if stop || x.Compare(value) == 0 {
stop = true
return stop
}
}
return stop
})
vis.Walk(x)
return stop
}
func sortbindings(bindings *ast.ValueMap) []*binding {
sorted := make([]*binding, 0, bindings.Len())
bindings.Iter(func(k ast.Value, v ast.Value) bool {
sorted = append(sorted, &binding{k, v})
return false
})
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].k.Compare(sorted[j].k) > 0
})
return sorted
}
// makeDisjointSets builds the union-find structure for the query. The structure
// is built by processing all of the equality exprs in the query. Sets represent
// vars that must be equal to each other. In addition to vars, each set can have
// at most one constant. If the query contains expressions that cannot be
// satisfied (e.g., because a set has multiple constants) this function returns
// false.
func makeDisjointSets(livevars ast.VarSet, query ast.Body) (*unionFind, bool) {
uf := newUnionFind(func(r1, r2 *unionFindRoot) (*unionFindRoot, *unionFindRoot) {
if v, ok := r1.key.(ast.Var); ok && livevars.Contains(v) {
return r1, r2
}
return r2, r1
})
for _, expr := range query {
if expr.IsEquality() && !expr.Negated && len(expr.With) == 0 {
a, b := expr.Operand(0), expr.Operand(1)
varA, ok1 := a.Value.(ast.Var)
varB, ok2 := b.Value.(ast.Var)
switch {
case ok1 && ok2:
if _, ok := uf.Merge(varA, varB); !ok {
return nil, false
}
case ok1 && ast.IsConstant(b.Value):
root := uf.MakeSet(varA)
if root.constant != nil && !root.constant.Equal(b) {
return nil, false
}
root.constant = b
case ok2 && ast.IsConstant(a.Value):
root := uf.MakeSet(varB)
if root.constant != nil && !root.constant.Equal(a) {
return nil, false
}
root.constant = a
}
}
}
return uf, true
}
func isNoop(expr *ast.Expr) bool {
if !expr.IsCall() && !expr.IsEvery() {
term := expr.Terms.(*ast.Term)
if !ast.IsConstant(term.Value) {
return false
}
return !ast.Boolean(false).Equal(term.Value)
}
// A==A can be ignored
if expr.Operator().Equal(ast.Equal.Ref()) {
return expr.Operand(0).Equal(expr.Operand(1))
}
return false
}
@@ -0,0 +1,131 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package copypropagation
import (
"fmt"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
type rankFunc func(*unionFindRoot, *unionFindRoot) (*unionFindRoot, *unionFindRoot)
type unionFind struct {
roots *util.HasherMap[ast.Value, *unionFindRoot]
parents *ast.ValueMap
rank rankFunc
}
func newUnionFind(rank rankFunc) *unionFind {
return &unionFind{
roots: util.NewHasherMap[ast.Value, *unionFindRoot](ast.ValueEqual),
parents: ast.NewValueMap(),
rank: rank,
}
}
func (uf *unionFind) MakeSet(v ast.Value) *unionFindRoot {
root, ok := uf.Find(v)
if ok {
return root
}
root = newUnionFindRoot(v)
uf.parents.Put(v, v)
uf.roots.Put(v, root)
return root
}
func (uf *unionFind) Find(v ast.Value) (*unionFindRoot, bool) {
parent := uf.parents.Get(v)
if parent == nil {
return nil, false
}
if parent.Compare(v) == 0 {
r, ok := uf.roots.Get(v)
return r, ok
}
return uf.Find(parent)
}
func (uf *unionFind) Merge(a, b ast.Value) (*unionFindRoot, bool) {
r1 := uf.MakeSet(a)
r2 := uf.MakeSet(b)
if r1 != r2 {
r1, r2 = uf.rank(r1, r2)
uf.parents.Put(r2.key, r1.key)
uf.roots.Delete(r2.key)
// Sets can have at most one constant value associated with them. When
// unioning, we must preserve this invariant. If a set has two constants,
// there will be no way to prove the query.
if r1.constant != nil && r2.constant != nil && !r1.constant.Equal(r2.constant) {
return nil, false
} else if r1.constant == nil {
r1.constant = r2.constant
}
}
return r1, true
}
func (uf *unionFind) String() string {
o := struct {
Roots map[string]any
Parents map[string]ast.Value
}{
map[string]any{},
map[string]ast.Value{},
}
uf.roots.Iter(func(k ast.Value, v *unionFindRoot) bool {
o.Roots[k.String()] = struct {
Constant *ast.Term
Key ast.Value
}{
v.constant,
v.key,
}
return true
})
uf.parents.Iter(func(k ast.Value, v ast.Value) bool {
o.Parents[k.String()] = v
return true
})
return string(util.MustMarshalJSON(o))
}
type unionFindRoot struct {
key ast.Value
constant *ast.Term
}
func newUnionFindRoot(key ast.Value) *unionFindRoot {
return &unionFindRoot{
key: key,
}
}
func (r *unionFindRoot) Value() ast.Value {
if r.constant != nil {
return r.constant.Value
}
return r.key
}
func (r *unionFindRoot) String() string {
return fmt.Sprintf("{key: %s, constant: %s", r.key, r.constant)
}
+755
View File
@@ -0,0 +1,755 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"bytes"
"crypto"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"hash"
"os"
"strings"
"time"
"github.com/lestrrat-go/jwx/v3/jwk"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/util"
)
const (
// blockTypeCertificate indicates this PEM block contains the signed certificate.
// Exported for tests.
blockTypeCertificate = "CERTIFICATE"
// blockTypeCertificateRequest indicates this PEM block contains a certificate
// request. Exported for tests.
blockTypeCertificateRequest = "CERTIFICATE REQUEST"
// blockTypeRSAPrivateKey indicates this PEM block contains a RSA private key.
// Exported for tests.
blockTypeRSAPrivateKey = "RSA PRIVATE KEY"
// blockTypeRSAPrivateKey indicates this PEM block contains a RSA private key.
// Exported for tests.
blockTypePrivateKey = "PRIVATE KEY"
blockTypeEcPrivateKey = "EC PRIVATE KEY"
)
func builtinCryptoX509ParseCertificates(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
certs, err := getX509CertsFromString(string(input))
if err != nil {
return err
}
v, err := ast.InterfaceToValue(extendCertificates(certs))
if err != nil {
return err
}
return iter(ast.NewTerm(v))
}
// extendedCert is a wrapper around x509.Certificate that adds additional fields for JSON serialization.
type extendedCert struct {
x509.Certificate
URIStrings []string
}
func extendCertificates(certs []*x509.Certificate) []extendedCert {
// add a field to certs containing the URIs as strings
processedCerts := make([]extendedCert, len(certs))
for i, cert := range certs {
processedCerts[i].Certificate = *cert
if cert.URIs != nil {
processedCerts[i].URIStrings = make([]string, len(cert.URIs))
for j, uri := range cert.URIs {
processedCerts[i].URIStrings[j] = uri.String()
}
}
}
return processedCerts
}
func builtinCryptoX509ParseAndVerifyCertificates(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
a := operands[0].Value
input, err := builtins.StringOperand(a, 1)
if err != nil {
return err
}
certs, err := getX509CertsFromString(string(input))
if err != nil {
return iter(ast.ArrayTerm(ast.InternedTerm(false), ast.InternedEmptyArray))
}
verified, err := verifyX509CertificateChain(certs, x509.VerifyOptions{})
if err != nil {
return iter(ast.ArrayTerm(ast.InternedTerm(false), ast.InternedEmptyArray))
}
value, err := ast.InterfaceToValue(extendCertificates(verified))
if err != nil {
return err
}
valid := ast.ArrayTerm(ast.InternedTerm(true), ast.NewTerm(value))
return iter(valid)
}
var allowedKeyUsages = map[string]x509.ExtKeyUsage{
"KeyUsageAny": x509.ExtKeyUsageAny,
"KeyUsageServerAuth": x509.ExtKeyUsageServerAuth,
"KeyUsageClientAuth": x509.ExtKeyUsageClientAuth,
"KeyUsageCodeSigning": x509.ExtKeyUsageCodeSigning,
"KeyUsageEmailProtection": x509.ExtKeyUsageEmailProtection,
"KeyUsageIPSECEndSystem": x509.ExtKeyUsageIPSECEndSystem,
"KeyUsageIPSECTunnel": x509.ExtKeyUsageIPSECTunnel,
"KeyUsageIPSECUser": x509.ExtKeyUsageIPSECUser,
"KeyUsageTimeStamping": x509.ExtKeyUsageTimeStamping,
"KeyUsageOCSPSigning": x509.ExtKeyUsageOCSPSigning,
"KeyUsageMicrosoftServerGatedCrypto": x509.ExtKeyUsageMicrosoftServerGatedCrypto,
"KeyUsageNetscapeServerGatedCrypto": x509.ExtKeyUsageNetscapeServerGatedCrypto,
"KeyUsageMicrosoftCommercialCodeSigning": x509.ExtKeyUsageMicrosoftCommercialCodeSigning,
"KeyUsageMicrosoftKernelCodeSigning": x509.ExtKeyUsageMicrosoftKernelCodeSigning,
}
func builtinCryptoX509ParseAndVerifyCertificatesWithOptions(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
options, err := builtins.ObjectOperand(operands[1].Value, 2)
if err != nil {
return err
}
certs, err := getX509CertsFromString(string(input))
if err != nil {
return iter(ast.ArrayTerm(ast.InternedTerm(false), ast.InternedEmptyArray))
}
// Collect the cert verification options
verifyOpt, err := extractVerifyOpts(options)
if err != nil {
return err
}
verified, err := verifyX509CertificateChain(certs, verifyOpt)
if err != nil {
return iter(ast.ArrayTerm(ast.InternedTerm(false), ast.InternedEmptyArray))
}
value, err := ast.InterfaceToValue(verified)
if err != nil {
return err
}
return iter(ast.ArrayTerm(ast.InternedTerm(true), ast.NewTerm(value)))
}
func extractVerifyOpts(options ast.Object) (verifyOpt x509.VerifyOptions, err error) {
for _, key := range options.Keys() {
k, err := ast.JSON(key.Value)
if err != nil {
return verifyOpt, err
}
k, ok := k.(string)
if !ok {
continue
}
switch k {
case "DNSName":
dns, ok := options.Get(key).Value.(ast.String)
if ok {
verifyOpt.DNSName = strings.Trim(string(dns), "\"")
} else {
return verifyOpt, errors.New("'DNSName' should be a string")
}
case "CurrentTime":
c, ok := options.Get(key).Value.(ast.Number)
if ok {
nanosecs, ok := c.Int64()
if ok {
verifyOpt.CurrentTime = time.Unix(0, nanosecs)
} else {
return verifyOpt, errors.New("'CurrentTime' should be a valid int64 number")
}
} else {
return verifyOpt, errors.New("'CurrentTime' should be a number")
}
case "MaxConstraintComparisons":
c, ok := options.Get(key).Value.(ast.Number)
if ok {
maxComparisons, ok := c.Int()
if ok {
verifyOpt.MaxConstraintComparisions = maxComparisons
} else {
return verifyOpt, errors.New("'MaxConstraintComparisons' should be a valid number")
}
} else {
return verifyOpt, errors.New("'MaxConstraintComparisons' should be a number")
}
case "KeyUsages":
type forEach interface {
Foreach(func(*ast.Term))
}
var ks forEach
switch v := options.Get(key).Value.(type) {
case *ast.Array:
ks = v
case ast.Set:
ks = v
default:
return verifyOpt, errors.New("'KeyUsages' should be an Array or Set")
}
// Collect the x509.ExtKeyUsage values by looking up the
// mapping of key usage strings to x509.ExtKeyUsage
var invalidKUsgs []string
ks.Foreach(func(t *ast.Term) {
u, ok := t.Value.(ast.String)
if ok {
v := strings.Trim(string(u), "\"")
if k, ok := allowedKeyUsages[v]; ok {
verifyOpt.KeyUsages = append(verifyOpt.KeyUsages, k)
} else {
invalidKUsgs = append(invalidKUsgs, v)
}
}
})
if len(invalidKUsgs) > 0 {
return x509.VerifyOptions{}, fmt.Errorf("invalid entries for 'KeyUsages' found: %s", invalidKUsgs)
}
default:
return verifyOpt, errors.New("invalid key option")
}
}
return verifyOpt, nil
}
func builtinCryptoX509ParseKeyPair(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
certificate, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
key, err := builtins.StringOperandByteSlice(operands[1].Value, 1)
if err != nil {
return err
}
certs, err := getTLSx509KeyPairFromString(certificate, key)
if err != nil {
return err
}
v, err := ast.InterfaceToValue(certs)
if err != nil {
return err
}
return iter(ast.NewTerm(v))
}
func builtinCryptoX509ParseCertificateRequest(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// data to be passed to x509.ParseCertificateRequest
bytes := []byte(input)
// if the input is not a PEM string, attempt to decode b64
if str := string(input); !strings.HasPrefix(str, "-----BEGIN CERTIFICATE REQUEST-----") {
bytes, err = base64.StdEncoding.DecodeString(str)
if err != nil {
return err
}
}
p, _ := pem.Decode(bytes)
if p != nil && p.Type != blockTypeCertificateRequest {
return errors.New("invalid PEM-encoded certificate signing request")
}
if p != nil {
bytes = p.Bytes
}
csr, err := x509.ParseCertificateRequest(bytes)
if err != nil {
return err
}
bs, err := json.Marshal(csr)
if err != nil {
return err
}
var x any
if err := util.UnmarshalJSON(bs, &x); err != nil {
return err
}
v, err := ast.InterfaceToValue(x)
if err != nil {
return err
}
return iter(ast.NewTerm(v))
}
func builtinCryptoJWKFromPrivateKey(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// get the raw private key
pemDataString := string(input)
if pemDataString == "" {
return errors.New("input PEM data was empty")
}
// This built in must be supplied a valid PEM or base64 encoded string.
// If the input is not a PEM string, attempt to decode b64.
// If the base64 decode fails - this is an error
if !strings.HasPrefix(pemDataString, "-----BEGIN") {
bs, err := base64.StdEncoding.DecodeString(pemDataString)
if err != nil {
return err
}
pemDataString = string(bs)
}
rawKeys, err := getPrivateKeysFromPEMData(pemDataString)
if err != nil {
return err
}
if len(rawKeys) == 0 {
return iter(ast.InternedNullTerm)
}
key, err := jwk.Import(rawKeys[0])
if err != nil {
return err
}
jsonKey, err := json.Marshal(key)
if err != nil {
return err
}
var x any
if err := util.UnmarshalJSON(jsonKey, &x); err != nil {
return err
}
value, err := ast.InterfaceToValue(x)
if err != nil {
return err
}
return iter(ast.NewTerm(value))
}
func builtinCryptoParsePrivateKeys(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
a := operands[0].Value
input, err := builtins.StringOperand(a, 1)
if err != nil {
return err
}
if string(input) == "" {
return iter(ast.InternedNullTerm)
}
// get the raw private key
rawKeys, err := getPrivateKeysFromPEMData(string(input))
if err != nil {
return err
}
if len(rawKeys) == 0 {
return iter(ast.InternedEmptyArray)
}
bs, err := json.Marshal(rawKeys)
if err != nil {
return err
}
var x any
if err := util.UnmarshalJSON(bs, &x); err != nil {
return err
}
value, err := ast.InterfaceToValue(x)
if err != nil {
return err
}
return iter(ast.NewTerm(value))
}
func toHexEncodedString(src []byte) string {
dst := make([]byte, hex.EncodedLen(len(src)))
hex.Encode(dst, src)
return util.ByteSliceToString(dst)
}
func builtinCryptoMd5(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
md5sum := md5.Sum(bs)
return iter(ast.StringTerm(toHexEncodedString(md5sum[:])))
}
func builtinCryptoSha1(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
sha1sum := sha1.Sum(bs)
return iter(ast.StringTerm(toHexEncodedString(sha1sum[:])))
}
func builtinCryptoSha256(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
sha256sum := sha256.Sum256(bs)
return iter(ast.StringTerm(toHexEncodedString(sha256sum[:])))
}
func hmacHelper(operands []*ast.Term, iter func(*ast.Term) error, h func() hash.Hash) error {
message, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
key, err := builtins.StringOperandByteSlice(operands[1].Value, 2)
if err != nil {
return err
}
mac := hmac.New(h, key)
mac.Write(message)
messageDigest := mac.Sum(nil)
return iter(ast.StringTerm(hex.EncodeToString(messageDigest)))
}
func builtinCryptoHmacMd5(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
return hmacHelper(operands, iter, md5.New)
}
func builtinCryptoHmacSha1(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
return hmacHelper(operands, iter, sha1.New)
}
func builtinCryptoHmacSha256(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
return hmacHelper(operands, iter, sha256.New)
}
func builtinCryptoHmacSha512(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
return hmacHelper(operands, iter, sha512.New)
}
func builtinCryptoHmacEqual(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
mac1, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
mac2, err := builtins.StringOperandByteSlice(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.InternedTerm(hmac.Equal(mac1, mac2)))
}
func init() {
RegisterBuiltinFunc(ast.CryptoX509ParseCertificates.Name, builtinCryptoX509ParseCertificates)
RegisterBuiltinFunc(ast.CryptoX509ParseAndVerifyCertificates.Name, builtinCryptoX509ParseAndVerifyCertificates)
RegisterBuiltinFunc(ast.CryptoX509ParseAndVerifyCertificatesWithOptions.Name, builtinCryptoX509ParseAndVerifyCertificatesWithOptions)
RegisterBuiltinFunc(ast.CryptoMd5.Name, builtinCryptoMd5)
RegisterBuiltinFunc(ast.CryptoSha1.Name, builtinCryptoSha1)
RegisterBuiltinFunc(ast.CryptoSha256.Name, builtinCryptoSha256)
RegisterBuiltinFunc(ast.CryptoX509ParseCertificateRequest.Name, builtinCryptoX509ParseCertificateRequest)
RegisterBuiltinFunc(ast.CryptoX509ParseRSAPrivateKey.Name, builtinCryptoJWKFromPrivateKey)
RegisterBuiltinFunc(ast.CryptoParsePrivateKeys.Name, builtinCryptoParsePrivateKeys)
RegisterBuiltinFunc(ast.CryptoX509ParseKeyPair.Name, builtinCryptoX509ParseKeyPair)
RegisterBuiltinFunc(ast.CryptoHmacMd5.Name, builtinCryptoHmacMd5)
RegisterBuiltinFunc(ast.CryptoHmacSha1.Name, builtinCryptoHmacSha1)
RegisterBuiltinFunc(ast.CryptoHmacSha256.Name, builtinCryptoHmacSha256)
RegisterBuiltinFunc(ast.CryptoHmacSha512.Name, builtinCryptoHmacSha512)
RegisterBuiltinFunc(ast.CryptoHmacEqual.Name, builtinCryptoHmacEqual)
}
func verifyX509CertificateChain(certs []*x509.Certificate, vo x509.VerifyOptions) ([]*x509.Certificate, error) {
if len(certs) < 2 {
return nil, builtins.NewOperandErr(1, "must supply at least two certificates to be able to verify")
}
// first cert is the root
roots := x509.NewCertPool()
roots.AddCert(certs[0])
// all other certs except the last are intermediates
intermediates := x509.NewCertPool()
for i := 1; i < len(certs)-1; i++ {
intermediates.AddCert(certs[i])
}
// last cert is the leaf
leaf := certs[len(certs)-1]
// verify the cert chain back to the root
verifyOpts := x509.VerifyOptions{
Roots: roots,
Intermediates: intermediates,
DNSName: vo.DNSName,
CurrentTime: vo.CurrentTime,
KeyUsages: vo.KeyUsages,
MaxConstraintComparisions: vo.MaxConstraintComparisions,
}
chains, err := leaf.Verify(verifyOpts)
if err != nil {
return nil, err
}
return chains[0], nil
}
func getX509CertsFromString(certs string) ([]*x509.Certificate, error) {
// if the input is PEM handle that
if strings.HasPrefix(certs, "-----BEGIN") {
return getX509CertsFromPem([]byte(certs))
}
// assume input is base64 if not PEM
b64, err := base64.StdEncoding.DecodeString(certs)
if err != nil {
return nil, err
}
// handle if the decoded base64 contains PEM rather than the expected DER
if bytes.HasPrefix(b64, []byte("-----BEGIN")) {
return getX509CertsFromPem(b64)
}
// otherwise assume the contents are DER
return x509.ParseCertificates(b64)
}
func getX509CertsFromPem(pemBlocks []byte) ([]*x509.Certificate, error) {
var decodedCerts []byte
for len(pemBlocks) > 0 {
p, r := pem.Decode(pemBlocks)
if p != nil && p.Type != blockTypeCertificate {
return nil, fmt.Errorf("PEM block type is '%s', expected %s", p.Type, blockTypeCertificate)
}
if p == nil {
break
}
pemBlocks = r
decodedCerts = append(decodedCerts, p.Bytes...)
}
return x509.ParseCertificates(decodedCerts)
}
func getPrivateKeysFromPEMData(pemData string) ([]crypto.PrivateKey, error) {
pemBlockString := pemData
var validPrivateKeys []crypto.PrivateKey
// if the input is base64, decode it
bs, err := base64.StdEncoding.DecodeString(pemBlockString)
if err == nil {
pemBlockString = string(bs)
}
bs = []byte(pemBlockString)
for len(bs) > 0 {
inputLen := len(bs)
var block *pem.Block
block, bs = pem.Decode(bs)
if block == nil && len(bs) == 0 {
break
}
// should only happen if end of input is not a valid PEM block. See TestParseRSAPrivateKeyVariedPemInput.
if inputLen == len(bs) {
break
}
if block == nil {
continue
}
switch block.Type {
case blockTypeRSAPrivateKey:
parsedKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return nil, err
}
validPrivateKeys = append(validPrivateKeys, parsedKey)
case blockTypePrivateKey:
parsedKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, err
}
validPrivateKeys = append(validPrivateKeys, parsedKey)
case blockTypeEcPrivateKey:
parsedKey, err := x509.ParseECPrivateKey(block.Bytes)
if err != nil {
return nil, err
}
validPrivateKeys = append(validPrivateKeys, parsedKey)
}
}
return validPrivateKeys, nil
}
// addCACertsFromFile adds CA certificates from filePath into the given pool.
// If pool is nil, it creates a new x509.CertPool. pool is returned.
func addCACertsFromFile(pool *x509.CertPool, filePath string) (*x509.CertPool, error) {
if pool == nil {
pool = x509.NewCertPool()
}
caCert, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
if ok := pool.AppendCertsFromPEM(caCert); !ok {
return nil, fmt.Errorf("could not append CA certificates from %q", filePath)
}
return pool, nil
}
// addCACertsFromBytes adds CA certificates from pemBytes into the given pool.
// If pool is nil, it creates a new x509.CertPool. pool is returned.
func addCACertsFromBytes(pool *x509.CertPool, pemBytes []byte) (*x509.CertPool, error) {
if pool == nil {
pool = x509.NewCertPool()
}
if ok := pool.AppendCertsFromPEM(pemBytes); !ok {
return nil, errors.New("could not append certificates")
}
return pool, nil
}
// addCACertsFromEnv adds CA certificates from the environment variable named
// by envName into the given pool. If pool is nil, it creates a new x509.CertPool.
// pool is returned.
func addCACertsFromEnv(pool *x509.CertPool, envName string) (*x509.CertPool, error) {
pool, err := addCACertsFromBytes(pool, []byte(os.Getenv(envName)))
if err != nil {
return nil, fmt.Errorf("could not add CA certificates from envvar %q: %w", envName, err)
}
return pool, nil
}
var beginPrefix = []byte("-----BEGIN ")
func getTLSx509KeyPairFromString(certPemBlock []byte, keyPemBlock []byte) (*tls.Certificate, error) {
if !bytes.HasPrefix(certPemBlock, beginPrefix) {
s, err := base64.StdEncoding.DecodeString(string(certPemBlock))
if err != nil {
return nil, err
}
certPemBlock = s
}
if !bytes.HasPrefix(keyPemBlock, beginPrefix) {
s, err := base64.StdEncoding.DecodeString(string(keyPemBlock))
if err != nil {
return nil, err
}
keyPemBlock = s
}
// we assume it a DER certificate and try to convert it to a PEM.
if !bytes.HasPrefix(certPemBlock, beginPrefix) {
pemBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: certPemBlock,
}
var buf bytes.Buffer
if err := pem.Encode(&buf, pemBlock); err != nil {
return nil, err
}
certPemBlock = buf.Bytes()
}
// we assume it a DER key and try to convert it to a PEM.
if !bytes.HasPrefix(keyPemBlock, []byte("-----BEGIN")) {
pemBlock := &pem.Block{
Type: "PRIVATE KEY",
Bytes: keyPemBlock,
}
var buf bytes.Buffer
if err := pem.Encode(&buf, pemBlock); err != nil {
return nil, err
}
keyPemBlock = buf.Bytes()
}
cert, err := tls.X509KeyPair(certPemBlock, keyPemBlock)
if err != nil {
return nil, err
}
return &cert, nil
}
+10
View File
@@ -0,0 +1,10 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package topdown provides low-level query evaluation support.
//
// The topdown implementation is a modified version of the standard top-down
// evaluation algorithm used in Datalog. References and comprehensions are
// evaluated eagerly while all other terms are evaluated lazily.
package topdown
+394
View File
@@ -0,0 +1,394 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"net/url"
"strings"
"sigs.k8s.io/yaml"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/util"
)
func builtinJSONMarshal(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
asJSON, err := ast.JSON(operands[0].Value)
if err != nil {
return err
}
bs, err := json.Marshal(asJSON)
if err != nil {
return err
}
return iter(ast.StringTerm(util.ByteSliceToString(bs)))
}
func builtinJSONMarshalWithOpts(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
asJSON, err := ast.JSON(operands[0].Value)
if err != nil {
return err
}
indentWith := "\t"
prefixWith := ""
implicitPrettyPrint := false
userDeclaredExplicitPrettyPrint := false
shouldPrettyPrint := false
marshalOpts, err := builtins.ObjectOperand(operands[1].Value, 2)
if err != nil {
return err
}
for idx, k := range marshalOpts.Keys() {
val := marshalOpts.Get(k)
key, err := builtins.StringOperand(k.Value, idx)
if err != nil {
return builtins.NewOperandErr(2, "failed to stringify key %v at index %d: %v", k, idx, err)
}
switch key {
case "prefix":
prefixOpt, err := builtins.StringOperand(val.Value, idx)
if err != nil {
return builtins.NewOperandErr(2, "key %s failed cast to string: %v", key, err)
}
prefixWith = string(prefixOpt)
implicitPrettyPrint = true
case "indent":
indentOpt, err := builtins.StringOperand(val.Value, idx)
if err != nil {
return builtins.NewOperandErr(2, "key %s failed cast to string: %v", key, err)
}
indentWith = string(indentOpt)
implicitPrettyPrint = true
case "pretty":
userDeclaredExplicitPrettyPrint = true
explicitPrettyPrint, ok := val.Value.(ast.Boolean)
if !ok {
return builtins.NewOperandErr(2, "key %s failed cast to bool", key)
}
shouldPrettyPrint = bool(explicitPrettyPrint)
default:
return builtins.NewOperandErr(2, "object contained unknown key %s", key)
}
}
if !userDeclaredExplicitPrettyPrint {
shouldPrettyPrint = implicitPrettyPrint
}
var bs []byte
if shouldPrettyPrint {
bs, err = json.MarshalIndent(asJSON, prefixWith, indentWith)
} else {
bs, err = json.Marshal(asJSON)
}
if err != nil {
return err
}
s := util.ByteSliceToString(bs)
if shouldPrettyPrint {
// json.MarshalIndent() function will not prefix the first line of emitted JSON
return iter(ast.StringTerm(prefixWith + s))
}
return iter(ast.StringTerm(s))
}
func builtinJSONUnmarshal(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
var x any
if err := util.UnmarshalJSON(bs, &x); err != nil {
return err
}
v, err := ast.InterfaceToValue(x)
if err != nil {
return err
}
return iter(ast.NewTerm(v))
}
func builtinJSONIsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return iter(ast.InternedTerm(false))
}
return iter(ast.InternedTerm(json.Valid(bs)))
}
func builtinBase64Encode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(base64.StdEncoding.EncodeToString(bs)))
}
func builtinBase64Decode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
result, err := base64.StdEncoding.DecodeString(string(str))
if err != nil {
return err
}
return iter(ast.InternedTerm(string(result)))
}
func builtinBase64IsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return iter(ast.InternedTerm(false))
}
_, err = base64.StdEncoding.DecodeString(string(str))
return iter(ast.InternedTerm(err == nil))
}
func builtinBase64UrlEncode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(base64.URLEncoding.EncodeToString(bs)))
}
func builtinBase64UrlEncodeNoPad(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(base64.RawURLEncoding.EncodeToString(bs)))
}
func builtinBase64UrlDecode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s := string(str)
// Some base64url encoders omit the padding at the end, so this case
// corrects such representations using the method given in RFC 7515
// Appendix C: https://tools.ietf.org/html/rfc7515#appendix-C
if !strings.HasSuffix(s, "=") {
switch len(s) % 4 {
case 0:
case 2:
s += "=="
case 3:
s += "="
default:
return fmt.Errorf("illegal base64url string: %s", s)
}
}
result, err := base64.URLEncoding.DecodeString(s)
if err != nil {
return err
}
return iter(ast.InternedTerm(string(result)))
}
func builtinURLQueryEncode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(url.QueryEscape(string(str))))
}
func builtinURLQueryDecode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s, err := url.QueryUnescape(string(str))
if err != nil {
return err
}
return iter(ast.StringTerm(s))
}
var encodeObjectErr = builtins.NewOperandErr(1, "values must be string, array[string], or set[string]")
func builtinURLQueryEncodeObject(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
asJSON, err := ast.JSON(operands[0].Value)
if err != nil {
return err
}
inputs, ok := asJSON.(map[string]any)
if !ok {
return builtins.NewOperandTypeErr(1, operands[0].Value, "object")
}
query := url.Values{}
for k, v := range inputs {
switch vv := v.(type) {
case string:
query.Set(k, vv)
case []any:
for _, val := range vv {
strVal, ok := val.(string)
if !ok {
return encodeObjectErr
}
query.Add(k, strVal)
}
default:
return encodeObjectErr
}
}
return iter(ast.StringTerm(query.Encode()))
}
func builtinURLQueryDecodeObject(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
query, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
queryParams, err := url.ParseQuery(string(query))
if err != nil {
return err
}
queryObject := ast.NewObjectWithCapacity(len(queryParams))
for k, v := range queryParams {
paramsArray := make([]*ast.Term, len(v))
for i, param := range v {
paramsArray[i] = ast.StringTerm(param)
}
queryObject.Insert(ast.StringTerm(k), ast.ArrayTerm(paramsArray...))
}
return iter(ast.NewTerm(queryObject))
}
func builtinYAMLMarshal(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
asJSON, err := ast.JSON(operands[0].Value)
if err != nil {
return err
}
bs, err := yaml.Marshal(asJSON)
if err != nil {
return err
}
return iter(ast.StringTerm(util.ByteSliceToString(bs)))
}
func builtinYAMLUnmarshal(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
js, err := yaml.YAMLToJSON(bs)
if err != nil {
return err
}
reader := ast.BytesReaderPool.Get()
defer ast.BytesReaderPool.Put(reader)
reader.Reset(js)
var val any
if err = util.NewJSONDecoder(reader).Decode(&val); err != nil {
return err
}
v, err := ast.InterfaceToValue(val)
if err != nil {
return err
}
return iter(ast.NewTerm(v))
}
func builtinYAMLIsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return iter(ast.InternedTerm(false))
}
var x any
err = yaml.Unmarshal(bs, &x)
return iter(ast.InternedTerm(err == nil))
}
func builtinHexEncode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
bs, err := builtins.StringOperandByteSlice(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(hex.EncodeToString(bs)))
}
func builtinHexDecode(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
val, err := hex.DecodeString(string(str))
if err != nil {
return err
}
return iter(ast.StringTerm(string(val)))
}
func init() {
RegisterBuiltinFunc(ast.JSONMarshal.Name, builtinJSONMarshal)
RegisterBuiltinFunc(ast.JSONMarshalWithOptions.Name, builtinJSONMarshalWithOpts)
RegisterBuiltinFunc(ast.JSONUnmarshal.Name, builtinJSONUnmarshal)
RegisterBuiltinFunc(ast.JSONIsValid.Name, builtinJSONIsValid)
RegisterBuiltinFunc(ast.Base64Encode.Name, builtinBase64Encode)
RegisterBuiltinFunc(ast.Base64Decode.Name, builtinBase64Decode)
RegisterBuiltinFunc(ast.Base64IsValid.Name, builtinBase64IsValid)
RegisterBuiltinFunc(ast.Base64UrlEncode.Name, builtinBase64UrlEncode)
RegisterBuiltinFunc(ast.Base64UrlEncodeNoPad.Name, builtinBase64UrlEncodeNoPad)
RegisterBuiltinFunc(ast.Base64UrlDecode.Name, builtinBase64UrlDecode)
RegisterBuiltinFunc(ast.URLQueryDecode.Name, builtinURLQueryDecode)
RegisterBuiltinFunc(ast.URLQueryEncode.Name, builtinURLQueryEncode)
RegisterBuiltinFunc(ast.URLQueryEncodeObject.Name, builtinURLQueryEncodeObject)
RegisterBuiltinFunc(ast.URLQueryDecodeObject.Name, builtinURLQueryDecodeObject)
RegisterBuiltinFunc(ast.YAMLMarshal.Name, builtinYAMLMarshal)
RegisterBuiltinFunc(ast.YAMLUnmarshal.Name, builtinYAMLUnmarshal)
RegisterBuiltinFunc(ast.YAMLIsValid.Name, builtinYAMLIsValid)
RegisterBuiltinFunc(ast.HexEncode.Name, builtinHexEncode)
RegisterBuiltinFunc(ast.HexDecode.Name, builtinHexDecode)
}
+163
View File
@@ -0,0 +1,163 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/util"
)
// Halt is a special error type that built-in function implementations return to indicate
// that policy evaluation should stop immediately.
type Halt struct {
Err error
}
func (h Halt) Error() string {
return h.Err.Error()
}
func (h Halt) Unwrap() error { return h.Err }
// Error is the error type returned by the Eval and Query functions when
// an evaluation error occurs.
type Error struct {
Code string `json:"code"`
Message string `json:"message"`
Location *ast.Location `json:"location,omitempty"`
err error `json:"-"`
}
const (
// InternalErr represents an unknown evaluation error.
InternalErr string = "eval_internal_error"
// CancelErr indicates the evaluation process was cancelled.
CancelErr string = "eval_cancel_error"
// ConflictErr indicates a conflict was encountered during evaluation. For
// instance, a conflict occurs if a rule produces multiple, differing values
// for the same key in an object. Conflict errors indicate the policy does
// not account for the data loaded into the policy engine.
ConflictErr string = "eval_conflict_error"
// TypeErr indicates evaluation stopped because an expression was applied to
// a value of an inappropriate type.
TypeErr string = "eval_type_error"
// BuiltinErr indicates a built-in function received a semantically invalid
// input or encountered some kind of runtime error, e.g., connection
// timeout, connection refused, etc.
BuiltinErr string = "eval_builtin_error"
// WithMergeErr indicates that the real and replacement data could not be merged.
WithMergeErr string = "eval_with_merge_error"
)
// IsError returns true if the err is an Error.
func IsError(err error) bool {
var e *Error
return errors.As(err, &e)
}
// IsCancel returns true if err was caused by cancellation.
func IsCancel(err error) bool {
return errors.Is(err, &Error{Code: CancelErr})
}
// Is allows matching topdown errors using errors.Is (see IsCancel).
func (e *Error) Is(target error) bool {
var t *Error
if errors.As(target, &t) {
return (t.Code == "" || e.Code == t.Code) &&
(t.Message == "" || e.Message == t.Message) &&
(t.Location == nil || t.Location.Compare(e.Location) == 0)
}
return false
}
func (e *Error) Error() string {
buf, _ := e.AppendText(make([]byte, 0, e.StringLength()))
return util.ByteSliceToString(buf)
}
func (e *Error) AppendText(buf []byte) ([]byte, error) {
if e.Location != nil {
buf, _ := e.Location.AppendText(buf)
buf = append(append(buf, ": "...), e.Code...)
buf = append(append(buf, ": "...), e.Message...)
return buf, nil
}
return append(append(append(buf, e.Code...), ": "...), e.Message...), nil
}
func (e *Error) StringLength() int {
l := len(e.Code) + 2 + len(e.Message)
if e.Location != nil {
l += e.Location.StringLength() + 2
}
return l
}
func (e *Error) Wrap(err error) *Error {
e.err = err
return e
}
func (e *Error) Unwrap() error {
return e.err
}
func functionConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "functions must not produce multiple outputs for same inputs",
}
}
func completeDocConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "complete rules must not produce multiple outputs",
}
}
func objectDocKeyConflictErr(loc *ast.Location) error {
return &Error{
Code: ConflictErr,
Location: loc,
Message: "object keys must be unique",
}
}
func unsupportedBuiltinErr(loc *ast.Location, name string) error {
return &Error{
Code: InternalErr,
Location: loc,
Message: "unsupported built-in: " + name,
}
}
func mergeConflictErr(loc *ast.Location) error {
return &Error{
Code: WithMergeErr,
Location: loc,
Message: "real and replacement data could not be merged",
}
}
func internalErr(loc *ast.Location, msg string) error {
return &Error{
Code: InternalErr,
Location: loc,
Message: msg,
}
}
File diff suppressed because it is too large Load Diff
+127
View File
@@ -0,0 +1,127 @@
package topdown
import (
"strings"
"sync"
"github.com/gobwas/glob"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
const globCacheMaxSize = 100
const globInterQueryValueCacheHits = "rego_builtin_glob_interquery_value_cache_hits"
var noDelimiters = []rune{}
var dotDelimiters = []rune{'.'}
var globCacheLock = sync.RWMutex{}
var globCache = map[string]glob.Glob{}
func builtinGlobMatch(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
pattern, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
var delimiters []rune
switch operands[1].Value.(type) {
case ast.Null:
delimiters = noDelimiters
case *ast.Array:
delimiters, err = builtins.RuneSliceOperand(operands[1].Value, 2)
if err != nil {
return err
}
if len(delimiters) == 0 {
delimiters = dotDelimiters
}
default:
return builtins.NewOperandTypeErr(2, operands[1].Value, "array", "null")
}
match, err := builtins.StringOperand(operands[2].Value, 3)
if err != nil {
return err
}
builder := strings.Builder{}
builder.WriteString(string(pattern))
builder.WriteRune('-')
for _, v := range delimiters {
builder.WriteRune(v)
}
id := builder.String()
m, err := globCompileAndMatch(bctx, id, string(pattern), string(match), delimiters)
if err != nil {
return err
}
return iter(ast.InternedTerm(m))
}
func globCompileAndMatch(bctx BuiltinContext, id, pattern, match string, delimiters []rune) (bool, error) {
if bctx.InterQueryBuiltinValueCache != nil {
// TODO: Use named cache
val, ok := bctx.InterQueryBuiltinValueCache.Get(ast.String(id))
if ok {
pat, valid := val.(glob.Glob)
if !valid {
// The cache key may exist for a different value type (eg. regex).
// In this case, we calculate the glob and return the result w/o updating the cache.
var err error
if pat, err = glob.Compile(pattern, delimiters...); err != nil {
return false, err
}
return pat.Match(match), nil
}
bctx.Metrics.Counter(globInterQueryValueCacheHits).Incr()
out := pat.Match(match)
return out, nil
}
res, err := glob.Compile(pattern, delimiters...)
if err != nil {
return false, err
}
bctx.InterQueryBuiltinValueCache.Insert(ast.String(id), res)
return res.Match(match), nil
}
globCacheLock.RLock()
p, ok := globCache[id]
globCacheLock.RUnlock()
if !ok {
var err error
if p, err = glob.Compile(pattern, delimiters...); err != nil {
return false, err
}
globCacheLock.Lock()
if len(globCache) >= globCacheMaxSize {
// Delete a (semi-)random key to make room for the new one.
for k := range globCache {
delete(globCache, k)
break
}
}
globCache[id] = p
globCacheLock.Unlock()
}
return p.Match(match), nil
}
func builtinGlobQuoteMeta(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
pattern, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.StringTerm(glob.QuoteMeta(string(pattern))))
}
func init() {
RegisterBuiltinFunc(ast.GlobMatch.Name, builtinGlobMatch)
RegisterBuiltinFunc(ast.GlobQuoteMeta.Name, builtinGlobQuoteMeta)
}
+688
View File
@@ -0,0 +1,688 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/json"
"fmt"
"strconv"
"strings"
gqlast "github.com/vektah/gqlparser/v2/ast"
gqlparser "github.com/vektah/gqlparser/v2/parser"
gqlvalidator "github.com/vektah/gqlparser/v2/validator"
"github.com/vektah/gqlparser/v2/validator/rules"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/topdown/cache"
)
var defaultRules = rules.NewDefaultRules()
// Parses a GraphQL schema, and returns the GraphQL AST for the schema.
func parseSchema(schema string) (*gqlast.SchemaDocument, error) {
// NOTE(philipc): We don't include the "built-in schema defs" from the
// underlying graphql parsing library here, because those definitions
// generate enormous AST blobs. In the future, if there is demand for
// a "full-spec" version of schema ASTs, we may need to provide a
// version of this function that includes the built-in schema
// definitions.
schemaAST, err := gqlparser.ParseSchema(&gqlast.Source{Input: schema})
if err != nil {
return nil, formatGqlParserError(err)
}
return schemaAST, nil
}
// Parses a GraphQL query, and returns the GraphQL AST for the query.
func parseQuery(query string) (*gqlast.QueryDocument, error) {
queryAST, err := gqlparser.ParseQuery(&gqlast.Source{Input: query})
if err != nil {
return nil, formatGqlParserError(err)
}
return queryAST, nil
}
// Validates a GraphQL query against a schema, and returns an error.
// In this case, we get a wrappered error list type, and pluck out
// just the first error message in the list.
func validateQuery(schema *gqlast.Schema, query *gqlast.QueryDocument) error {
// Validate the query against the schema, erroring if there's an issue.
if err := gqlvalidator.ValidateWithRules(schema, query, defaultRules); err != nil {
return formatGqlParserError(err)
}
return nil
}
func getBuiltinSchema() *gqlast.SchemaDocument {
schema, err := gqlparser.ParseSchema(gqlvalidator.Prelude)
if err != nil {
panic(fmt.Errorf("Error in gqlparser Prelude (should be impossible): %w", err))
}
return schema
}
// NOTE(philipc): This function expects *validated* schema documents, and will break
// if it is fed arbitrary structures.
func mergeSchemaDocuments(docA *gqlast.SchemaDocument, docB *gqlast.SchemaDocument) *gqlast.SchemaDocument {
ast := &gqlast.SchemaDocument{}
ast.Merge(docA)
ast.Merge(docB)
return ast
}
// Converts a SchemaDocument into a gqlast.Schema object that can be used for validation.
// It merges in the builtin schema typedefs exactly as gqltop.LoadSchema did internally.
func convertSchema(schemaDoc *gqlast.SchemaDocument) (*gqlast.Schema, error) {
// Merge builtin schema + schema we were provided.
builtinsSchemaDoc := getBuiltinSchema()
mergedSchemaDoc := mergeSchemaDocuments(builtinsSchemaDoc, schemaDoc)
schema, err := gqlvalidator.ValidateSchemaDocument(mergedSchemaDoc)
if err != nil {
return nil, fmt.Errorf("Error in gqlparser SchemaDocument to Schema conversion: %w", err)
}
return schema, nil
}
// Converts an ast.Object into a gqlast.QueryDocument object.
func objectToQueryDocument(value ast.Object) (*gqlast.QueryDocument, error) {
// Convert ast.Term to any for JSON encoding below.
asJSON, err := ast.JSON(value)
if err != nil {
return nil, err
}
// Marshal to JSON.
bs, err := json.Marshal(asJSON)
if err != nil {
return nil, err
}
// Unmarshal from JSON -> gqlast.QueryDocument.
var result gqlast.QueryDocument
err = json.Unmarshal(bs, &result)
if err != nil {
return nil, err
}
return &result, nil
}
// Converts an ast.Object into a gqlast.SchemaDocument object.
func objectToSchemaDocument(value ast.Object) (*gqlast.SchemaDocument, error) {
// Convert ast.Term to any for JSON encoding below.
asJSON, err := ast.JSON(value)
if err != nil {
return nil, err
}
// Marshal to JSON.
bs, err := json.Marshal(asJSON)
if err != nil {
return nil, err
}
// Unmarshal from JSON -> gqlast.SchemaDocument.
var result gqlast.SchemaDocument
err = json.Unmarshal(bs, &result)
if err != nil {
return nil, err
}
return &result, nil
}
// Recursively traverses an AST that has been run through InterfaceToValue,
// and prunes away the fields with null or empty values, and all `Position`
// structs.
// NOTE(philipc): We currently prune away null values to reduce the level
// of clutter in the returned AST objects. In the future, if there is demand
// for ASTs that have a more regular/fixed structure, we may need to provide
// a "raw" version of the AST, where we still prune away the `Position`
// structs, but leave in the null fields.
func pruneIrrelevantGraphQLASTNodes(value ast.Value) ast.Value {
// We iterate over the Value we've been provided, and recurse down
// in the case of complex types, such as Arrays/Objects.
// We are guaranteed to only have to deal with standard JSON types,
// so this is much less ugly than what we'd need for supporting every
// extant ast type!
switch x := value.(type) {
case *ast.Array:
result := ast.NewArrayWithCapacity(x.Len())
// Iterate over the array's elements, and do the following:
// - Drop any Nulls
// - Drop any any empty object/array value (after running the pruner)
for i := range x.Len() {
vTerm := x.Elem(i)
switch v := vTerm.Value.(type) {
case ast.Null:
continue
case *ast.Array:
// Safe, because we knew the type before going to prune it.
va := pruneIrrelevantGraphQLASTNodes(v).(*ast.Array)
if va.Len() > 0 {
result = result.Append(ast.NewTerm(va))
}
case ast.Object:
// Safe, because we knew the type before going to prune it.
vo := pruneIrrelevantGraphQLASTNodes(v).(ast.Object)
if vo.Len() > 0 {
result = result.Append(ast.NewTerm(vo))
}
default:
result = result.Append(vTerm)
}
}
return result
case ast.Object:
result := ast.NewObjectWithCapacity(x.Len())
// Iterate over our object's keys, and do the following:
// - Drop "Position".
// - Drop any key with a Null value.
// - Drop any key with an empty object/array value (after running the pruner)
keys := x.Keys()
for _, k := range keys {
// We drop the "Position" objects because we don't need the
// source-backref/location info they provide for policy rules.
// Note that keys are ast.Strings.
if ast.String("Position").Equal(k.Value) {
continue
}
vTerm := x.Get(k)
switch v := vTerm.Value.(type) {
case ast.Null:
continue
case *ast.Array:
// Safe, because we knew the type before going to prune it.
va := pruneIrrelevantGraphQLASTNodes(v).(*ast.Array)
if va.Len() > 0 {
result.Insert(k, ast.NewTerm(va))
}
case ast.Object:
// Safe, because we knew the type before going to prune it.
vo := pruneIrrelevantGraphQLASTNodes(v).(ast.Object)
if vo.Len() > 0 {
result.Insert(k, ast.NewTerm(vo))
}
default:
result.Insert(k, vTerm)
}
}
return result
default:
return x
}
}
func formatGqlParserError(err error) error {
// We use strings.TrimSuffix to remove the '.' characters that the library
// authors include on most of their validation errors. This should be safe,
// since variable names in their error messages are usually quoted, and
// this affects only the last character(s) in the string.
// NOTE(philipc): We know the error location will be in the query string,
// because schema validation always happens before this function is called.
// NOTE(rm): gqlparser does not _always_ return the error location
// so only populate location if it is available
if err == nil {
return nil
}
// If the error contains location information, format it nicely
errorParts := strings.SplitN(err.Error(), ":", 4)
if len(errorParts) >= 4 {
row, err := strconv.ParseUint(errorParts[1], 10, 64)
if err == nil {
col, err := strconv.ParseUint(errorParts[2], 10, 64)
if err == nil {
msg := strings.TrimSuffix(strings.TrimLeft(errorParts[len(errorParts)-1], " "), ".\n")
return fmt.Errorf("%s in GraphQL string at location %d:%d", msg, row, col)
}
}
}
// Wrap and return the full error if location information is not available
return fmt.Errorf("GraphQL parse error: %w", err)
}
// Reports errors from parsing/validation.
func builtinGraphQLParse(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var queryDoc *gqlast.QueryDocument
var schemaDoc *gqlast.SchemaDocument
var schemaASTValue ast.Value
var querySchema ast.Value
var err error
// Parse/translate query if it's a string/object.
switch x := operands[0].Value.(type) {
case ast.String:
queryDoc, err = parseQuery(string(x))
case ast.Object:
queryDoc, err = objectToQueryDocument(x)
default:
// Error if wrong type.
return builtins.NewOperandTypeErr(0, x, "string", "object")
}
if err != nil {
return err
}
schemaCacheKey, schema := cacheGetSchema(bctx, operands[1])
schemaASTCacheKey, querySchema := cacheGetSchemaAST(bctx, operands[1])
if schema == nil || querySchema == nil {
// Parse/translate schema if it's a string/object.
switch x := operands[1].Value.(type) {
case ast.String:
schemaDoc, err = parseSchema(string(x))
case ast.Object:
schemaDoc, err = objectToSchemaDocument(x)
default:
// Error if wrong type.
return builtins.NewOperandTypeErr(1, x, "string", "object")
}
if err != nil {
return err
}
// Convert SchemaDoc to Object before validating and converting it to a Schema
// This precludes inclusion of extra definitions from the default GraphQL schema
if querySchema == nil {
schemaASTValue, err = ast.InterfaceToValue(schemaDoc)
if err != nil {
return err
}
querySchema = pruneIrrelevantGraphQLASTNodes(schemaASTValue.(ast.Object))
cacheInsertSchemaAST(bctx, schemaASTCacheKey, querySchema)
}
// Validate the query against the schema, erroring if there's an issue.
if schema == nil {
schema, err = convertSchema(schemaDoc)
if err != nil {
return err
}
cacheInsertSchema(bctx, schemaCacheKey, schema)
}
}
// Transform the ASTs into Objects.
queryASTValue, err := ast.InterfaceToValue(queryDoc)
if err != nil {
return err
}
if err := validateQuery(schema, queryDoc); err != nil {
return err
}
// Recursively remove irrelevant AST structures.
queryResult := pruneIrrelevantGraphQLASTNodes(queryASTValue.(ast.Object))
// Construct return value.
verified := ast.ArrayTerm(
ast.NewTerm(queryResult),
ast.NewTerm(querySchema),
)
return iter(verified)
}
// Returns default value when errors occur.
func builtinGraphQLParseAndVerify(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var queryDoc *gqlast.QueryDocument
var schemaDoc *gqlast.SchemaDocument
var schemaASTValue ast.Value
var querySchema ast.Value
var err error
unverified := ast.ArrayTerm(
ast.InternedTerm(false),
ast.NewTerm(ast.NewObject()),
ast.NewTerm(ast.NewObject()),
)
// Parse/translate query if it's a string/object.
switch x := operands[0].Value.(type) {
case ast.String:
queryDoc, err = parseQuery(string(x))
case ast.Object:
queryDoc, err = objectToQueryDocument(x)
default:
// Error if wrong type.
return iter(unverified)
}
if err != nil {
return iter(unverified)
}
// Transform the ASTs into Objects.
queryASTValue, err := ast.InterfaceToValue(queryDoc)
if err != nil {
return iter(unverified)
}
schemaCacheKey, schema := cacheGetSchema(bctx, operands[1])
schemaASTCacheKey, querySchema := cacheGetSchemaAST(bctx, operands[1])
if schema == nil || querySchema == nil {
// Parse/translate schema if it's a string/object.
switch x := operands[1].Value.(type) {
case ast.String:
schemaDoc, err = parseSchema(string(x))
case ast.Object:
schemaDoc, err = objectToSchemaDocument(x)
default:
// Error if wrong type.
return iter(unverified)
}
if err != nil {
return iter(unverified)
}
// Convert SchemaDoc to Object before validating and converting it to a Schema
// This precludes inclusion of extra definitions from the default GraphQL schema
if querySchema == nil {
schemaASTValue, err = ast.InterfaceToValue(schemaDoc)
if err != nil {
return iter(unverified)
}
querySchema = pruneIrrelevantGraphQLASTNodes(schemaASTValue.(ast.Object))
cacheInsertSchemaAST(bctx, schemaASTCacheKey, querySchema)
}
if schema == nil {
schema, err = convertSchema(schemaDoc)
if err != nil {
return iter(unverified)
}
cacheInsertSchema(bctx, schemaCacheKey, schema)
}
}
// Validate the query against the schema, erroring if there's an issue.
if err := validateQuery(schema, queryDoc); err != nil {
return iter(unverified)
}
// Recursively remove irrelevant AST structures.
queryResult := pruneIrrelevantGraphQLASTNodes(queryASTValue.(ast.Object))
// Construct return value.
verified := ast.ArrayTerm(
ast.InternedTerm(true),
ast.NewTerm(queryResult),
ast.NewTerm(querySchema),
)
return iter(verified)
}
func builtinGraphQLParseQuery(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
raw, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Get the highly-nested AST struct, along with any errors generated.
query, err := parseQuery(string(raw))
if err != nil {
return err
}
// Transform the AST into an Object.
value, err := ast.InterfaceToValue(query)
if err != nil {
return err
}
// Recursively remove irrelevant AST structures.
result := pruneIrrelevantGraphQLASTNodes(value.(ast.Object))
return iter(ast.NewTerm(result))
}
func builtinGraphQLParseSchema(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
schemaDocCacheKey, schemaDoc := cacheGetSchemaDoc(bctx, operands[0])
if schemaDoc == nil {
raw, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Get the highly-nested AST struct, along with any errors generated.
schemaDoc, err = parseSchema(string(raw))
if err != nil {
return err
}
// Note SchemaDoc is not validated
cacheInsertSchemaDoc(bctx, schemaDocCacheKey, schemaDoc)
}
schemaASTCacheKey, schemaAST := cacheGetSchemaAST(bctx, operands[0])
if schemaAST == nil {
// Transform the AST into an Object.
value, err := ast.InterfaceToValue(schemaDoc)
if err != nil {
return err
}
// Recursively remove irrelevant AST structures.
schemaAST = pruneIrrelevantGraphQLASTNodes(value.(ast.Object))
cacheInsertSchemaAST(bctx, schemaASTCacheKey, schemaAST)
}
return iter(ast.NewTerm(schemaAST))
}
func builtinGraphQLIsValid(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var queryDoc *gqlast.QueryDocument
var schemaDoc *gqlast.SchemaDocument
var schema *gqlast.Schema
var err error
switch x := operands[0].Value.(type) {
case ast.String:
queryDoc, err = parseQuery(string(x))
case ast.Object:
queryDoc, err = objectToQueryDocument(x)
default:
// Error if wrong type.
return iter(ast.InternedTerm(false))
}
if err != nil {
return iter(ast.InternedTerm(false))
}
schemaCacheKey, schema := cacheGetSchema(bctx, operands[1])
if schema == nil {
switch x := operands[1].Value.(type) {
case ast.String:
schemaDoc, err = parseSchema(string(x))
case ast.Object:
schemaDoc, err = objectToSchemaDocument(x)
default:
// Error if wrong type.
return iter(ast.InternedTerm(false))
}
if err != nil {
return iter(ast.InternedTerm(false))
}
// Validate the query against the schema, erroring if there's an issue.
schema, err = convertSchema(schemaDoc)
if err != nil {
return iter(ast.InternedTerm(false))
}
cacheInsertSchema(bctx, schemaCacheKey, schema)
}
if err := validateQuery(schema, queryDoc); err != nil {
return iter(ast.InternedTerm(false))
}
// If we got this far, the GraphQL query passed validation.
return iter(ast.InternedTerm(true))
}
func builtinGraphQLSchemaIsValid(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var err error
// Schemas are only cached if they are valid
schemaCacheKey, schema := cacheGetSchema(bctx, operands[0])
if schema == nil {
var schemaDoc *gqlast.SchemaDocument
var validatedSchema *gqlast.Schema
switch x := operands[0].Value.(type) {
case ast.String:
schemaDoc, err = parseSchema(string(x))
case ast.Object:
schemaDoc, err = objectToSchemaDocument(x)
default:
// Error if wrong type.
return iter(ast.InternedTerm(false))
}
if err != nil {
return iter(ast.InternedTerm(false))
}
// Validate the schema, this determines the result
// and whether there is a schema to cache
validatedSchema, err = convertSchema(schemaDoc)
if err == nil {
cacheInsertSchema(bctx, schemaCacheKey, validatedSchema)
}
}
return iter(ast.InternedTerm(err == nil))
}
// Insert Schema into cache
func cacheInsertSchema(bctx BuiltinContext, key string, schema *gqlast.Schema) {
if bctx.InterQueryBuiltinValueCache == nil || key == "" {
return
}
cacheKey := ast.String(key)
c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName)
if c == nil {
return
}
c.Insert(cacheKey, schema)
}
// Insert SchemaAST into cache
func cacheInsertSchemaAST(bctx BuiltinContext, key string, schemaAST ast.Value) {
if bctx.InterQueryBuiltinValueCache == nil || key == "" {
return
}
cacheKeyAST := ast.String(key)
c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName)
if c == nil {
return
}
c.Insert(cacheKeyAST, schemaAST)
}
// Insert SchemaDocument into cache
func cacheInsertSchemaDoc(bctx BuiltinContext, key string, schemaDoc *gqlast.SchemaDocument) {
if bctx.InterQueryBuiltinValueCache == nil || key == "" {
return
}
cacheKey := ast.String(key)
c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName)
if c == nil {
return
}
c.Insert(cacheKey, schemaDoc)
}
// Returns the cache key and a Schema if this key already exists in the cache
func cacheGetSchema(bctx BuiltinContext, t *ast.Term) (string, *gqlast.Schema) {
if bctx.InterQueryBuiltinValueCache != nil {
if c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName); c != nil {
if key, keyOk := cacheKeyWithPrefix(bctx, t, "gql_schema-"); keyOk {
if val, ok := c.Get(ast.String(key)); ok {
if schema, isSchema := val.(*gqlast.Schema); isSchema {
return key, schema
}
}
return key, nil
}
}
}
return "", nil
}
// Returns the cache key and a SchemaDocument if this key already exists in the cache
// Note: the SchemaDocument is not a validated Schema
func cacheGetSchemaDoc(bctx BuiltinContext, t *ast.Term) (string, *gqlast.SchemaDocument) {
if bctx.InterQueryBuiltinValueCache != nil {
if c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName); c != nil {
if key, keyOk := cacheKeyWithPrefix(bctx, t, "gql_schema_doc-"); keyOk {
if val, ok := c.Get(ast.String(key)); ok {
if schemaDoc, isSchemaDoc := val.(*gqlast.SchemaDocument); isSchemaDoc {
return key, schemaDoc
}
}
return key, nil
}
}
}
return "", nil
}
// Returns the cache key and a SchemaDocument if this key already exists in the cache
// Note: the AST should be pruned
func cacheGetSchemaAST(bctx BuiltinContext, t *ast.Term) (string, ast.Value) {
if bctx.InterQueryBuiltinValueCache != nil {
if c := bctx.InterQueryBuiltinValueCache.GetCache(gqlCacheName); c != nil {
if key, keyOk := cacheKeyWithPrefix(bctx, t, "gql_schema_ast-"); keyOk {
if val, ok := c.Get(ast.String(key)); ok {
if schemaAST, isSchemaAST := val.(ast.Value); isSchemaAST {
return key, schemaAST
}
}
return key, nil
}
}
}
return "", nil
}
// Compute a constant size key for use with the cache
func cacheKeyWithPrefix(bctx BuiltinContext, t *ast.Term, prefix string) (string, bool) {
var cacheKey ast.String
var ok = false
if bctx.InterQueryBuiltinValueCache != nil {
switch t.Value.(type) {
case ast.String:
err := builtinCryptoSha256(bctx, []*ast.Term{t}, func(term *ast.Term) error {
cacheKey = term.Value.(ast.String)
return nil
})
ok = (len(cacheKey) > 0) && (err == nil)
case ast.Object:
objTerm := ast.StringTerm(t.String())
err := builtinCryptoSha256(bctx, []*ast.Term{objTerm}, func(term *ast.Term) error {
cacheKey = term.Value.(ast.String)
return nil
})
ok = (len(cacheKey) > 0) && (err == nil)
default:
ok = false
}
}
return prefix + string(cacheKey), ok
}
const gqlCacheName = "graphql"
func init() {
var defaultCacheEntries = 10
var graphqlCacheConfig = cache.NamedValueCacheConfig{
MaxNumEntries: &defaultCacheEntries,
}
cache.RegisterDefaultInterQueryBuiltinValueCacheConfig(gqlCacheName, &graphqlCacheConfig)
RegisterBuiltinFunc(ast.GraphQLParse.Name, builtinGraphQLParse)
RegisterBuiltinFunc(ast.GraphQLParseAndVerify.Name, builtinGraphQLParseAndVerify)
RegisterBuiltinFunc(ast.GraphQLParseQuery.Name, builtinGraphQLParseQuery)
RegisterBuiltinFunc(ast.GraphQLParseSchema.Name, builtinGraphQLParseSchema)
RegisterBuiltinFunc(ast.GraphQLIsValid.Name, builtinGraphQLIsValid)
RegisterBuiltinFunc(ast.GraphQLSchemaIsValid.Name, builtinGraphQLSchemaIsValid)
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,7 @@
//go:build !darwin
package topdown
func fixupDarwinGo118(x string, _ string) string {
return x
}
@@ -0,0 +1,10 @@
package topdown
func fixupDarwinGo118(x, y string) string {
switch x {
case "x509: certificate signed by unknown authority":
return y
default:
return x
}
}
+100
View File
@@ -0,0 +1,100 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"github.com/open-policy-agent/opa/v1/ast"
)
var errBadPath = errors.New("bad document path")
func mergeTermWithValues(exist *ast.Term, pairs [][2]*ast.Term) (*ast.Term, error) {
var result *ast.Term
var init bool
for i, pair := range pairs {
if err := ast.IsValidImportPath(pair[0].Value); err != nil {
return nil, errBadPath
}
target := pair[0].Value.(ast.Ref)
// Copy the value if subsequent pairs in the slice would modify it.
for j := i + 1; j < len(pairs); j++ {
other := pairs[j][0].Value.(ast.Ref)
if len(other) > len(target) && other.HasPrefix(target) {
pair[1] = pair[1].Copy()
break
}
}
if len(target) == 1 {
result = pair[1]
init = true
} else {
if !init {
result = exist.Copy()
init = true
}
if result == nil {
result = ast.NewTerm(makeTree(target[1:], pair[1]))
} else {
node := result
done := false
for i := 1; i < len(target)-1 && !done; i++ {
obj, ok := node.Value.(ast.Object)
if !ok {
result = ast.NewTerm(makeTree(target[i:], pair[1]))
done = true
continue
}
if child := obj.Get(target[i]); !isObject(child) {
obj.Insert(target[i], ast.NewTerm(makeTree(target[i+1:], pair[1])))
done = true
} else { // child is object
node = child
}
}
if !done {
if obj, ok := node.Value.(ast.Object); ok {
obj.Insert(target[len(target)-1], pair[1])
} else {
result = ast.NewTerm(makeTree(target[len(target)-1:], pair[1]))
}
}
}
}
}
if !init {
result = exist
}
return result, nil
}
// makeTree returns an object that represents a document where the value v is
// the leaf and elements in k represent intermediate objects.
func makeTree(k ast.Ref, v *ast.Term) ast.Object {
var obj ast.Object
for i := len(k) - 1; i >= 1; i-- {
obj = ast.NewObject(ast.Item(k[i], v))
v = &ast.Term{Value: obj}
}
obj = ast.NewObject(ast.Item(k[0], v))
return obj
}
func isObject(x *ast.Term) bool {
if x == nil {
return false
}
_, ok := x.Value.(ast.Object)
return ok
}
@@ -0,0 +1,63 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import "github.com/open-policy-agent/opa/v1/metrics"
const (
evalOpPlug = "eval_op_plug"
evalOpResolve = "eval_op_resolve"
evalOpRuleIndex = "eval_op_rule_index"
evalOpBuiltinCall = "eval_op_builtin_call"
evalOpVirtualCacheHit = "eval_op_virtual_cache_hit"
evalOpVirtualCacheMiss = "eval_op_virtual_cache_miss"
evalOpBaseCacheHit = "eval_op_base_cache_hit"
evalOpBaseCacheMiss = "eval_op_base_cache_miss"
evalOpComprehensionCacheSkip = "eval_op_comprehension_cache_skip"
evalOpComprehensionCacheBuild = "eval_op_comprehension_cache_build"
evalOpComprehensionCacheHit = "eval_op_comprehension_cache_hit"
evalOpComprehensionCacheMiss = "eval_op_comprehension_cache_miss"
partialOpSaveUnify = "partial_op_save_unify"
partialOpSaveSetContains = "partial_op_save_set_contains"
partialOpSaveSetContainsRec = "partial_op_save_set_contains_rec"
partialOpCopyPropagation = "partial_op_copy_propagation"
)
// Instrumentation implements helper functions to instrument query evaluation
// to diagnose performance issues. Instrumentation may be expensive in some
// cases, so it is disabled by default.
type Instrumentation struct {
m metrics.Metrics
}
// NewInstrumentation returns a new Instrumentation object. Performance
// diagnostics recorded on this Instrumentation object will stored in m.
func NewInstrumentation(m metrics.Metrics) *Instrumentation {
return &Instrumentation{
m: m,
}
}
func (instr *Instrumentation) startTimer(name string) {
if instr == nil {
return
}
instr.m.Timer(name).Start()
}
func (instr *Instrumentation) stopTimer(name string) {
if instr == nil {
return
}
delta := instr.m.Timer(name).Stop()
instr.m.Histogram(name).Update(delta)
}
func (instr *Instrumentation) counterIncr(name string) {
if instr == nil {
return
}
instr.m.Counter(name).Incr()
}
+382
View File
@@ -0,0 +1,382 @@
// Copyright 2019 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"fmt"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/internal/edittree"
)
func builtinJSONRemove(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Expect an object and a string or array/set of strings
if _, err := builtins.ObjectOperand(operands[0].Value, 1); err != nil {
return err
}
// Build a list of json pointers to remove
paths, err := getJSONPaths(operands[1].Value)
if err != nil {
return err
}
newObj, err := jsonRemove(operands[0], ast.NewTerm(pathsToObject(paths)))
if err != nil {
return err
}
if newObj == nil {
return nil
}
return iter(newObj)
}
// jsonRemove returns a new term that is the result of walking
// through a and omitting removing any values that are in b but
// have ast.Null values (ie leaf nodes for b).
func jsonRemove(a *ast.Term, b *ast.Term) (*ast.Term, error) {
if b == nil {
// The paths diverged, return a
return a, nil
}
var bObj ast.Object
switch bValue := b.Value.(type) {
case ast.Object:
bObj = bValue
case ast.Null:
// Means we hit a leaf node on "b", dont add the value for a
return nil, nil
default:
// The paths diverged, return a
return a, nil
}
switch aValue := a.Value.(type) {
case ast.String, ast.Number, ast.Boolean, ast.Null:
return a, nil
case ast.Object:
newObj := ast.NewObjectWithCapacity(aValue.Len())
err := aValue.Iter(func(k *ast.Term, v *ast.Term) error {
// recurse and add the diff of sub objects as needed
diffValue, err := jsonRemove(v, bObj.Get(k))
if err != nil || diffValue == nil {
return err
}
newObj.Insert(k, diffValue)
return nil
})
if err != nil {
return nil, err
}
return ast.NewTerm(newObj), nil
case ast.Set:
newSet := ast.NewSetWithCapacity(aValue.Len())
err := aValue.Iter(func(v *ast.Term) error {
// recurse and add the diff of sub objects as needed
diffValue, err := jsonRemove(v, bObj.Get(v))
if err != nil || diffValue == nil {
return err
}
newSet.Add(diffValue)
return nil
})
if err != nil {
return nil, err
}
return ast.NewTerm(newSet), nil
case *ast.Array:
// When indexes are removed we shift left to close empty spots in the array
// as per the JSON patch spec.
newArraySlice := make([]*ast.Term, 0, aValue.Len())
for i := range aValue.Len() {
v := aValue.Elem(i)
// recurse and add the diff of sub objects as needed
// Note: Keys in b will be strings for the index, eg path /a/1/b => {"a": {"1": {"b": null}}}
diffValue, err := jsonRemove(v, bObj.Get(ast.InternedIntegerString(i)))
if err != nil {
return nil, err
}
if diffValue != nil {
newArraySlice = append(newArraySlice, diffValue)
}
}
return ast.ArrayTerm(newArraySlice...), nil
default:
return nil, fmt.Errorf("invalid value type %T", a)
}
}
func builtinJSONFilter(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Ensure we have the right parameters, expect an object and a string or array/set of strings
obj, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Build a list of filter strings
filters, err := getJSONPaths(operands[1].Value)
if err != nil {
return err
}
// Actually do the filtering
filterObj := pathsToObject(filters)
r, err := obj.Filter(filterObj)
if err != nil {
return err
}
return iter(ast.NewTerm(r))
}
func getJSONPaths(operand ast.Value) (paths []ast.Ref, err error) {
switch v := operand.(type) {
case *ast.Array:
paths = make([]ast.Ref, 0, v.Len())
for i := range v.Len() {
filter, err := parsePath(v.Elem(i))
if err != nil {
return nil, err
}
paths = append(paths, filter)
}
case ast.Set:
paths = make([]ast.Ref, 0, v.Len())
for _, item := range v.Slice() {
filter, err := parsePath(item)
if err != nil {
return nil, err
}
paths = append(paths, filter)
}
default:
return nil, builtins.NewOperandTypeErr(2, v, "set", "array")
}
return paths, nil
}
// parsePath parses a JSON pointer path or array of path segments into an ast.Ref.
func parsePath(path *ast.Term) (ast.Ref, error) {
// paths can either be a `/` separated json path or
// an array or set of values
var pathSegments ast.Ref
switch p := path.Value.(type) {
case ast.String:
if p == "" {
return ast.InternedEmptyRefValue.(ast.Ref), nil
}
s := strings.TrimLeft(string(p), "/")
n := strings.Count(s, "/") + 1
pathSegments = make(ast.Ref, 0, n)
part, remaining, found := strings.Cut(s, "/")
unescaped := strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~")
pathSegments = append(pathSegments, ast.InternedTerm(unescaped))
for found {
part, remaining, found = strings.Cut(remaining, "/")
unescaped := strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~")
pathSegments = append(pathSegments, ast.InternedTerm(unescaped))
}
case *ast.Array:
pathSegments = make(ast.Ref, 0, p.Len())
for i := range p.Len() {
pathSegments = append(pathSegments, p.Elem(i))
}
default:
return nil, builtins.NewOperandErr(2,
"must be one of {set, array} containing string paths or array of path segments but got "+ast.ValueName(p),
)
}
return pathSegments, nil
}
func pathsToObject(paths []ast.Ref) ast.Object {
root := ast.NewObjectWithCapacity(len(paths))
for _, path := range paths {
node := root
done := false
// If the path is an empty JSON path, skip all further processing.
if len(path) == 0 {
done = true
}
// Otherwise, we should have 1+ path segments to work with.
for i := 0; i < len(path)-1 && !done; i++ {
k := path[i]
child := node.Get(k)
if child == nil {
obj := ast.NewObject()
node.Insert(k, ast.NewTerm(obj))
node = obj
continue
}
switch v := child.Value.(type) {
case ast.Null:
done = true
case ast.Object:
node = v
default:
panic("unreachable")
}
}
if !done {
node.Insert(path[len(path)-1], ast.InternedNullTerm)
}
}
return root
}
func applyPatches(source *ast.Term, operations *ast.Array) (*ast.Term, error) {
et := edittree.EditTreeFromPool(source)
defer edittree.Dispose(et)
for i := range operations.Len() {
object, ok := operations.Elem(i).Value.(ast.Object)
if !ok {
return nil, errors.New("must be an array of JSON-Patch objects, but at least one element is not an object")
}
// Validate
if object.Get(ast.InternedTerm("path")) == nil {
return nil, errors.New("missing required attribute 'path'")
}
opTerm := object.Get(ast.InternedTerm("op"))
if opTerm == nil {
return nil, errors.New("missing required attribute 'op'")
}
opStr, ok := opTerm.Value.(ast.String)
if !ok {
return nil, errors.New("attribute 'op' must be a string but found: " + ast.ValueName(opTerm.Value))
}
path, err := parsePath(object.Get(ast.InternedTerm("path")))
if err != nil {
return nil, err
}
switch string(opStr) {
case "add":
value := object.Get(ast.InternedTerm("value"))
if value == nil {
return nil, errors.New("missing required attribute 'value'")
}
if _, err = et.InsertAtPath(path, value); err != nil {
return nil, err
}
case "remove":
if _, err = et.DeleteAtPath(path); err != nil {
return nil, err
}
case "replace":
if _, err = et.DeleteAtPath(path); err != nil {
return nil, err
}
value := object.Get(ast.InternedTerm("value"))
if value == nil {
return nil, errors.New("missing required attribute 'value'")
}
if _, err = et.InsertAtPath(path, value); err != nil {
return nil, err
}
case "move":
fromValue := object.Get(ast.InternedTerm("from"))
if fromValue == nil {
return nil, errors.New("missing required attribute 'from'")
}
from, err := parsePath(fromValue)
if err != nil {
return nil, err
}
chunk, err := et.RenderAtPath(from)
if err != nil {
return nil, err
}
if _, err = et.DeleteAtPath(from); err != nil {
return nil, err
}
if _, err = et.InsertAtPath(path, chunk); err != nil {
return nil, err
}
case "copy":
fromValue := object.Get(ast.InternedTerm("from"))
if fromValue == nil {
return nil, errors.New("missing required attribute 'from'")
}
from, err := parsePath(fromValue)
if err != nil {
return nil, err
}
chunk, err := et.RenderAtPath(from)
if err != nil {
return nil, err
}
if _, err = et.InsertAtPath(path, chunk); err != nil {
return nil, err
}
case "test":
chunk, err := et.RenderAtPath(path)
if err != nil {
return nil, err
}
value := object.Get(ast.InternedTerm("value"))
if value == nil {
return nil, errors.New("missing required attribute 'value'")
}
if !chunk.Equal(value) {
return nil, fmt.Errorf("value from EditTree != patch value.\n\nExpected: %v\n\nFound: %v", value, chunk)
}
default:
return nil, fmt.Errorf("unrecognized op: '%s'", string(opStr))
}
}
return et.Render(), nil
}
func builtinJSONPatch(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Expect an array of operations.
operations, err := builtins.ArrayOperand(operands[1].Value, 2)
if err != nil {
return err
}
// JSON patch supports arrays, objects as well as values as the target.
patched, err := applyPatches(operands[0], operations)
if err != nil {
return err
}
return iter(patched)
}
func init() {
for _, key := range []string{"op", "path", "from", "value", "add", "remove", "replace", "move", "copy", "test"} {
ast.InternStringTerm(key)
}
RegisterBuiltinFunc(ast.JSONFilter.Name, builtinJSONFilter)
RegisterBuiltinFunc(ast.JSONRemove.Name, builtinJSONRemove)
RegisterBuiltinFunc(ast.JSONPatch.Name, builtinJSONPatch)
}
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/json"
"errors"
"github.com/open-policy-agent/opa/internal/gojsonschema"
"github.com/open-policy-agent/opa/v1/ast"
)
// astValueToJSONSchemaLoader converts a value to JSON Loader.
// Value can be ast.String or ast.Object.
func astValueToJSONSchemaLoader(value ast.Value) (gojsonschema.JSONLoader, error) {
var loader gojsonschema.JSONLoader
var err error
// ast.Value type selector.
switch x := value.(type) {
case ast.String:
// In case of string pass it as is as a raw JSON string.
// Make pre-check that it's a valid JSON at all because gojsonschema won't do that.
if !json.Valid([]byte(x)) {
return nil, errors.New("invalid JSON string")
}
loader = gojsonschema.NewStringLoader(string(x))
case ast.Object, *ast.Array:
// In case of object serialize it to JSON representation.
var data any
data, err = ast.JSON(value)
if err != nil {
return nil, err
}
loader = gojsonschema.NewGoLoader(data)
default:
// Any other cases will produce an error.
return nil, errors.New("wrong type, expected string or object")
}
return loader, nil
}
func newResultTerm(valid bool, data *ast.Term) *ast.Term {
return ast.ArrayTerm(ast.InternedTerm(valid), data)
}
// builtinJSONSchemaVerify accepts 1 argument which can be string or object and checks if it is valid JSON schema.
// Returns array [false, <string>] with error string at index 1, or [true, ""] with empty string at index 1 otherwise.
func builtinJSONSchemaVerify(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Take first argument and make JSON Loader from it.
loader, err := astValueToJSONSchemaLoader(operands[0].Value)
if err != nil {
return iter(newResultTerm(false, ast.StringTerm("jsonschema: "+err.Error())))
}
// Check that schema is correct and parses without errors.
if _, err = gojsonschema.NewSchema(loader); err != nil {
return iter(newResultTerm(false, ast.StringTerm("jsonschema: "+err.Error())))
}
return iter(newResultTerm(true, ast.InternedNullTerm))
}
// builtinJSONMatchSchema accepts 2 arguments both can be string or object and verifies if the document matches the JSON schema.
// Returns an array where first element is a boolean indicating a successful match, and the second is an array of errors that is empty on success and populated on failure.
// In case of internal error returns empty array.
func builtinJSONMatchSchema(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var schema *gojsonschema.Schema
if bctx.InterQueryBuiltinValueCache != nil {
if val, ok := bctx.InterQueryBuiltinValueCache.Get(operands[1].Value); ok {
if s, isSchema := val.(*gojsonschema.Schema); isSchema {
schema = s
}
}
}
// Take first argument and make JSON Loader from it.
// This is a JSON document made from Rego JSON string or object.
documentLoader, err := astValueToJSONSchemaLoader(operands[0].Value)
if err != nil {
return err
}
if schema == nil {
// Take second argument and make JSON Loader from it.
// This is a JSON schema made from Rego JSON string or object.
schemaLoader, err := astValueToJSONSchemaLoader(operands[1].Value)
if err != nil {
return err
}
schema, err = gojsonschema.NewSchema(schemaLoader)
if err != nil {
return err
}
if bctx.InterQueryBuiltinValueCache != nil {
bctx.InterQueryBuiltinValueCache.Insert(operands[1].Value, schema)
}
}
// Use schema to validate document.
result, err := schema.Validate(documentLoader)
if err != nil {
return err
}
// In case of validation errors produce Rego array of objects to describe the errors.
arr := ast.NewArrayWithCapacity(len(result.Errors()))
for _, re := range result.Errors() {
o := ast.NewObject(
[...]*ast.Term{ast.StringTerm("error"), ast.StringTerm(re.String())},
[...]*ast.Term{ast.StringTerm("type"), ast.StringTerm(re.Type())},
[...]*ast.Term{ast.StringTerm("field"), ast.StringTerm(re.Field())},
[...]*ast.Term{ast.StringTerm("desc"), ast.StringTerm(re.Description())},
)
arr = arr.Append(ast.NewTerm(o))
}
return iter(newResultTerm(result.Valid(), ast.NewTerm(arr)))
}
func init() {
RegisterBuiltinFunc(ast.JSONSchemaVerify.Name, builtinJSONSchemaVerify)
RegisterBuiltinFunc(ast.JSONMatchSchema.Name, builtinJSONMatchSchema)
}
+64
View File
@@ -0,0 +1,64 @@
// Copyright 2021 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"net"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
type lookupIPAddrCacheKey string
// resolv is the same as net.DefaultResolver -- this is for mocking it out in tests
var resolv = &net.Resolver{}
func builtinLookupIPAddr(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
a, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
name := string(a)
err = verifyHost(bctx, name)
if err != nil {
return err
}
key := lookupIPAddrCacheKey(name)
if val, ok := bctx.Cache.Get(key); ok {
return iter(val.(*ast.Term))
}
addrs, err := resolv.LookupIPAddr(bctx.Context, name)
if err != nil {
// NOTE(sr): We can't do better than this right now, see https://github.com/golang/go/issues/36208
if strings.Contains(err.Error(), "operation was canceled") || strings.Contains(err.Error(), "i/o timeout") {
return Halt{
Err: &Error{
Code: CancelErr,
Message: ast.NetLookupIPAddr.Name + ": " + err.Error(),
Location: bctx.Location,
},
}
}
return err
}
ret := ast.NewSetWithCapacity(len(addrs))
for _, a := range addrs {
ret.Add(ast.StringTerm(a.String()))
}
t := ast.NewTerm(ret)
bctx.Cache.Put(key, t)
return iter(t)
}
func init() {
RegisterBuiltinFunc(ast.NetLookupIPAddr.Name, builtinLookupIPAddr)
}
+202
View File
@@ -0,0 +1,202 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"fmt"
"math/big"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
type randIntCachingKey string
var (
zero = big.NewInt(0)
one = big.NewInt(1)
)
func builtinNumbersRange(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
if canGenerateCheapRange(operands) {
return generateCheapRange(operands, 1, iter)
}
x, err := builtins.BigIntOperand(operands[0].Value, 1)
if err != nil {
return err
}
y, err := builtins.BigIntOperand(operands[1].Value, 2)
if err != nil {
return err
}
ast, err := generateRange(bctx, x, y, one, "numbers.range")
if err != nil {
return err
}
return iter(ast)
}
func builtinNumbersRangeStep(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
if canGenerateCheapRangeStep(operands) {
step, _ := builtins.IntOperand(operands[2].Value, 3)
if step <= 0 {
return errors.New("numbers.range_step: step must be a positive integer")
}
return generateCheapRange(operands, step, iter)
}
x, err := builtins.BigIntOperand(operands[0].Value, 1)
if err != nil {
return err
}
y, err := builtins.BigIntOperand(operands[1].Value, 2)
if err != nil {
return err
}
step, err := builtins.BigIntOperand(operands[2].Value, 3)
if err != nil {
return err
}
if step.Cmp(zero) <= 0 {
return errors.New("numbers.range_step: step must be a positive integer")
}
ast, err := generateRange(bctx, x, y, step, "numbers.range_step")
if err != nil {
return err
}
return iter(ast)
}
func canGenerateCheapRange(operands []*ast.Term) bool {
x, err := builtins.IntOperand(operands[0].Value, 1)
if err != nil || !ast.HasInternedIntNumberTerm(x) {
return false
}
y, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil || !ast.HasInternedIntNumberTerm(y) {
return false
}
return true
}
func canGenerateCheapRangeStep(operands []*ast.Term) bool {
if canGenerateCheapRange(operands) {
step, err := builtins.IntOperand(operands[2].Value, 3)
if err == nil && ast.HasInternedIntNumberTerm(step) {
return true
}
}
return false
}
func generateCheapRange(operands []*ast.Term, step int, iter func(*ast.Term) error) error {
x, err := builtins.IntOperand(operands[0].Value, 1)
if err != nil {
return err
}
y, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil {
return err
}
terms := make([]*ast.Term, 0, y+1)
if x <= y {
for i := x; i <= y; i += step {
terms = append(terms, ast.InternedTerm(i))
}
} else {
for i := x; i >= y; i -= step {
terms = append(terms, ast.InternedTerm(i))
}
}
return iter(ast.ArrayTerm(terms...))
}
func generateRange(bctx BuiltinContext, x *big.Int, y *big.Int, step *big.Int, funcName string) (*ast.Term, error) {
cmp := x.Cmp(y)
comp := func(i *big.Int, y *big.Int) bool { return i.Cmp(y) <= 0 }
iter := func(i *big.Int) *big.Int { return i.Add(i, step) }
if cmp > 0 {
comp = func(i *big.Int, y *big.Int) bool { return i.Cmp(y) >= 0 }
iter = func(i *big.Int) *big.Int { return i.Sub(i, step) }
}
result := ast.NewArray()
haltErr := Halt{
Err: &Error{
Code: CancelErr,
Message: funcName + ": timed out before generating all numbers in range",
},
}
for i := new(big.Int).Set(x); comp(i, y); i = iter(i) {
if bctx.Cancel != nil && bctx.Cancel.Cancelled() {
return nil, haltErr
}
result = result.Append(ast.NewTerm(builtins.IntToNumber(i)))
}
return ast.NewTerm(result), nil
}
func builtinRandIntn(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
strOp, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
n, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil {
return err
}
if n == 0 {
return iter(ast.InternedTerm(0))
}
if n < 0 {
n = -n
}
key := randIntCachingKey(fmt.Sprintf("%s-%d", strOp, n))
if val, ok := bctx.Cache.Get(key); ok {
return iter(val.(*ast.Term))
}
r, err := bctx.Rand()
if err != nil {
return err
}
result := ast.InternedTerm(r.Intn(n))
bctx.Cache.Put(key, result)
return iter(result)
}
func init() {
RegisterBuiltinFunc(ast.NumbersRange.Name, builtinNumbersRange)
RegisterBuiltinFunc(ast.NumbersRangeStep.Name, builtinNumbersRangeStep)
RegisterBuiltinFunc(ast.RandIntn.Name, builtinRandIntn)
}
+250
View File
@@ -0,0 +1,250 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/internal/ref"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinObjectUnion(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
objA, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
objB, err := builtins.ObjectOperand(operands[1].Value, 2)
if err != nil {
return err
}
if objA.Len() == 0 {
return iter(operands[1])
}
if objB.Len() == 0 {
return iter(operands[0])
}
if objA.Compare(objB) == 0 {
return iter(operands[0])
}
r := mergeWithOverwrite(objA, objB)
return iter(ast.NewTerm(r))
}
func builtinObjectUnionN(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Because we need merge-with-overwrite behavior, we can iterate
// back-to-front, and get a mostly correct set of key assignments that
// give us the "last assignment wins, with merges" behavior we want.
// However, if a non-object overwrites an object value anywhere in the
// chain of assignments for a key, we have to "freeze" that key to
// prevent accidentally picking up nested objects that could merge with
// it from earlier in the input array.
// Example:
// Input: [{"a": {"b": 2}}, {"a": 4}, {"a": {"c": 3}}]
// Want Output: {"a": {"c": 3}}
// First pass: count total keys for pre-allocation
totalSize := 0
for i := range arr.Len() {
o, ok := arr.Elem(i).Value.(ast.Object)
if !ok {
return builtins.NewOperandElementErr(1, arr, arr.Elem(i).Value, "object")
}
totalSize += o.Len()
}
result := ast.NewObjectWithCapacity(totalSize)
frozenKeys := make(map[*ast.Term]struct{}, totalSize)
for i := arr.Len() - 1; i >= 0; i-- {
o := arr.Elem(i).Value.(ast.Object) // Already validated above
mergewithOverwriteInPlace(result, o, frozenKeys)
}
return iter(ast.NewTerm(result))
}
func builtinObjectRemove(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Expect an object and an array/set/object of keys
obj, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Build a set of keys to remove
keysToRemove, err := getObjectKeysParam(operands[1].Value)
if err != nil {
return err
}
// Pre-allocate with obj size (upper bound for result)
r := ast.NewObjectWithCapacity(obj.Len())
obj.Foreach(func(key *ast.Term, value *ast.Term) {
if !keysToRemove.Contains(key) {
r.Insert(key, value)
}
})
return iter(ast.NewTerm(r))
}
func builtinObjectFilter(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Expect an object and an array/set/object of keys
obj, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// Build a new object from the supplied filter keys
keys, err := getObjectKeysParam(operands[1].Value)
if err != nil {
return err
}
// Pre-allocate with keys size (upper bound for filter object)
filterObj := ast.NewObjectWithCapacity(keys.Len())
keys.Foreach(func(key *ast.Term) {
filterObj.Insert(key, ast.InternedNullTerm)
})
// Actually do the filtering
r, err := obj.Filter(filterObj)
if err != nil {
return err
}
return iter(ast.NewTerm(r))
}
func builtinObjectGet(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
object, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// if the get key is not an array, attempt to get the top level key for the operand value in the object
path, ok := operands[1].Value.(*ast.Array)
if !ok {
if ret := object.Get(operands[1]); ret != nil {
return iter(ret)
}
return iter(operands[2])
}
// if the path is empty, then we skip selecting nested keys and return the whole object
if path.Len() == 0 {
return iter(operands[0])
}
// build an ast.Ref from the array and see if it matches within the object
pathRef := ref.ArrayPath(path)
value, err := object.Find(pathRef)
if err != nil {
return iter(operands[2])
}
return iter(ast.NewTerm(value))
}
func builtinObjectKeys(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
object, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
if object.Len() == 0 {
return iter(ast.InternedEmptySet)
}
return iter(ast.SetTerm(object.Keys()...))
}
// getObjectKeysParam returns a set of key values
// from a supplied ast array, object, set value.
// The returned set must not be mutated. For Set
// inputs, it may be the original.
func getObjectKeysParam(arrayOrSet ast.Value) (ast.Set, error) {
switch v := arrayOrSet.(type) {
case *ast.Array:
keys := ast.NewSetWithCapacity(v.Len())
v.Foreach(keys.Add)
return keys, nil
case ast.Set:
// Return directly. Callers only use this for Contains() checks
// without mutating the set.
return v, nil
case ast.Object:
return ast.NewSet(v.Keys()...), nil
}
return nil, builtins.NewOperandTypeErr(2, arrayOrSet, "object", "set", "array")
}
func mergeWithOverwrite(objA, objB ast.Object) ast.Object {
merged, _ := objA.MergeWith(objB, func(v1, v2 *ast.Term) (*ast.Term, bool) {
originalValueObj, ok2 := v1.Value.(ast.Object)
updateValueObj, ok1 := v2.Value.(ast.Object)
if !ok1 || !ok2 {
// If we can't merge, stick with the right-hand value
return v2, false
}
// Recursively update the existing value
merged := mergeWithOverwrite(originalValueObj, updateValueObj)
return ast.NewTerm(merged), false
})
return merged
}
// Modifies obj with any new keys from other, and recursively
// merges any keys where the values are both objects.
func mergewithOverwriteInPlace(obj, other ast.Object, frozenKeys map[*ast.Term]struct{}) {
other.Foreach(func(k, v *ast.Term) {
v2 := obj.Get(k)
// The key didn't exist in other, keep the original value.
if v2 == nil {
nestedObj, ok := v.Value.(ast.Object)
if !ok {
// v is not an object
obj.Insert(k, v)
} else {
// Copy the nested object so the original object would not be modified
nestedObjCopy := nestedObj.Copy()
obj.Insert(k, ast.NewTerm(nestedObjCopy))
}
return
}
// The key exists in both. Merge or reject change.
updateValueObj, ok2 := v.Value.(ast.Object)
originalValueObj, ok1 := v2.Value.(ast.Object)
// Both are objects? Merge recursively.
if ok1 && ok2 {
// Check to make sure that this key isn't frozen before merging.
if _, ok := frozenKeys[v2]; !ok {
mergewithOverwriteInPlace(originalValueObj, updateValueObj, frozenKeys)
}
} else {
// Else, original value wins. Freeze the key.
frozenKeys[v2] = struct{}{}
}
})
}
func init() {
RegisterBuiltinFunc(ast.ObjectUnion.Name, builtinObjectUnion)
RegisterBuiltinFunc(ast.ObjectUnionN.Name, builtinObjectUnionN)
RegisterBuiltinFunc(ast.ObjectRemove.Name, builtinObjectRemove)
RegisterBuiltinFunc(ast.ObjectFilter.Name, builtinObjectFilter)
RegisterBuiltinFunc(ast.ObjectGet.Name, builtinObjectGet)
RegisterBuiltinFunc(ast.ObjectKeys.Name, builtinObjectKeys)
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"bytes"
"encoding/json"
"fmt"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinRegoParseModule(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
filename, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
input, err := builtins.StringOperand(operands[1].Value, 1)
if err != nil {
return err
}
// FIXME: Use configured rego-version?
module, err := ast.ParseModule(string(filename), string(input))
if err != nil {
return err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(module); err != nil {
return err
}
term, err := ast.ParseTerm(buf.String())
if err != nil {
return err
}
return iter(term)
}
func registerRegoMetadataBuiltinFunction(builtin *ast.Builtin) {
f := func(BuiltinContext, []*ast.Term, func(*ast.Term) error) error {
// The compiler should replace all usage of this function, so the only way to get here is within a query;
// which cannot define rules.
return fmt.Errorf("the %s function must only be called within the scope of a rule", builtin.Name)
}
RegisterBuiltinFunc(builtin.Name, f)
}
func init() {
RegisterBuiltinFunc(ast.RegoParseModule.Name, builtinRegoParseModule)
registerRegoMetadataBuiltinFunction(ast.RegoMetadataChain)
registerRegoMetadataBuiltinFunction(ast.RegoMetadataRule)
}
@@ -0,0 +1,157 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"math/big"
"strings"
"unicode"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
const (
none uint64 = 1 << (10 * iota)
ki
mi
gi
ti
pi
ei
kb uint64 = 1000
mb = kb * 1000
gb = mb * 1000
tb = gb * 1000
pb = tb * 1000
eb = pb * 1000
)
func parseNumBytesError(msg string) error {
return fmt.Errorf("%s: %s", ast.UnitsParseBytes.Name, msg)
}
func errBytesUnitNotRecognized(unit string) error {
return parseNumBytesError(fmt.Sprintf("byte unit %s not recognized", unit))
}
var (
errBytesValueNoAmount = parseNumBytesError("no byte amount provided")
errBytesValueNumConv = parseNumBytesError("could not parse byte amount to a number")
errBytesValueIncludesSpaces = parseNumBytesError("spaces not allowed in resource strings")
)
func builtinNumBytes(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var m big.Float
raw, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s := formatString(raw)
if strings.Contains(s, " ") {
return errBytesValueIncludesSpaces
}
num, unit := extractNumAndUnit(s)
if num == "" {
return errBytesValueNoAmount
}
switch unit {
case "":
m.SetUint64(none)
case "kb", "k":
m.SetUint64(kb)
case "kib", "ki":
m.SetUint64(ki)
case "mb", "m":
m.SetUint64(mb)
case "mib", "mi":
m.SetUint64(mi)
case "gb", "g":
m.SetUint64(gb)
case "gib", "gi":
m.SetUint64(gi)
case "tb", "t":
m.SetUint64(tb)
case "tib", "ti":
m.SetUint64(ti)
case "pb", "p":
m.SetUint64(pb)
case "pib", "pi":
m.SetUint64(pi)
case "eb", "e":
m.SetUint64(eb)
case "eib", "ei":
m.SetUint64(ei)
default:
return errBytesUnitNotRecognized(unit)
}
numFloat, ok := new(big.Float).SetString(num)
if !ok {
return errBytesValueNumConv
}
var total big.Int
numFloat.Mul(numFloat, &m).Int(&total)
return iter(ast.NewTerm(builtins.IntToNumber(&total)))
}
// Makes the string lower case and removes quotation marks
func formatString(s ast.String) string {
str := string(s)
lower := strings.ToLower(str)
return strings.ReplaceAll(lower, "\"", "")
}
// Splits the string into a number string à la "10" or "10.2" and a unit
// string à la "gb" or "MiB" or "foo". Either can be an empty string
// (error handling is provided elsewhere).
func extractNumAndUnit(s string) (string, string) {
isNum := func(r rune) bool {
return unicode.IsDigit(r) || r == '.'
}
firstNonNumIdx := -1
for idx := 0; idx < len(s); idx++ {
r := rune(s[idx])
// Identify the first non-numeric character, marking the boundary between the number and the unit.
if !isNum(r) && r != 'e' && r != 'E' && r != '+' && r != '-' {
firstNonNumIdx = idx
break
}
if r == 'e' || r == 'E' {
// Check if the next character is a valid digit or +/- for scientific notation
if idx == len(s)-1 || (!unicode.IsDigit(rune(s[idx+1])) && rune(s[idx+1]) != '+' && rune(s[idx+1]) != '-') {
firstNonNumIdx = idx
break
}
// Skip the next character if it is '+' or '-'
if idx+1 < len(s) && (s[idx+1] == '+' || s[idx+1] == '-') {
idx++
}
}
}
if firstNonNumIdx == -1 { // only digits, '.', or valid scientific notation
return s, ""
}
if firstNonNumIdx == 0 { // only units (starts with non-digit)
return "", s
}
// Return the number and the rest as the unit
return s[:firstNonNumIdx], s[firstNonNumIdx:]
}
func init() {
RegisterBuiltinFunc(ast.UnitsParseBytes.Name, builtinNumBytes)
}
@@ -0,0 +1,125 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/json"
"fmt"
"math/big"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
// Binary Si unit constants are borrowed from topdown/parse_bytes
const siMilli = 0.001
const (
siK uint64 = 1000
siM = siK * 1000
siG = siM * 1000
siT = siG * 1000
siP = siT * 1000
siE = siP * 1000
)
func parseUnitsError(msg string) error {
return fmt.Errorf("%s: %s", ast.UnitsParse.Name, msg)
}
func errUnitNotRecognized(unit string) error {
return parseUnitsError(fmt.Sprintf("unit %s not recognized", unit))
}
var (
errNoAmount = parseUnitsError("no amount provided")
errNumConv = parseUnitsError("could not parse amount to a number")
errIncludesSpaces = parseUnitsError("spaces not allowed in resource strings")
)
// Accepts both normal SI and binary SI units.
func builtinUnits(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var x big.Rat
raw, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// We remove escaped quotes from strings here to retain parity with units.parse_bytes.
s := string(raw)
s = strings.ReplaceAll(s, "\"", "")
if strings.Contains(s, " ") {
return errIncludesSpaces
}
num, unit := extractNumAndUnit(s)
if num == "" {
return errNoAmount
}
// Unlike in units.parse_bytes, we only lowercase after the first letter,
// so that we can distinguish between 'm' and 'M'.
if len(unit) > 1 {
lower := strings.ToLower(unit[1:])
unit = unit[:1] + lower
}
switch unit {
case "m":
x.SetFloat64(siMilli)
case "":
x.SetUint64(none)
case "k", "K":
x.SetUint64(siK)
case "ki", "Ki":
x.SetUint64(ki)
case "M":
x.SetUint64(siM)
case "mi", "Mi":
x.SetUint64(mi)
case "g", "G":
x.SetUint64(siG)
case "gi", "Gi":
x.SetUint64(gi)
case "t", "T":
x.SetUint64(siT)
case "ti", "Ti":
x.SetUint64(ti)
case "p", "P":
x.SetUint64(siP)
case "pi", "Pi":
x.SetUint64(pi)
case "e", "E":
x.SetUint64(siE)
case "ei", "Ei":
x.SetUint64(ei)
default:
return errUnitNotRecognized(unit)
}
numRat, ok := new(big.Rat).SetString(num)
if !ok {
return errNumConv
}
numRat.Mul(numRat, &x)
// Cleaner printout when we have a pure integer value.
if numRat.IsInt() {
return iter(ast.NumberTerm(json.Number(numRat.Num().String())))
}
// When using just big.Float, we had floating-point precision
// issues because quantities like 0.001 are not exactly representable.
// Rationals (such as big.Rat) do not suffer this problem, but are
// more expensive to compute with in general.
return iter(ast.NumberTerm(json.Number(numRat.FloatString(10))))
}
func init() {
RegisterBuiltinFunc(ast.UnitsParse.Name, builtinUnits)
}
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2021 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"io"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/topdown/print"
)
func NewPrintHook(w io.Writer) print.Hook {
return printHook{w: w}
}
type printHook struct {
w io.Writer
}
func (h printHook) Print(_ print.Context, msg string) error {
_, err := fmt.Fprintln(h.w, msg)
return err
}
func builtinPrint(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
if bctx.PrintHook == nil {
return iter(nil)
}
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
buf := make([]string, arr.Len())
err = builtinPrintCrossProductOperands(bctx.Location, buf, arr, 0, func(buf []string) error {
pctx := print.Context{
Context: bctx.Context,
Location: bctx.Location,
}
return bctx.PrintHook.Print(pctx, strings.Join(buf, " "))
})
if err != nil {
return err
}
return iter(nil)
}
func builtinPrintCrossProductOperands(loc *ast.Location, buf []string, operands *ast.Array, i int, f func([]string) error) error {
if i >= operands.Len() {
return f(buf)
}
operand := operands.Elem(i)
// We allow primitives ...
switch x := operand.Value.(type) {
case ast.String:
buf[i] = string(x)
return builtinPrintCrossProductOperands(loc, buf, operands, i+1, f)
case ast.Number, ast.Boolean, ast.Null:
buf[i] = x.String()
return builtinPrintCrossProductOperands(loc, buf, operands, i+1, f)
}
// ... but all other operand types must be sets.
xs, ok := operand.Value.(ast.Set)
if !ok {
return Halt{Err: internalErr(loc, "illegal argument type: "+ast.ValueName(operand.Value))}
}
if xs.Len() == 0 {
buf[i] = "<undefined>"
return builtinPrintCrossProductOperands(loc, buf, operands, i+1, f)
}
return xs.Iter(func(x *ast.Term) error {
switch v := x.Value.(type) {
case ast.String:
buf[i] = string(v)
default:
buf[i] = v.String()
}
return builtinPrintCrossProductOperands(loc, buf, operands, i+1, f)
})
}
func init() {
RegisterBuiltinFunc(ast.InternalPrint.Name, builtinPrint)
}
@@ -0,0 +1,21 @@
package print
import (
"context"
"github.com/open-policy-agent/opa/v1/ast"
)
// Context provides the Hook implementation context about the print() call.
type Context struct {
Context context.Context // request context passed when query executed
Location *ast.Location // location of print call
}
// Hook defines the interface that callers can implement to receive print
// statement outputs. If the hook returns an error, it will be surfaced if
// strict builtin error checking is enabled (otherwise, it will not halt
// execution.)
type Hook interface {
Print(Context, string) error
}
+220
View File
@@ -0,0 +1,220 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/json"
"net/url"
"time"
"github.com/open-policy-agent/opa/internal/providers/aws"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
var awsRequiredConfigKeyNames ast.Set
func stringFromTerm(t *ast.Term) string {
if v, ok := t.Value.(ast.String); ok {
return string(v)
}
return ""
}
func getReqBodyBytes(body, rawBody *ast.Term) ([]byte, error) {
var out []byte
switch {
case rawBody != nil:
out = []byte(stringFromTerm(rawBody))
case body != nil:
bodyVal := body.Value
bodyValInterface, err := ast.JSON(bodyVal)
if err != nil {
return nil, err
}
bodyValBytes, err := json.Marshal(bodyValInterface)
if err != nil {
return nil, err
}
out = bodyValBytes
default:
out = []byte("")
}
return out, nil
}
func objectToMap(o ast.Object) map[string][]string {
out := make(map[string][]string, o.Len())
o.Foreach(func(k, v *ast.Term) {
ks := stringFromTerm(k)
vs := stringFromTerm(v)
out[ks] = []string{vs}
})
return out
}
// Note(philipc): This is roughly the same approach used for http.send.
func validateAWSAuthParameters(o ast.Object) error {
awsKeys := ast.NewSet(o.Keys()...)
missingKeys := awsRequiredConfigKeyNames.Diff(awsKeys)
if missingKeys.Len() != 0 {
return builtins.NewOperandErr(2, "missing required AWS config parameters(s): %v", missingKeys)
}
invalidKeys := ast.NewSet()
awsRequiredConfigKeyNames.Foreach(func(t *ast.Term) {
if v := o.Get(t); v != nil {
if _, ok := v.Value.(ast.String); !ok {
invalidKeys.Add(t)
}
}
})
if invalidKeys.Len() != 0 {
return builtins.NewOperandErr(2, "invalid values for required AWS config parameters(s): %v", invalidKeys)
}
return nil
}
func builtinAWSSigV4SignReq(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Request object.
reqObj, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// AWS SigV4 config info object.
awsConfigObj, err := builtins.ObjectOperand(operands[1].Value, 1)
if err != nil {
return err
}
// Make sure our required keys exist!
err = validateAWSAuthParameters(awsConfigObj)
if err != nil {
return err
}
service := stringFromTerm(awsConfigObj.Get(ast.InternedTerm("aws_service")))
awsCreds := aws.CredentialsFromObject(awsConfigObj)
// Timestamp for signing.
var signingTimestamp time.Time
timestamp, err := builtins.NumberOperand(operands[2].Value, 1)
if err != nil {
return err
}
ts, ok := timestamp.Int64()
if !ok {
return builtins.NewOperandErr(3, "could not convert time_ns value into a unix timestamp")
}
signingTimestamp = time.Unix(0, ts)
// Make sure our required keys exist!
// This check is stricter than required, but better to break here than downstream.
_, err = validateHTTPRequestOperand(operands[0], 1)
if err != nil {
return err
}
// Prepare required fields from the HTTP request object.
var theURL *url.URL
var method string
reqURL := reqObj.Get(ast.InternedTerm("url"))
keyMethod := ast.InternedTerm("method")
reqMethod := reqObj.Get(keyMethod)
headers := ast.NewObject()
headersTerm := reqObj.Get(ast.InternedTerm("headers"))
if headersTerm != nil {
var ok bool
headers, ok = headersTerm.Value.(ast.Object)
if !ok {
return builtins.NewOperandTypeErr(0, headersTerm.Value, "object")
}
}
// Check types on the request parameters.
invalidParameters := ast.NewSet()
if _, ok := reqURL.Value.(ast.String); !ok {
invalidParameters.Add(ast.InternedTerm("url"))
}
if _, ok := reqMethod.Value.(ast.String); !ok {
invalidParameters.Add(keyMethod)
}
if invalidParameters.Len() > 0 {
return builtins.NewOperandErr(1, "invalid values for required request parameters(s): %v", invalidParameters)
}
theURL, err = url.Parse(stringFromTerm(reqURL))
if err != nil {
return err
}
method = stringFromTerm(reqMethod)
bodyTerm := reqObj.Get(ast.InternedTerm("body"))
rawBodyTerm := reqObj.Get(ast.InternedTerm("raw_body"))
body, err := getReqBodyBytes(bodyTerm, rawBodyTerm)
if err != nil {
return err
}
// Sign the request object's headers, and reconstruct the headers map.
headersMap := objectToMap(headers)
// if payload signing config is set, pass it down to the signing method
disablePayloadSigning := false
t := awsConfigObj.Get(ast.InternedTerm("disable_payload_signing"))
if t != nil {
if v, ok := t.Value.(ast.Boolean); ok {
disablePayloadSigning = bool(v)
} else {
return builtins.NewOperandErr(2, "invalid value for 'disable_payload_signing' in AWS config")
}
}
authHeader, awsHeadersMap := aws.SignV4(headersMap, method, theURL, body, service, awsCreds, signingTimestamp, disablePayloadSigning)
signedHeadersObj := ast.NewObject()
// Restore original headers
for k, v := range headersMap {
// objectToMap doesn't support arrays
if len(v) == 1 {
signedHeadersObj.Insert(ast.InternedTerm(k), ast.StringTerm(v[0]))
}
}
// Set authorization header
signedHeadersObj.Insert(ast.InternedTerm("Authorization"), ast.StringTerm(authHeader))
// set aws signature headers
for k, v := range awsHeadersMap {
signedHeadersObj.Insert(ast.InternedTerm(k), ast.StringTerm(v))
}
// Create new request object with updated headers.
out := reqObj.Copy()
out.Insert(ast.InternedTerm("headers"), ast.NewTerm(signedHeadersObj))
return iter(ast.NewTerm(out))
}
func init() {
for _, key := range []string{
"aws_service", "aws_access_key", "aws_secret_access_key", "aws_region", "disable_payload_signing",
} {
ast.InternStringTerm(key)
}
awsRequiredConfigKeyNames = ast.NewSet(
ast.InternedTerm("aws_service"),
ast.InternedTerm("aws_access_key"),
ast.InternedTerm("aws_secret_access_key"),
ast.InternedTerm("aws_region"),
)
RegisterBuiltinFunc(ast.ProvidersAWSSignReqObj.Name, builtinAWSSigV4SignReq)
}
+640
View File
@@ -0,0 +1,640 @@
package topdown
import (
"context"
"crypto/rand"
"io"
"sort"
"time"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/metrics"
"github.com/open-policy-agent/opa/v1/resolver"
"github.com/open-policy-agent/opa/v1/storage"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/topdown/cache"
"github.com/open-policy-agent/opa/v1/topdown/copypropagation"
"github.com/open-policy-agent/opa/v1/topdown/print"
"github.com/open-policy-agent/opa/v1/tracing"
)
// QueryResultSet represents a collection of results returned by a query.
type QueryResultSet []QueryResult
// QueryResult represents a single result returned by a query. The result
// contains bindings for all variables that appear in the query.
type QueryResult map[ast.Var]*ast.Term
// Query provides a configurable interface for performing query evaluation.
type Query struct {
seed io.Reader
time time.Time
cancel Cancel
query ast.Body
queryCompiler ast.QueryCompiler
compiler *ast.Compiler
store storage.Store
txn storage.Transaction
input *ast.Term
external *resolverTrie
tracers []QueryTracer
plugTraceVars bool
unknowns []*ast.Term
partialNamespace string
skipSaveNamespace bool
metrics metrics.Metrics
instr *Instrumentation
disableInlining []ast.Ref
shallowInlining bool
nondeterministicBuiltins bool
genvarprefix string
runtime *ast.Term
builtins map[string]*Builtin
indexing bool
earlyExit bool
interQueryBuiltinCache cache.InterQueryCache
interQueryBuiltinValueCache cache.InterQueryValueCache
ndBuiltinCache builtins.NDBCache
strictBuiltinErrors bool
builtinErrorList *[]Error
strictObjects bool
roundTripper CustomizeRoundTripper
printHook print.Hook
tracingOpts tracing.Options
virtualCache VirtualCache
baseCache BaseCache
}
// Builtin represents a built-in function that queries can call.
type Builtin struct {
Decl *ast.Builtin
Func BuiltinFunc
}
// NewQuery returns a new Query object that can be run.
func NewQuery(query ast.Body) *Query {
return &Query{
query: query,
genvarprefix: ast.WildcardPrefix,
indexing: true,
earlyExit: true,
}
}
// WithQueryCompiler sets the queryCompiler used for the query.
func (q *Query) WithQueryCompiler(queryCompiler ast.QueryCompiler) *Query {
q.queryCompiler = queryCompiler
return q
}
// WithCompiler sets the compiler to use for the query.
func (q *Query) WithCompiler(compiler *ast.Compiler) *Query {
q.compiler = compiler
return q
}
// WithStore sets the store to use for the query.
func (q *Query) WithStore(store storage.Store) *Query {
q.store = store
return q
}
// WithTransaction sets the transaction to use for the query. All queries
// should be performed over a consistent snapshot of the storage layer.
func (q *Query) WithTransaction(txn storage.Transaction) *Query {
q.txn = txn
return q
}
// WithCancel sets the cancellation object to use for the query. Set this if
// you need to abort queries based on a deadline. This is optional.
func (q *Query) WithCancel(cancel Cancel) *Query {
q.cancel = cancel
return q
}
// WithInput sets the input object to use for the query. References rooted at
// input will be evaluated against this value. This is optional.
func (q *Query) WithInput(input *ast.Term) *Query {
q.input = input
return q
}
// WithTracer adds a query tracer to use during evaluation. This is optional.
//
// Deprecated: Use WithQueryTracer instead.
func (q *Query) WithTracer(tracer Tracer) *Query {
qt, ok := tracer.(QueryTracer)
if !ok {
qt = WrapLegacyTracer(tracer)
}
return q.WithQueryTracer(qt)
}
// WithQueryTracer adds a query tracer to use during evaluation. This is optional.
// Disabled QueryTracers will be ignored.
func (q *Query) WithQueryTracer(tracer QueryTracer) *Query {
if tracer == nil || !tracer.Enabled() {
return q
}
q.tracers = append(q.tracers, tracer)
// If *any* of the tracers require local variable metadata we need to
// enabled plugging local trace variables.
conf := tracer.Config()
if conf.PlugLocalVars {
q.plugTraceVars = true
}
return q
}
// WithMetrics sets the metrics collection to add evaluation metrics to. This
// is optional.
func (q *Query) WithMetrics(m metrics.Metrics) *Query {
q.metrics = m
return q
}
// WithInstrumentation sets the instrumentation configuration to enable on the
// evaluation process. By default, instrumentation is turned off.
func (q *Query) WithInstrumentation(instr *Instrumentation) *Query {
q.instr = instr
return q
}
// WithUnknowns sets the initial set of variables or references to treat as
// unknown during query evaluation. This is required for partial evaluation.
func (q *Query) WithUnknowns(terms []*ast.Term) *Query {
q.unknowns = terms
return q
}
// WithPartialNamespace sets the namespace to use for supporting rules
// generated as part of the partial evaluation process. The ns value must be a
// valid package path component.
func (q *Query) WithPartialNamespace(ns string) *Query {
q.partialNamespace = ns
return q
}
// WithSkipPartialNamespace disables namespacing of saved support rules that are generated
// from the original policy (rules which are completely synthetic are still namespaced.)
func (q *Query) WithSkipPartialNamespace(yes bool) *Query {
q.skipSaveNamespace = yes
return q
}
// WithDisableInlining adds a set of paths to the query that should be excluded from
// inlining. Inlining during partial evaluation can be expensive in some cases
// (e.g., when a cross-product is computed.) Disabling inlining avoids expensive
// computation at the cost of generating support rules.
func (q *Query) WithDisableInlining(paths []ast.Ref) *Query {
q.disableInlining = paths
return q
}
// WithShallowInlining disables aggressive inlining performed during partial evaluation.
// When shallow inlining is enabled rules that depend (transitively) on unknowns are not inlined.
// Only rules/values that are completely known will be inlined.
func (q *Query) WithShallowInlining(yes bool) *Query {
q.shallowInlining = yes
return q
}
// WithRuntime sets the runtime data to execute the query with. The runtime data
// can be returned by the `opa.runtime` built-in function.
func (q *Query) WithRuntime(runtime *ast.Term) *Query {
q.runtime = runtime
return q
}
// WithBuiltins adds a set of built-in functions that can be called by the
// query.
func (q *Query) WithBuiltins(builtins map[string]*Builtin) *Query {
q.builtins = builtins
return q
}
// WithIndexing will enable or disable using rule indexing for the evaluation
// of the query. The default is enabled.
func (q *Query) WithIndexing(enabled bool) *Query {
q.indexing = enabled
return q
}
// WithEarlyExit will enable or disable using 'early exit' for the evaluation
// of the query. The default is enabled.
func (q *Query) WithEarlyExit(enabled bool) *Query {
q.earlyExit = enabled
return q
}
// WithSeed sets a reader that will seed randomization required by built-in functions.
// If a seed is not provided crypto/rand.Reader is used.
func (q *Query) WithSeed(r io.Reader) *Query {
q.seed = r
return q
}
// WithTime sets the time that will be returned by the time.now_ns() built-in function.
func (q *Query) WithTime(x time.Time) *Query {
q.time = x
return q
}
// WithInterQueryBuiltinCache sets the inter-query cache that built-in functions can utilize.
func (q *Query) WithInterQueryBuiltinCache(c cache.InterQueryCache) *Query {
q.interQueryBuiltinCache = c
return q
}
// WithInterQueryBuiltinValueCache sets the inter-query value cache that built-in functions can utilize.
func (q *Query) WithInterQueryBuiltinValueCache(c cache.InterQueryValueCache) *Query {
q.interQueryBuiltinValueCache = c
return q
}
// WithNDBuiltinCache sets the non-deterministic builtin cache.
func (q *Query) WithNDBuiltinCache(c builtins.NDBCache) *Query {
q.ndBuiltinCache = c
return q
}
// WithStrictBuiltinErrors tells the evaluator to treat all built-in function errors as fatal errors.
func (q *Query) WithStrictBuiltinErrors(yes bool) *Query {
q.strictBuiltinErrors = yes
return q
}
// WithBuiltinErrorList supplies a pointer to an Error slice to store built-in function errors
// encountered during evaluation. This error slice can be inspected after evaluation to determine
// which built-in function errors occurred.
func (q *Query) WithBuiltinErrorList(list *[]Error) *Query {
q.builtinErrorList = list
return q
}
// WithResolver configures an external resolver to use for the given ref.
func (q *Query) WithResolver(ref ast.Ref, r resolver.Resolver) *Query {
if q.external == nil {
q.external = newResolverTrie()
}
q.external.Put(ref, r)
return q
}
// WithHTTPRoundTripper configures a custom HTTP transport for built-in functions that make HTTP requests.
func (q *Query) WithHTTPRoundTripper(t CustomizeRoundTripper) *Query {
q.roundTripper = t
return q
}
func (q *Query) WithPrintHook(h print.Hook) *Query {
q.printHook = h
return q
}
// WithDistributedTracingOpts sets the options to be used by distributed tracing.
func (q *Query) WithDistributedTracingOpts(tr tracing.Options) *Query {
q.tracingOpts = tr
return q
}
// WithStrictObjects tells the evaluator to avoid the "lazy object" optimization
// applied when reading objects from the store. It will result in higher memory
// usage and should only be used temporarily while adjusting code that breaks
// because of the optimization.
func (q *Query) WithStrictObjects(yes bool) *Query {
q.strictObjects = yes
return q
}
// WithVirtualCache sets the VirtualCache to use during evaluation. This is
// optional, and if not set, the default cache is used.
func (q *Query) WithVirtualCache(vc VirtualCache) *Query {
q.virtualCache = vc
return q
}
// WithBaseCache sets the BaseCache to use during evaluation. This is
// optional, and if not set, the default cache is used.
func (q *Query) WithBaseCache(bc BaseCache) *Query {
q.baseCache = bc
return q
}
// WithNondeterministicBuiltins causes non-deterministic builtins to be evalued
// during partial evaluation. This is needed to pull in external data, or validate
// a JWT, during PE, so that the result informs what queries are returned.
func (q *Query) WithNondeterministicBuiltins(yes bool) *Query {
q.nondeterministicBuiltins = yes
return q
}
// PartialRun executes partial evaluation on the query with respect to unknown
// values. Partial evaluation attempts to evaluate as much of the query as
// possible without requiring values for the unknowns set on the query. The
// result of partial evaluation is a new set of queries that can be evaluated
// once the unknown value is known. In addition to new queries, partial
// evaluation may produce additional support modules that should be used in
// conjunction with the partially evaluated queries.
func (q *Query) PartialRun(ctx context.Context) (partials []ast.Body, support []*ast.Module, err error) {
if q.partialNamespace == "" {
q.partialNamespace = "partial" // lazily initialize partial namespace
}
if q.seed == nil {
q.seed = rand.Reader
}
if q.time.IsZero() {
q.time = time.Now()
}
if q.metrics == nil {
q.metrics = metrics.New()
}
f := &queryIDFactory{}
b := newBindings(0, q.instr)
var vc VirtualCache
if q.virtualCache != nil {
vc = q.virtualCache
} else {
vc = NewVirtualCache()
}
var bc BaseCache
if q.baseCache != nil {
bc = q.baseCache
} else {
bc = newBaseCache()
}
e := &eval{
ctx: ctx,
metrics: q.metrics,
seed: q.seed,
timeStart: q.time.UnixNano(),
cancel: q.cancel,
query: q.query,
queryCompiler: q.queryCompiler,
queryIDFact: f,
queryID: f.Next(),
bindings: b,
compiler: q.compiler,
store: q.store,
baseCache: bc,
txn: q.txn,
input: q.input,
external: q.external,
tracers: q.tracers,
traceEnabled: len(q.tracers) > 0,
plugTraceVars: q.plugTraceVars,
instr: q.instr,
builtins: q.builtins,
builtinCache: builtins.Cache{},
interQueryBuiltinCache: q.interQueryBuiltinCache,
interQueryBuiltinValueCache: q.interQueryBuiltinValueCache,
ndBuiltinCache: q.ndBuiltinCache,
virtualCache: vc,
saveSet: newSaveSet(q.unknowns, b, q.instr),
saveStack: newSaveStack(),
saveSupport: newSaveSupport(),
saveNamespace: ast.InternedTerm(q.partialNamespace),
skipSaveNamespace: q.skipSaveNamespace,
inliningControl: &inliningControl{
shallow: q.shallowInlining,
nondeterministicBuiltins: q.nondeterministicBuiltins,
},
genvarprefix: q.genvarprefix,
runtime: q.runtime,
indexing: q.indexing,
earlyExit: q.earlyExit,
builtinErrors: &builtinErrors{},
printHook: q.printHook,
strictObjects: q.strictObjects,
}
if len(q.disableInlining) > 0 {
e.inliningControl.PushDisable(q.disableInlining, false)
}
e.caller = e
q.metrics.Timer(metrics.RegoPartialEval).Start()
defer q.metrics.Timer(metrics.RegoPartialEval).Stop()
livevars := ast.NewVarSet()
for _, t := range q.unknowns {
switch v := t.Value.(type) {
case ast.Var:
livevars.Add(v)
case ast.Ref:
livevars.Add(v[0].Value.(ast.Var))
}
}
ast.WalkVars(q.query, func(x ast.Var) bool {
if !x.IsGenerated() {
livevars.Add(x)
}
return false
})
p := copypropagation.New(livevars).WithCompiler(q.compiler)
err = e.Run(func(e *eval) error {
// Build output from saved expressions.
body := ast.NewBody()
for _, elem := range e.saveStack.Stack[len(e.saveStack.Stack)-1] {
body.Append(elem.Plug(e.bindings))
}
// Include bindings as exprs so that when caller evals the result, they
// can obtain values for the vars in their query.
bindingExprs := []*ast.Expr{}
_ = e.bindings.Iter(e.bindings, func(a, b *ast.Term) error {
bindingExprs = append(bindingExprs, ast.Equality.Expr(a, b))
return nil
}) // cannot return error
// Sort binding expressions so that results are deterministic.
sort.Slice(bindingExprs, func(i, j int) bool {
return bindingExprs[i].Compare(bindingExprs[j]) < 0
})
for i := range bindingExprs {
body.Append(bindingExprs[i])
}
// Skip this rule body if it fails to type-check.
// Type-checking failure means the rule body will never succeed.
if !e.compiler.PassesTypeCheck(body) {
return nil
}
if !q.shallowInlining {
body = applyCopyPropagation(p, e.instr, body)
}
partials = append(partials, body)
return nil
})
support = e.saveSupport.List()
if len(e.builtinErrors.errs) > 0 {
if q.strictBuiltinErrors {
err = e.builtinErrors.errs[0]
} else if q.builtinErrorList != nil {
// If a builtinErrorList has been supplied, we must use pointer indirection
// to append to it. builtinErrorList is a slice pointer so that errors can be
// appended to it without returning a new slice and changing the interface
// of PartialRun.
for _, err := range e.builtinErrors.errs {
if tdError, ok := err.(*Error); ok {
*(q.builtinErrorList) = append(*(q.builtinErrorList), *tdError)
} else {
*(q.builtinErrorList) = append(*(q.builtinErrorList), Error{
Code: BuiltinErr,
Message: err.Error(),
})
}
}
}
}
for i, m := range support {
if regoVersion := q.compiler.DefaultRegoVersion(); regoVersion != ast.RegoUndefined {
ast.SetModuleRegoVersion(m, q.compiler.DefaultRegoVersion())
}
sort.Slice(support[i].Rules, func(j, k int) bool {
return support[i].Rules[j].Compare(support[i].Rules[k]) < 0
})
}
return partials, support, err
}
// Run is a wrapper around Iter that accumulates query results and returns them
// in one shot.
func (q *Query) Run(ctx context.Context) (QueryResultSet, error) {
qrs := QueryResultSet{}
return qrs, q.Iter(ctx, func(qr QueryResult) error {
qrs = append(qrs, qr)
return nil
})
}
// Iter executes the query and invokes the iter function with query results
// produced by evaluating the query.
func (q *Query) Iter(ctx context.Context, iter func(QueryResult) error) error {
// Query evaluation must not be allowed if the compiler has errors and is in an undefined, possibly inconsistent state
if q.compiler != nil && len(q.compiler.Errors) > 0 {
return &Error{
Code: InternalErr,
Message: "compiler has errors",
}
}
if q.seed == nil {
q.seed = rand.Reader
}
if q.time.IsZero() {
q.time = time.Now()
}
if q.metrics == nil {
q.metrics = metrics.New()
}
f := &queryIDFactory{}
var vc VirtualCache
if q.virtualCache != nil {
vc = q.virtualCache
} else {
vc = NewVirtualCache()
}
var bc BaseCache
if q.baseCache != nil {
bc = q.baseCache
} else {
bc = newBaseCache()
}
e := &eval{
ctx: ctx,
metrics: q.metrics,
seed: q.seed,
timeStart: q.time.UnixNano(),
cancel: q.cancel,
query: q.query,
queryCompiler: q.queryCompiler,
queryIDFact: f,
queryID: f.Next(),
bindings: newBindings(0, q.instr),
compiler: q.compiler,
store: q.store,
baseCache: bc,
txn: q.txn,
input: q.input,
external: q.external,
tracers: q.tracers,
traceEnabled: len(q.tracers) > 0,
plugTraceVars: q.plugTraceVars,
instr: q.instr,
builtins: q.builtins,
builtinCache: builtins.Cache{},
interQueryBuiltinCache: q.interQueryBuiltinCache,
interQueryBuiltinValueCache: q.interQueryBuiltinValueCache,
ndBuiltinCache: q.ndBuiltinCache,
virtualCache: vc,
genvarprefix: q.genvarprefix,
runtime: q.runtime,
indexing: q.indexing,
earlyExit: q.earlyExit,
builtinErrors: &builtinErrors{},
printHook: q.printHook,
tracingOpts: q.tracingOpts,
strictObjects: q.strictObjects,
roundTripper: q.roundTripper,
}
e.caller = e
q.metrics.Timer(metrics.RegoQueryEval).Start()
err := e.Run(func(e *eval) error {
qr := QueryResult{}
_ = e.bindings.Iter(nil, func(k, v *ast.Term) error {
qr[k.Value.(ast.Var)] = v
return nil
}) // cannot return error
return iter(qr)
})
if len(e.builtinErrors.errs) > 0 {
if q.strictBuiltinErrors {
err = e.builtinErrors.errs[0]
} else if q.builtinErrorList != nil {
// If a builtinErrorList has been supplied, we must use pointer indirection
// to append to it. builtinErrorList is a slice pointer so that errors can be
// appended to it without returning a new slice and changing the interface
// of Iter.
for _, err := range e.builtinErrors.errs {
if tdError, ok := err.(*Error); ok {
*(q.builtinErrorList) = append(*(q.builtinErrorList), *tdError)
} else {
*(q.builtinErrorList) = append(*(q.builtinErrorList), Error{
Code: BuiltinErr,
Message: err.Error(),
})
}
}
}
}
q.metrics.Timer(metrics.RegoQueryEval).Stop()
return err
}
+151
View File
@@ -0,0 +1,151 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
// Helper: sets of vertices can be represented as Arrays or Sets.
func foreachVertex(collection *ast.Term, f func(*ast.Term)) {
switch v := collection.Value.(type) {
case ast.Set:
v.Foreach(f)
case *ast.Array:
v.Foreach(f)
}
}
// numberOfEdges returns the number of elements of an array or a set (of edges)
func numberOfEdges(collection *ast.Term) int {
switch v := collection.Value.(type) {
case ast.Set:
return v.Len()
case *ast.Array:
return v.Len()
}
return 0
}
func builtinReachable(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// Error on wrong types for args.
graph, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
var queue []*ast.Term
switch initial := operands[1].Value.(type) {
case *ast.Array, ast.Set:
foreachVertex(ast.NewTerm(initial), func(t *ast.Term) {
queue = append(queue, t)
})
default:
return builtins.NewOperandTypeErr(2, initial, "{array, set}")
}
// This is the set of nodes we have reached.
reached := ast.NewSet()
// Keep going as long as we have nodes in the queue.
for len(queue) > 0 {
// Get the edges for this node. If the node was not in the graph,
// `edges` will be `nil` and we can ignore it.
node := queue[0]
if edges := graph.Get(node); edges != nil {
// Add all the newly discovered neighbors.
foreachVertex(edges, func(neighbor *ast.Term) {
if !reached.Contains(neighbor) {
queue = append(queue, neighbor)
}
})
// Mark the node as reached.
reached.Add(node)
}
queue = queue[1:]
}
return iter(ast.NewTerm(reached))
}
// pathBuilder is called recursively to build a Set of paths that are reachable from the root
func pathBuilder(graph ast.Object, root *ast.Term, path []*ast.Term, edgeRslt ast.Set, reached ast.Set) {
paths := []*ast.Term{}
if edges := graph.Get(root); edges != nil {
path = append(path, root)
if numberOfEdges(edges) >= 1 {
foreachVertex(edges, func(neighbor *ast.Term) {
if reached.Contains(neighbor) {
// If we've already reached this node, return current path (avoid infinite recursion)
paths = append(paths, path...)
edgeRslt.Add(ast.ArrayTerm(paths...))
} else {
reached.Add(root)
pathBuilder(graph, neighbor, path, edgeRslt, reached)
}
})
} else {
paths = append(paths, path...)
edgeRslt.Add(ast.ArrayTerm(paths...))
}
} else {
// Node is nonexistent (not in graph). Commit the current path (without adding this root)
paths = append(paths, path...)
edgeRslt.Add(ast.ArrayTerm(paths...))
}
}
func builtinReachablePaths(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var traceResult = ast.NewSet()
// Error on wrong types for args.
graph, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
// This is a queue that holds all nodes we still need to visit. It is
// initialised to the initial set of nodes we start out with.
var queue []*ast.Term
switch initial := operands[1].Value.(type) {
case *ast.Array, ast.Set:
foreachVertex(ast.NewTerm(initial), func(t *ast.Term) {
queue = append(queue, t)
})
default:
return builtins.NewOperandTypeErr(2, initial, "{array, set}")
}
for _, node := range queue {
// Find reachable paths from edges in root node in queue and append arrays to the results set
if edges := graph.Get(node); edges != nil {
if numberOfEdges(edges) >= 1 {
foreachVertex(edges, func(neighbor *ast.Term) {
pathBuilder(graph, neighbor, []*ast.Term{node}, traceResult, ast.NewSet(node))
})
} else {
traceResult.Add(ast.ArrayTerm(node))
}
}
}
return iter(ast.NewTerm(traceResult))
}
func init() {
RegisterBuiltinFunc(ast.ReachableBuiltin.Name, builtinReachable)
RegisterBuiltinFunc(ast.ReachablePathsBuiltin.Name, builtinReachablePaths)
}
+321
View File
@@ -0,0 +1,321 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"regexp"
"regexp/syntax"
"sync"
gintersect "github.com/yashtewari/glob-intersection"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
const (
regexCacheMaxSize = 100
regexInterQueryValueCacheHits = "rego_builtin_regex_interquery_value_cache_hits"
)
var (
regexpCacheLock = sync.RWMutex{}
regexpCache = make(map[string]*regexp.Regexp)
)
func builtinRegexIsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
if s, err := builtins.StringOperand(operands[0].Value, 1); err == nil {
if _, err = syntax.Parse(string(s), syntax.Perl); err == nil {
return iter(ast.InternedTerm(true))
}
}
return iter(ast.InternedTerm(false))
}
func builtinRegexMatch(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
re, err := getRegexp(bctx, string(s1))
if err != nil {
return err
}
return iter(ast.InternedTerm(re.MatchString(string(s2))))
}
func builtinRegexMatchTemplate(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
pattern, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
match, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
start, err := builtins.StringOperand(operands[2].Value, 3)
if err != nil {
return err
}
end, err := builtins.StringOperand(operands[3].Value, 4)
if err != nil {
return err
}
if len(start) != 1 {
return fmt.Errorf("start delimiter has to be exactly one character long but is %d long", len(start))
}
if len(end) != 1 {
return fmt.Errorf("end delimiter has to be exactly one character long but is %d long", len(start))
}
re, err := getRegexpTemplate(string(pattern), string(start)[0], string(end)[0])
if err != nil {
return err
}
return iter(ast.InternedTerm(re.MatchString(string(match))))
}
func builtinRegexSplit(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
re, err := getRegexp(bctx, string(s1))
if err != nil {
return err
}
elems := re.Split(string(s2), -1)
arr := make([]*ast.Term, len(elems))
for i := range elems {
arr[i] = ast.StringTerm(elems[i])
}
return iter(ast.ArrayTerm(arr...))
}
func getRegexp(bctx BuiltinContext, pat string) (*regexp.Regexp, error) {
if bctx.InterQueryBuiltinValueCache != nil {
// TODO: Use named cache
var key ast.Value = ast.String(pat)
val, ok := bctx.InterQueryBuiltinValueCache.Get(key)
if ok {
res, valid := val.(*regexp.Regexp)
if !valid {
// The cache key may exist for a different value type (eg. glob).
// In this case, we calculate the regex and return the result w/o updating the cache.
return regexp.Compile(pat)
}
bctx.Metrics.Counter(regexInterQueryValueCacheHits).Incr()
return res, nil
}
re, err := regexp.Compile(pat)
if err != nil {
return nil, err
}
bctx.InterQueryBuiltinValueCache.Insert(key, re)
return re, nil
}
regexpCacheLock.RLock()
re, ok := regexpCache[pat]
numCached := len(regexpCache)
regexpCacheLock.RUnlock()
if !ok {
var err error
re, err = regexp.Compile(pat)
if err != nil {
return nil, err
}
regexpCacheLock.Lock()
if numCached >= regexCacheMaxSize {
// Delete a (semi-)random key to make room for the new one.
for k := range regexpCache {
delete(regexpCache, k)
break
}
}
regexpCache[pat] = re
regexpCacheLock.Unlock()
}
return re, nil
}
func getRegexpTemplate(pat string, delimStart, delimEnd byte) (*regexp.Regexp, error) {
regexpCacheLock.RLock()
re, ok := regexpCache[pat]
regexpCacheLock.RUnlock()
if !ok {
var err error
re, err = compileRegexTemplate(pat, delimStart, delimEnd)
if err != nil {
return nil, err
}
regexpCacheLock.Lock()
regexpCache[pat] = re
regexpCacheLock.Unlock()
}
return re, nil
}
func builtinGlobsMatch(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
ne, err := gintersect.NonEmpty(string(s1), string(s2))
if err != nil {
return err
}
return iter(ast.InternedTerm(ne))
}
func builtinRegexFind(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
n, err := builtins.IntOperand(operands[2].Value, 3)
if err != nil {
return err
}
re, err := getRegexp(bctx, string(s1))
if err != nil {
return err
}
elems := re.FindAllString(string(s2), n)
arr := make([]*ast.Term, len(elems))
for i := range elems {
arr[i] = ast.StringTerm(elems[i])
}
return iter(ast.ArrayTerm(arr...))
}
func builtinRegexFindAllStringSubmatch(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
n, err := builtins.IntOperand(operands[2].Value, 3)
if err != nil {
return err
}
re, err := getRegexp(bctx, string(s1))
if err != nil {
return err
}
matches := re.FindAllStringSubmatch(string(s2), n)
outer := make([]*ast.Term, len(matches))
for i := range matches {
inner := make([]*ast.Term, len(matches[i]))
for j := range matches[i] {
inner[j] = ast.StringTerm(matches[i][j])
}
outer[i] = ast.ArrayTerm(inner...)
}
return iter(ast.ArrayTerm(outer...))
}
func builtinRegexReplace(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
base, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
pattern, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
value, err := builtins.StringOperand(operands[2].Value, 3)
if err != nil {
return err
}
re, err := getRegexp(bctx, string(pattern))
if err != nil {
return err
}
// If no cancellation context, use the fast path
if bctx.Cancel == nil {
res := re.ReplaceAllString(string(base), string(value))
if res == string(base) {
return iter(operands[0])
}
return iter(ast.InternedTerm(res))
}
// Use sink writer for cancellation-aware replacement
sink := newSink(ast.RegexReplace.Name, len(base), bctx.Cancel)
src := []byte(base)
repl := []byte(value)
// Find all matches at once to preserve anchor behavior: replace("foo", "^[a-z]", "F") => "Foo"
allMatches := re.FindAllSubmatchIndex(src, -1)
lastEnd := 0
for _, match := range allMatches {
if _, err := sink.Write(src[lastEnd:match[0]]); err != nil {
return err
}
if _, err := sink.Write(re.Expand(nil, repl, src, match)); err != nil {
return err
}
lastEnd = match[1]
}
if _, err := sink.Write(src[lastEnd:]); err != nil {
return err
}
res := sink.String()
if res == string(base) {
return iter(operands[0])
}
return iter(ast.InternedTerm(res))
}
func init() {
RegisterBuiltinFunc(ast.RegexIsValid.Name, builtinRegexIsValid)
RegisterBuiltinFunc(ast.RegexMatch.Name, builtinRegexMatch)
RegisterBuiltinFunc(ast.RegexMatchDeprecated.Name, builtinRegexMatch)
RegisterBuiltinFunc(ast.RegexSplit.Name, builtinRegexSplit)
RegisterBuiltinFunc(ast.GlobsMatch.Name, builtinGlobsMatch)
RegisterBuiltinFunc(ast.RegexTemplateMatch.Name, builtinRegexMatchTemplate)
RegisterBuiltinFunc(ast.RegexFind.Name, builtinRegexFind)
RegisterBuiltinFunc(ast.RegexFindAllStringSubmatch.Name, builtinRegexFindAllStringSubmatch)
RegisterBuiltinFunc(ast.RegexReplace.Name, builtinRegexReplace)
}
@@ -0,0 +1,122 @@
package topdown
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license as follows:
// Copyright (c) 2012 Rodrigo Moraes. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// This file was forked from https://github.com/gorilla/mux/commit/eac83ba2c004bb75
import (
"bytes"
"fmt"
"regexp"
)
// delimiterIndices returns the first level delimiter indices from a string.
// It returns an error in case of unbalanced delimiters.
func delimiterIndices(s string, delimiterStart, delimiterEnd byte) ([]int, error) {
var level, idx int
idxs := make([]int, 0)
for i := range len(s) {
switch s[i] {
case delimiterStart:
if level++; level == 1 {
idx = i
}
case delimiterEnd:
if level--; level == 0 {
idxs = append(idxs, idx, i+1)
} else if level < 0 {
return nil, fmt.Errorf(`unbalanced braces in %q`, s)
}
}
}
if level != 0 {
return nil, fmt.Errorf(`unbalanced braces in %q`, s)
}
return idxs, nil
}
// compileRegexTemplate parses a template and returns a Regexp.
//
// You can define your own delimiters. It is e.g. common to use curly braces {} but I recommend using characters
// which have no special meaning in Regex, e.g.: <, >
//
// reg, err := compiler.CompileRegex("foo:bar.baz:<[0-9]{2,10}>", '<', '>')
// // if err != nil ...
// reg.MatchString("foo:bar.baz:123")
func compileRegexTemplate(tpl string, delimiterStart, delimiterEnd byte) (*regexp.Regexp, error) {
// Check if it is well-formed.
idxs, errBraces := delimiterIndices(tpl, delimiterStart, delimiterEnd)
if errBraces != nil {
return nil, errBraces
}
varsR := make([]*regexp.Regexp, len(idxs)/2)
pattern := bytes.NewBufferString("")
// WriteByte's error value is always nil for bytes.Buffer, no need to check it.
pattern.WriteByte('^')
var end int
var err error
for i := 0; i < len(idxs); i += 2 {
// Set all values we are interested in.
raw := tpl[end:idxs[i]]
end = idxs[i+1]
patt := tpl[idxs[i]+1 : end-1]
// Build the regexp pattern.
varIdx := i / 2
fmt.Fprintf(pattern, "%s(%s)", regexp.QuoteMeta(raw), patt)
varsR[varIdx], err = regexp.Compile(fmt.Sprintf("^%s$", patt))
if err != nil {
return nil, err
}
}
// Add the remaining.
raw := tpl[end:]
// WriteString's error value is always nil for bytes.Buffer, no need to check it.
pattern.WriteString(regexp.QuoteMeta(raw))
// WriteByte's error value is always nil for bytes.Buffer, no need to check it.
pattern.WriteByte('$')
// Compile full regexp.
reg, errCompile := regexp.Compile(pattern.String())
if errCompile != nil {
return nil, errCompile
}
return reg, nil
}
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/metrics"
"github.com/open-policy-agent/opa/v1/resolver"
)
type resolverTrie struct {
r resolver.Resolver
children map[ast.Value]*resolverTrie
}
func newResolverTrie() *resolverTrie {
return &resolverTrie{children: map[ast.Value]*resolverTrie{}}
}
func (t *resolverTrie) Put(ref ast.Ref, r resolver.Resolver) {
node := t
for _, t := range ref {
child, ok := node.children[t.Value]
if !ok {
child = &resolverTrie{children: map[ast.Value]*resolverTrie{}}
node.children[t.Value] = child
}
node = child
}
node.r = r
}
func (t *resolverTrie) Resolve(e *eval, ref ast.Ref) (ast.Value, error) {
e.metrics.Timer(metrics.RegoExternalResolve).Start()
defer e.metrics.Timer(metrics.RegoExternalResolve).Stop()
if t == nil {
return nil, nil
}
node := t
for i, t := range ref {
child, ok := node.children[t.Value]
if !ok {
return nil, nil
}
node = child
if node.r != nil {
in := resolver.Input{
Ref: ref[:i+1],
Input: e.input,
Metrics: e.metrics,
}
if e.traceEnabled {
// avoid leaking pointer if trace is disabled
cpy := in.Ref
e.traceWasm(e.query[e.index], &cpy)
}
if e.data != nil {
return nil, errInScopeWithStmt
}
result, err := node.r.Eval(e.ctx, in)
if err != nil {
return nil, err
}
if result.Value == nil {
return nil, nil
}
val, err := result.Value.Find(ref[i+1:])
if err != nil {
return nil, nil
}
return val, nil
}
}
return node.mktree(e, resolver.Input{
Ref: ref,
Input: e.input,
Metrics: e.metrics,
})
}
func (t *resolverTrie) mktree(e *eval, in resolver.Input) (ast.Value, error) {
if t.r != nil {
if e.traceEnabled {
cpy := in.Ref
e.traceWasm(e.query[e.index], &cpy)
}
if e.data != nil {
return nil, errInScopeWithStmt
}
result, err := t.r.Eval(e.ctx, in)
if err != nil {
return nil, err
}
if result.Value == nil {
return nil, nil
}
return result.Value, nil
}
obj := ast.NewObjectWithCapacity(len(t.children))
for k, child := range t.children {
v, err := child.mktree(e, resolver.Input{Ref: append(in.Ref, ast.NewTerm(k)), Input: in.Input, Metrics: in.Metrics})
if err != nil {
return nil, err
}
if v != nil {
obj.Insert(ast.NewTerm(k), ast.NewTerm(v))
}
}
return obj, nil
}
var errInScopeWithStmt = &Error{
Code: InternalErr,
Message: "wasm cannot be executed when 'with' statements are in-scope",
}
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"fmt"
"github.com/open-policy-agent/opa/v1/ast"
)
var nothingResolver ast.Resolver = illegalResolver{}
func builtinOPARuntime(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error {
if bctx.Runtime == nil {
return iter(ast.InternedEmptyObject)
}
if bctx.Runtime.Get(ast.InternedTerm("config")) != nil {
iface, err := ast.ValueToInterface(bctx.Runtime.Value, nothingResolver)
if err != nil {
return err
}
if object, ok := iface.(map[string]any); ok {
if cfgRaw, ok := object["config"]; ok {
if config, ok := cfgRaw.(map[string]any); ok {
configPurged, err := activeConfig(config)
if err != nil {
return err
}
object["config"] = configPurged
value, err := ast.InterfaceToValue(object)
if err != nil {
return err
}
return iter(ast.NewTerm(value))
}
}
}
}
return iter(bctx.Runtime)
}
func init() {
RegisterBuiltinFunc(ast.OPARuntime.Name, builtinOPARuntime)
}
func activeConfig(config map[string]any) (any, error) {
if config["services"] != nil {
err := removeServiceCredentials(config["services"])
if err != nil {
return nil, err
}
}
if config["keys"] != nil {
err := removeCryptoKeys(config["keys"])
if err != nil {
return nil, err
}
}
return config, nil
}
func removeServiceCredentials(x any) error {
switch x := x.(type) {
case []any:
for _, v := range x {
err := removeKey(v, "credentials")
if err != nil {
return err
}
}
case map[string]any:
for _, v := range x {
err := removeKey(v, "credentials")
if err != nil {
return err
}
}
default:
return fmt.Errorf("illegal service config type: %T", x)
}
return nil
}
func removeCryptoKeys(x any) error {
switch x := x.(type) {
case map[string]any:
for _, v := range x {
err := removeKey(v, "key", "private_key")
if err != nil {
return err
}
}
default:
return fmt.Errorf("illegal keys config type: %T", x)
}
return nil
}
func removeKey(x any, keys ...string) error {
val, ok := x.(map[string]any)
if !ok {
return errors.New("type assertion error")
}
for _, key := range keys {
delete(val, key)
}
return nil
}
type illegalResolver struct{}
func (illegalResolver) Resolve(ref ast.Ref) (any, error) {
return nil, fmt.Errorf("illegal value: %v", ref)
}
+523
View File
@@ -0,0 +1,523 @@
package topdown
import (
"cmp"
"container/list"
"fmt"
"slices"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
)
// saveSet contains a stack of terms that are considered 'unknown' during
// partial evaluation. Only var and ref terms (rooted at one of the root
// documents) can be added to the save set. Vars added to the save set are
// namespaced by the binding list they are added with. This means the save set
// can be shared across queries.
type saveSet struct {
instr *Instrumentation
l *list.List
}
func newSaveSet(ts []*ast.Term, b *bindings, instr *Instrumentation) *saveSet {
ss := &saveSet{
l: list.New(),
instr: instr,
}
ss.Push(ts, b)
return ss
}
func (ss *saveSet) Push(ts []*ast.Term, b *bindings) {
ss.l.PushBack(newSaveSetElem(ts, b))
}
func (ss *saveSet) Pop() {
ss.l.Remove(ss.l.Back())
}
// Contains returns true if the term t is contained in the save set. Non-var and
// non-ref terms are never contained. Ref terms are contained if they share a
// prefix with a ref that was added (in either direction).
func (ss *saveSet) Contains(t *ast.Term, b *bindings) bool {
if ss != nil {
ss.instr.startTimer(partialOpSaveSetContains)
ret := ss.contains(t, b)
ss.instr.stopTimer(partialOpSaveSetContains)
return ret
}
return false
}
func (ss *saveSet) contains(t *ast.Term, b *bindings) bool {
for el := ss.l.Back(); el != nil; el = el.Prev() {
if el.Value.(*saveSetElem).Contains(t, b) {
return true
}
}
return false
}
// ContainsRecursive returns true if the term t is or contains a term that is
// contained in the save set. This function will close over the binding list
// when it encounters vars.
func (ss *saveSet) ContainsRecursive(t *ast.Term, b *bindings) bool {
if ss != nil {
ss.instr.startTimer(partialOpSaveSetContainsRec)
ret := ss.containsrec(t, b)
ss.instr.stopTimer(partialOpSaveSetContainsRec)
return ret
}
return false
}
func (ss *saveSet) containsrec(t *ast.Term, b *bindings) bool {
var found bool
ast.WalkTerms(t, func(x *ast.Term) bool {
if _, ok := x.Value.(ast.Var); ok {
x1, b1 := b.apply(x)
if x1 != x || b1 != b {
if ss.containsrec(x1, b1) {
found = true
}
} else if ss.contains(x1, b1) {
found = true
}
}
return found
})
return found
}
func (ss *saveSet) Vars(caller *bindings) ast.VarSet {
result := ast.NewVarSet()
for x := ss.l.Front(); x != nil; x = x.Next() {
elem := x.Value.(*saveSetElem)
for _, v := range elem.vars {
if v, ok := elem.b.PlugNamespaced(v, caller).Value.(ast.Var); ok {
result.Add(v)
}
}
}
return result
}
func (ss *saveSet) String() string {
var buf []string
for x := ss.l.Front(); x != nil; x = x.Next() {
buf = append(buf, x.Value.(*saveSetElem).String())
}
return "(" + strings.Join(buf, " ") + ")"
}
type saveSetElem struct {
refs []ast.Ref
vars []*ast.Term
b *bindings
}
func newSaveSetElem(ts []*ast.Term, b *bindings) *saveSetElem {
var refs []ast.Ref
var vars []*ast.Term
for _, t := range ts {
switch v := t.Value.(type) {
case ast.Var:
vars = append(vars, t)
case ast.Ref:
refs = append(refs, v)
default:
panic("illegal value")
}
}
return &saveSetElem{
b: b,
vars: vars,
refs: refs,
}
}
func (sse *saveSetElem) Contains(t *ast.Term, b *bindings) bool {
switch other := t.Value.(type) {
case ast.Var:
return sse.containsVar(t, b)
case ast.Ref:
for _, ref := range sse.refs {
if ref.HasPrefix(other) || other.HasPrefix(ref) {
return true
}
}
return sse.containsVar(other[0], b)
}
return false
}
func (sse *saveSetElem) String() string {
return fmt.Sprintf("(refs: %v, vars: %v, b: %v)", sse.refs, sse.vars, sse.b)
}
func (sse *saveSetElem) containsVar(t *ast.Term, b *bindings) bool {
if b == sse.b {
for _, v := range sse.vars {
if v.Equal(t) {
return true
}
}
}
return false
}
// saveStack contains a stack of queries that represent the result of partial
// evaluation. When partial evaluation completes, the top of the stack
// represents a complete, partially evaluated query that can be saved and
// evaluated later.
//
// The result is stored in a stack so that partial evaluation of a query can be
// paused and then resumed in cases where different queries make up the result
// of partial evaluation, such as when a rule with a default clause is
// partially evaluated. In this case, the partially evaluated rule will be
// output in the support module.
type saveStack struct {
Stack []saveStackQuery
}
func newSaveStack() *saveStack {
return &saveStack{
Stack: []saveStackQuery{
{},
},
}
}
func (s *saveStack) PushQuery(query saveStackQuery) {
s.Stack = append(s.Stack, query)
}
func (s *saveStack) PopQuery() saveStackQuery {
last := s.Stack[len(s.Stack)-1]
s.Stack = s.Stack[:len(s.Stack)-1]
return last
}
func (s *saveStack) Peek() saveStackQuery {
return s.Stack[len(s.Stack)-1]
}
func (s *saveStack) Push(expr *ast.Expr, b1 *bindings, b2 *bindings) {
idx := len(s.Stack) - 1
s.Stack[idx] = append(s.Stack[idx], saveStackElem{expr, b1, b2})
}
func (s *saveStack) Pop() {
idx := len(s.Stack) - 1
query := s.Stack[idx]
s.Stack[idx] = query[:len(query)-1]
}
type saveStackQuery []saveStackElem
func (s saveStackQuery) Plug(b *bindings) ast.Body {
if len(s) == 0 {
return ast.NewBody(ast.NewExpr(ast.BooleanTerm(true)))
}
result := make(ast.Body, len(s))
for i := range s {
expr := s[i].Plug(b)
result.Set(expr, i)
}
return result
}
type saveStackElem struct {
Expr *ast.Expr
B1 *bindings
B2 *bindings
}
func (e saveStackElem) Plug(caller *bindings) *ast.Expr {
if e.B1 == nil && e.B2 == nil {
return e.Expr
}
expr := e.Expr.Copy()
switch terms := expr.Terms.(type) {
case []*ast.Term:
if expr.IsEquality() {
terms[1] = e.B1.PlugNamespaced(terms[1], caller)
terms[2] = e.B2.PlugNamespaced(terms[2], caller)
} else {
for i := 1; i < len(terms); i++ {
terms[i] = e.B1.PlugNamespaced(terms[i], caller)
}
}
case *ast.Term:
expr.Terms = e.B1.PlugNamespaced(terms, caller)
}
for i := range expr.With {
expr.With[i].Value = e.B1.PlugNamespaced(expr.With[i].Value, caller)
}
return expr
}
// saveSupport contains additional partially evaluated policies that are part
// of the output of partial evaluation.
//
// The support structure is accumulated as partial evaluation runs and then
// considered complete once partial evaluation finishes (but not before). This
// differs from partially evaluated queries which are considered complete as
// soon as each one finishes.
type saveSupport struct {
modules map[string]*ast.Module
}
func newSaveSupport() *saveSupport {
return &saveSupport{
modules: map[string]*ast.Module{},
}
}
func (s *saveSupport) List() []*ast.Module {
result := make([]*ast.Module, 0, len(s.modules))
for _, module := range s.modules {
result = append(result, module)
}
return result
}
func (s *saveSupport) Exists(path ast.Ref) bool {
pkg, ruleRef := splitPackageAndRule(path)
module, ok := s.modules[pkg.String()]
if !ok {
return false
}
if len(ruleRef) == 1 {
name := ruleRef[0].Value.(ast.Var)
for _, rule := range module.Rules {
if rule.Head.Name.Equal(name) {
return true
}
}
return false
}
for _, rule := range module.Rules {
if rule.Head.Ref().HasPrefix(ruleRef) {
return true
}
}
return false
}
func (s *saveSupport) Insert(path ast.Ref, rule *ast.Rule) {
pkg, _ := splitPackageAndRule(path)
s.InsertByPkg(pkg, rule)
}
func (s *saveSupport) InsertByPkg(pkg ast.Ref, rule *ast.Rule) {
k := pkg.String()
module, ok := s.modules[k]
if !ok {
module = &ast.Module{
Package: &ast.Package{
Path: pkg,
},
}
s.modules[k] = module
}
rule.Module = module
module.Rules = append(module.Rules, rule)
}
func splitPackageAndRule(path ast.Ref) (ast.Ref, ast.Ref) {
p := path.Copy()
ruleRefStart := 2 // path always contains at least 3 terms (data. + one term in package + rule name)
for i := ruleRefStart; i < len(p.StringPrefix()); i++ {
t := p[i]
if str, ok := t.Value.(ast.String); ok && ast.IsVarCompatibleString(string(str)) {
ruleRefStart = i
} else {
break
}
}
pkg := p[:ruleRefStart]
rule := p[ruleRefStart:]
rule[0].Value = ast.Var(rule[0].Value.(ast.String))
return pkg, rule
}
// saveRequired returns true if the statement x will result in some expressions
// being saved. This check allows the evaluator to evaluate statements
// completely during partial evaluation as long as they do not depend on any
// kind of unknown value or statements that would generate saves.
func saveRequired(c *ast.Compiler, ic *inliningControl, icIgnoreInternal bool, ss *saveSet, b *bindings, x any, rec bool) bool {
var found bool
vis := ast.NewGenericVisitor(func(node any) bool {
if found {
return found
}
switch node := node.(type) {
case *ast.Expr:
found = len(node.With) > 0
if found {
return found
}
if !ic.nondeterministicBuiltins { // skip evaluating non-det builtins for PE
found = ignoreExprDuringPartial(node)
}
case *ast.Term:
switch v := node.Value.(type) {
case ast.Var:
// Variables only need to be tested in the node from call site
// because once traversal recurses into a rule existing unknown
// variables are out-of-scope.
if !rec && ss.ContainsRecursive(node, b) {
found = true
}
case ast.Ref:
if ss.Contains(node, b) {
found = true
} else if ic.Disabled(v.ConstantPrefix(), icIgnoreInternal) {
found = true
} else {
for _, rule := range c.GetRulesDynamicWithOpts(v, ast.RulesOptions{IncludeHiddenModules: false}) {
if saveRequired(c, ic, icIgnoreInternal, ss, b, rule, true) {
found = true
break
}
}
}
}
}
return found
})
vis.Walk(x)
return found
}
func ignoreExprDuringPartial(expr *ast.Expr) bool {
if !expr.IsCall() {
return false
}
bi, ok := ast.BuiltinMap[expr.Operator().String()]
return ok && ignoreDuringPartial(bi)
}
func ignoreDuringPartial(bi *ast.Builtin) bool {
// Note(philipc): We keep this legacy check around to avoid breaking
// existing library users.
//nolint:staticcheck // We specifically ignore our own linter warning here.
return cmp.Or(slices.Contains(ast.IgnoreDuringPartialEval, bi), bi.Nondeterministic)
}
type inliningControl struct {
shallow bool
disable []disableInliningFrame
nondeterministicBuiltins bool // evaluate non-det builtins during PE (if args are known)
}
type disableInliningFrame struct {
internal bool
refs []ast.Ref
v ast.Var
}
func (i *inliningControl) PushDisable(x any, internal bool) {
if i == nil {
return
}
switch x := x.(type) {
case []ast.Ref:
i.PushDisableRefs(x, internal)
case ast.Var:
i.PushDisableVar(x, internal)
}
}
func (i *inliningControl) PushDisableRefs(refs []ast.Ref, internal bool) {
if i == nil {
return
}
i.disable = append(i.disable, disableInliningFrame{
internal: internal,
refs: refs,
})
}
func (i *inliningControl) PushDisableVar(v ast.Var, internal bool) {
if i == nil {
return
}
i.disable = append(i.disable, disableInliningFrame{
internal: internal,
v: v,
})
}
func (i *inliningControl) PopDisable() {
if i == nil {
return
}
i.disable = i.disable[:len(i.disable)-1]
}
func (i *inliningControl) Disabled(x any, ignoreInternal bool) bool {
if i == nil {
return false
}
switch x := x.(type) {
case ast.Ref:
return i.DisabledRef(x, ignoreInternal)
case ast.Var:
return i.DisabledVar(x, ignoreInternal)
}
return false
}
func (i *inliningControl) DisabledRef(ref ast.Ref, ignoreInternal bool) bool {
if i == nil {
return false
}
for _, frame := range i.disable {
if !frame.internal || !ignoreInternal {
for _, other := range frame.refs {
if other.HasPrefix(ref) || ref.HasPrefix(other) {
return true
}
}
}
}
return false
}
func (i *inliningControl) DisabledVar(v ast.Var, ignoreInternal bool) bool {
if i == nil {
return false
}
for _, frame := range i.disable {
if (!frame.internal || !ignoreInternal) && frame.v.Equal(v) {
return true
}
}
return false
}
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"fmt"
"github.com/open-policy-agent/opa/internal/semver"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinSemVerCompare(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
versionStringA, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
versionStringB, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
versionA, err := semver.Parse(string(versionStringA))
if err != nil {
return fmt.Errorf("operand 1: string %s is not a valid SemVer", versionStringA)
}
versionB, err := semver.Parse(string(versionStringB))
if err != nil {
return fmt.Errorf("operand 2: string %s is not a valid SemVer", versionStringB)
}
return iter(ast.InternedTerm(versionA.Compare(versionB)))
}
func builtinSemVerIsValid(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
versionString, err := builtins.StringOperand(operands[0].Value, 1)
if err == nil {
_, err = semver.Parse(string(versionString))
}
return iter(ast.InternedTerm(err == nil))
}
func init() {
RegisterBuiltinFunc(ast.SemVerCompare.Name, builtinSemVerCompare)
RegisterBuiltinFunc(ast.SemVerIsValid.Name, builtinSemVerIsValid)
}
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
// Deprecated: deprecated in v0.4.2 in favour of minus/infix "-" operation.
func builtinSetDiff(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s1, err := builtins.SetOperand(operands[0].Value, 1)
if err != nil {
return err
}
s2, err := builtins.SetOperand(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.NewTerm(s1.Diff(s2)))
}
// builtinSetIntersection returns the intersection of the given input sets
func builtinSetIntersection(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
inputSet, err := builtins.SetOperand(operands[0].Value, 1)
if err != nil {
return err
}
// empty input set
if inputSet.Len() == 0 {
return iter(ast.InternedEmptySet)
}
var result ast.Set
err = inputSet.Iter(func(x *ast.Term) error {
n, err := builtins.SetOperand(x.Value, 1)
if err != nil {
return err
}
if result == nil {
result = n
} else {
result = result.Intersect(n)
}
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(result))
}
// builtinSetUnion returns the union of the given input sets
func builtinSetUnion(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
// The set union logic here is manually inlined on purpose. By lifting
// this logic up a level and not doing pairwise set unions, we avoid
// many heap allocations. We also pre-allocate the result set by first
// counting total elements across all input sets.
inputSet, err := builtins.SetOperand(operands[0].Value, 1)
if err != nil {
return err
}
// First pass: count total elements for pre-allocation
totalSize := 0
err = inputSet.Iter(func(x *ast.Term) error {
item, err := builtins.SetOperand(x.Value, 1)
if err != nil {
return err
}
totalSize += item.Len()
return nil
})
if err != nil {
return err
}
// Pre-allocate result set with estimated capacity
result := ast.NewSetWithCapacity(totalSize)
err = inputSet.Iter(func(x *ast.Term) error {
item, _ := builtins.SetOperand(x.Value, 1) // error checked above
item.Foreach(result.Add)
return nil
})
if err != nil {
return err
}
return iter(ast.NewTerm(result))
}
func init() {
RegisterBuiltinFunc(ast.SetDiff.Name, builtinSetDiff)
RegisterBuiltinFunc(ast.Intersection.Name, builtinSetIntersection)
RegisterBuiltinFunc(ast.Union.Name, builtinSetUnion)
}
+73
View File
@@ -0,0 +1,73 @@
package topdown
import (
"bytes"
"io"
)
var _ io.Writer = (*sinkW)(nil)
type sinkWriter interface {
io.Writer
String() string
Grow(int)
WriteByte(byte) error
WriteString(string) (int, error)
}
type sinkW struct {
buf *bytes.Buffer
cancel Cancel
err error
}
func newSink(name string, hint int, c Cancel) sinkWriter {
b := &bytes.Buffer{}
if hint > 0 {
b.Grow(hint)
}
if c == nil {
return b
}
return &sinkW{
cancel: c,
buf: b,
err: Halt{
Err: &Error{
Code: CancelErr,
Message: name + ": timed out before finishing",
},
},
}
}
func (sw *sinkW) Grow(n int) {
sw.buf.Grow(n)
}
func (sw *sinkW) Write(bs []byte) (int, error) {
if sw.cancel.Cancelled() {
return 0, sw.err
}
return sw.buf.Write(bs)
}
func (sw *sinkW) WriteByte(b byte) error {
if sw.cancel.Cancelled() {
return sw.err
}
return sw.buf.WriteByte(b)
}
func (sw *sinkW) WriteString(s string) (int, error) {
if sw.cancel.Cancelled() {
return 0, sw.err
}
return sw.buf.WriteString(s)
}
func (sw *sinkW) String() string {
return sw.buf.String()
}
+816
View File
@@ -0,0 +1,816 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"fmt"
"math/big"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/tchap/go-patricia/v2/patricia"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
"github.com/open-policy-agent/opa/v1/util"
)
func builtinAnyPrefixMatch(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
a, b := operands[0].Value, operands[1].Value
var strs []string
switch a := a.(type) {
case ast.String:
strs = []string{string(a)}
case *ast.Array, ast.Set:
var err error
strs, err = builtins.StringSliceOperand(a, 1)
if err != nil {
return err
}
default:
return builtins.NewOperandTypeErr(1, a, "string", "set", "array")
}
var prefixes []string
switch b := b.(type) {
case ast.String:
prefixes = []string{string(b)}
case *ast.Array, ast.Set:
var err error
prefixes, err = builtins.StringSliceOperand(b, 2)
if err != nil {
return err
}
default:
return builtins.NewOperandTypeErr(2, b, "string", "set", "array")
}
return iter(ast.InternedTerm(anyStartsWithAny(strs, prefixes)))
}
func builtinAnySuffixMatch(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
a, b := operands[0].Value, operands[1].Value
var strsReversed []string
switch a := a.(type) {
case ast.String:
strsReversed = []string{reverseString(string(a))}
case *ast.Array, ast.Set:
strs, err := builtins.StringSliceOperand(a, 1)
if err != nil {
return err
}
strsReversed = make([]string, len(strs))
for i := range strs {
strsReversed[i] = reverseString(strs[i])
}
default:
return builtins.NewOperandTypeErr(1, a, "string", "set", "array")
}
var suffixesReversed []string
switch b := b.(type) {
case ast.String:
suffixesReversed = []string{reverseString(string(b))}
case *ast.Array, ast.Set:
suffixes, err := builtins.StringSliceOperand(b, 2)
if err != nil {
return err
}
suffixesReversed = make([]string, len(suffixes))
for i := range suffixes {
suffixesReversed[i] = reverseString(suffixes[i])
}
default:
return builtins.NewOperandTypeErr(2, b, "string", "set", "array")
}
return iter(ast.InternedTerm(anyStartsWithAny(strsReversed, suffixesReversed)))
}
func anyStartsWithAny(strs []string, prefixes []string) bool {
if len(strs) == 0 || len(prefixes) == 0 {
return false
}
if len(strs) == 1 && len(prefixes) == 1 {
return strings.HasPrefix(strs[0], prefixes[0])
}
trie := patricia.NewTrie()
for i := range strs {
trie.Insert([]byte(strs[i]), true)
}
for i := range prefixes {
if trie.MatchSubtree([]byte(prefixes[i])) {
return true
}
}
return false
}
func builtinFormatInt(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input, err := builtins.NumberOperand(operands[0].Value, 1)
if err != nil {
return err
}
base, err := builtins.NumberOperand(operands[1].Value, 2)
if err != nil {
return err
}
var format string
switch base {
case ast.Number("2"):
format = "%b"
case ast.Number("8"):
format = "%o"
case ast.Number("10"):
if i, ok := input.Int(); ok {
return iter(ast.InternedIntegerString(i))
}
format = "%d"
case ast.Number("16"):
format = "%x"
default:
return builtins.NewOperandEnumErr(2, "2", "8", "10", "16")
}
f := builtins.NumberToFloat(input)
i, _ := f.Int(nil)
return iter(ast.InternedTerm(fmt.Sprintf(format, i)))
}
func builtinConcat(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
join, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
// fast path for empty or single string array/set, allocates no memory
if term, ok := zeroOrOneStringTerm(operands[1].Value); ok {
return iter(term)
}
sb := newSink(ast.Concat.Name, 0, bctx.Cancel)
// NOTE(anderseknert):
// More or less Go's strings.Join implementation, but where we avoid
// creating an intermediate []string slice to pass to that function,
// as that's expensive (3.5x more space allocated). Instead we build
// the string directly using the sink to concatenate the string
// values from the array/set with the separator.
n := 0
switch b := operands[1].Value.(type) {
case *ast.Array:
l := b.Len()
for i := range l {
s, ok := b.Elem(i).Value.(ast.String)
if !ok {
return builtins.NewOperandElementErr(2, b, b.Elem(i).Value, "string")
}
n += len(s)
}
sep := string(join)
n += len(sep) * (l - 1)
sb.Grow(n)
if _, err := sb.WriteString(string(b.Elem(0).Value.(ast.String))); err != nil {
return err
}
if sep == "" {
for i := 1; i < l; i++ {
if _, err := sb.WriteString(string(b.Elem(i).Value.(ast.String))); err != nil {
return err
}
}
} else if len(sep) == 1 {
// when the separator is a single byte, sb.WriteByte is substantially faster
bsep := sep[0]
for i := 1; i < l; i++ {
if err := sb.WriteByte(bsep); err != nil {
return err
}
if _, err := sb.WriteString(string(b.Elem(i).Value.(ast.String))); err != nil {
return err
}
}
} else {
// for longer separators, there is no such difference between WriteString and Write
for i := 1; i < l; i++ {
if _, err := sb.WriteString(sep); err != nil {
return err
}
if _, err := sb.WriteString(string(b.Elem(i).Value.(ast.String))); err != nil {
return err
}
}
}
return iter(ast.InternedTerm(sb.String()))
case ast.Set:
for _, v := range b.Slice() {
s, ok := v.Value.(ast.String)
if !ok {
return builtins.NewOperandElementErr(2, b, v.Value, "string")
}
n += len(s)
}
sep := string(join)
l := b.Len()
n += len(sep) * (l - 1)
sb.Grow(n)
for i, v := range b.Slice() {
if _, err := sb.WriteString(string(v.Value.(ast.String))); err != nil {
return err
}
if i < l-1 {
if _, err := sb.WriteString(sep); err != nil {
return err
}
}
}
return iter(ast.InternedTerm(sb.String()))
}
return builtins.NewOperandTypeErr(2, operands[1].Value, "set", "array")
}
func zeroOrOneStringTerm(a ast.Value) (*ast.Term, bool) {
switch b := a.(type) {
case *ast.Array:
if b.Len() == 0 {
return ast.InternedEmptyString, true
}
if b.Len() == 1 {
e := b.Elem(0)
if _, ok := e.Value.(ast.String); ok {
return e, true
}
}
case ast.Set:
if b.Len() == 0 {
return ast.InternedEmptyString, true
}
if b.Len() == 1 {
e := b.Slice()[0]
if _, ok := e.Value.(ast.String); ok {
return e, true
}
}
}
return nil, false
}
func runesEqual(a, b []rune) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
func builtinIndexOf(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
base, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
search, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
if len(string(search)) == 0 {
return errors.New("empty search character")
}
if isASCII(string(base)) && isASCII(string(search)) {
// this is a false positive in the indexAlloc rule that thinks
// we're converting byte arrays to strings
//nolint:gocritic
return iter(ast.InternedTerm(strings.Index(string(base), string(search))))
}
baseRunes := []rune(string(base))
searchRunes := []rune(string(search))
searchLen := len(searchRunes)
for i, r := range baseRunes {
if len(baseRunes) >= i+searchLen {
if r == searchRunes[0] && runesEqual(baseRunes[i:i+searchLen], searchRunes) {
return iter(ast.InternedTerm(i))
}
} else {
break
}
}
return iter(ast.InternedTerm(-1))
}
func builtinIndexOfN(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
base, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
search, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
if len(string(search)) == 0 {
return errors.New("empty search character")
}
baseRunes := []rune(string(base))
searchRunes := []rune(string(search))
searchLen := len(searchRunes)
var arr []*ast.Term
for i, r := range baseRunes {
if len(baseRunes) >= i+searchLen {
if r == searchRunes[0] && runesEqual(baseRunes[i:i+searchLen], searchRunes) {
arr = append(arr, ast.InternedTerm(i))
}
} else {
break
}
}
return iter(ast.ArrayTerm(arr...))
}
func builtinSubstring(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
base, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
startIndex, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil {
return err
}
length, err := builtins.IntOperand(operands[2].Value, 3)
if err != nil {
return err
}
if startIndex < 0 {
return errors.New("negative offset")
}
sbase := string(base)
if sbase == "" {
return iter(ast.InternedEmptyString)
}
// Optimized path for the likely common case of ASCII strings.
// This allocates less memory and runs in about 1/3 the time.
if isASCII(sbase) {
if startIndex >= len(sbase) {
return iter(ast.InternedEmptyString)
}
if length < 0 {
return iter(ast.InternedTerm(sbase[startIndex:]))
}
if startIndex == 0 && length >= len(sbase) {
return iter(operands[0])
}
upto := min(len(sbase), startIndex+length)
return iter(ast.InternedTerm(sbase[startIndex:upto]))
}
if startIndex == 0 && length >= utf8.RuneCountInString(sbase) {
return iter(operands[0])
}
runes := []rune(base)
if startIndex >= len(runes) {
return iter(ast.InternedEmptyString)
}
var s string
if length < 0 {
s = string(runes[startIndex:])
} else {
upto := min(len(runes), startIndex+length)
s = string(runes[startIndex:upto])
}
return iter(ast.InternedTerm(s))
}
func isASCII(s string) bool {
for i := range len(s) {
if s[i] > unicode.MaxASCII {
return false
}
}
return true
}
func builtinContains(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
substr, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.InternedTerm(strings.Contains(string(s), string(substr))))
}
func builtinStringCount(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
substr, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
baseTerm := string(s)
searchTerm := string(substr)
count := strings.Count(baseTerm, searchTerm)
return iter(ast.InternedTerm(count))
}
func builtinStartsWith(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
prefix, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.InternedTerm(strings.HasPrefix(string(s), string(prefix))))
}
func builtinEndsWith(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
suffix, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
return iter(ast.InternedTerm(strings.HasSuffix(string(s), string(suffix))))
}
func builtinLower(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
arg := string(s)
low := strings.ToLower(arg)
if arg == low {
return iter(operands[0])
}
return iter(ast.InternedTerm(low))
}
func builtinUpper(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
arg := string(s)
upp := strings.ToUpper(arg)
if arg == upp {
return iter(operands[0])
}
return iter(ast.InternedTerm(upp))
}
func builtinSplit(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
d, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
text, delim := string(s), string(d)
if !strings.Contains(text, delim) {
return iter(ast.ArrayTerm(operands[0]))
}
return iter(ast.ArrayTerm(util.SplitMap(text, delim, ast.InternedTerm)...))
}
func builtinReplace(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
old, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
n, err := builtins.StringOperand(operands[2].Value, 3)
if err != nil {
return err
}
sink := newSink(ast.Replace.Name, len(s), bctx.Cancel)
replacer := strings.NewReplacer(string(old), string(n))
if _, err := replacer.WriteString(sink, string(s)); err != nil {
return err
}
replaced := sink.String()
if replaced == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(replaced))
}
func builtinReplaceN(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
patterns, err := builtins.ObjectOperand(operands[0].Value, 1)
if err != nil {
return err
}
s, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
keys := util.SortedFunc(patterns.Keys(), ast.TermValueCompare)
pairs := make([]string, 0, len(keys)*2)
for _, k := range keys {
keyVal, ok := k.Value.(ast.String)
if !ok {
return builtins.NewOperandErr(1, "non-string key found in pattern object")
}
strVal, ok := patterns.Get(k).Value.(ast.String)
if !ok {
return builtins.NewOperandErr(1, "non-string value found in pattern object")
}
pairs = append(pairs, string(keyVal), string(strVal))
}
sink := newSink(ast.ReplaceN.Name, len(s), bctx.Cancel)
replacer := strings.NewReplacer(pairs...)
if _, err := replacer.WriteString(sink, string(s)); err != nil {
return err
}
return iter(ast.InternedTerm(sink.String()))
}
func builtinTrim(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
c, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
str := string(s)
trimmed := strings.Trim(str, string(c))
if trimmed == str {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinTrimLeft(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
c, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
trimmed := strings.TrimLeft(string(s), string(c))
if trimmed == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinTrimPrefix(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
pre, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
trimmed := strings.TrimPrefix(string(s), string(pre))
if trimmed == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinTrimRight(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
c, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
trimmed := strings.TrimRight(string(s), string(c))
if trimmed == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinTrimSuffix(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
suf, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
trimmed := strings.TrimSuffix(string(s), string(suf))
if trimmed == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinTrimSpace(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
trimmed := strings.TrimSpace(string(s))
if trimmed == string(s) {
return iter(operands[0])
}
return iter(ast.InternedTerm(trimmed))
}
func builtinSprintf(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
astArr, ok := operands[1].Value.(*ast.Array)
if !ok {
return builtins.NewOperandTypeErr(2, operands[1].Value, "array")
}
// Optimized path for where sprintf is used as a "to_string" function for
// a single integer, i.e. sprintf("%d", [x]) where x is an integer.
if s == "%d" && astArr.Len() == 1 {
if n, ok := astArr.Elem(0).Value.(ast.Number); ok {
if i, ok := n.Int(); ok {
if interned := ast.InternedIntegerString(i); interned != nil {
return iter(interned)
}
return iter(ast.StringTerm(strconv.Itoa(i)))
}
}
}
args := make([]any, astArr.Len())
for i := range args {
switch v := astArr.Elem(i).Value.(type) {
case ast.Number:
if n, ok := v.Int(); ok {
args[i] = n
} else if b, ok := new(big.Int).SetString(v.String(), 10); ok {
args[i] = b
} else if f, ok := v.Float64(); ok {
args[i] = f
} else {
args[i] = v.String()
}
case ast.String:
args[i] = string(v)
default:
args[i] = astArr.Elem(i).String()
}
}
return iter(ast.InternedTerm(fmt.Sprintf(string(s), args...)))
}
func builtinReverse(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
s, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
return iter(ast.InternedTerm(reverseString(string(s))))
}
func reverseString(str string) string {
var buf []byte
var arr [255]byte
size := len(str)
if size < 255 {
buf = arr[:size:size]
} else {
buf = make([]byte, size)
}
for start := 0; start < size; {
r, n := utf8.DecodeRuneInString(str[start:])
start += n
utf8.EncodeRune(buf[size-start:], r)
}
return string(buf)
}
func init() {
RegisterBuiltinFunc(ast.FormatInt.Name, builtinFormatInt)
RegisterBuiltinFunc(ast.Concat.Name, builtinConcat)
RegisterBuiltinFunc(ast.IndexOf.Name, builtinIndexOf)
RegisterBuiltinFunc(ast.IndexOfN.Name, builtinIndexOfN)
RegisterBuiltinFunc(ast.Substring.Name, builtinSubstring)
RegisterBuiltinFunc(ast.Contains.Name, builtinContains)
RegisterBuiltinFunc(ast.StringCount.Name, builtinStringCount)
RegisterBuiltinFunc(ast.StartsWith.Name, builtinStartsWith)
RegisterBuiltinFunc(ast.EndsWith.Name, builtinEndsWith)
RegisterBuiltinFunc(ast.Upper.Name, builtinUpper)
RegisterBuiltinFunc(ast.Lower.Name, builtinLower)
RegisterBuiltinFunc(ast.Split.Name, builtinSplit)
RegisterBuiltinFunc(ast.Replace.Name, builtinReplace)
RegisterBuiltinFunc(ast.ReplaceN.Name, builtinReplaceN)
RegisterBuiltinFunc(ast.Trim.Name, builtinTrim)
RegisterBuiltinFunc(ast.TrimLeft.Name, builtinTrimLeft)
RegisterBuiltinFunc(ast.TrimPrefix.Name, builtinTrimPrefix)
RegisterBuiltinFunc(ast.TrimRight.Name, builtinTrimRight)
RegisterBuiltinFunc(ast.TrimSuffix.Name, builtinTrimSuffix)
RegisterBuiltinFunc(ast.TrimSpace.Name, builtinTrimSpace)
RegisterBuiltinFunc(ast.Sprintf.Name, builtinSprintf)
RegisterBuiltinFunc(ast.AnyPrefixMatch.Name, builtinAnyPrefixMatch)
RegisterBuiltinFunc(ast.AnySuffixMatch.Name, builtinAnySuffixMatch)
RegisterBuiltinFunc(ast.StringReverse.Name, builtinReverse)
}
+242
View File
@@ -0,0 +1,242 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func bothObjects(t1, t2 *ast.Term) (bool, ast.Object, ast.Object) {
if (t1 == nil) || (t2 == nil) {
return false, nil, nil
}
obj1, ok := t1.Value.(ast.Object)
if !ok {
return false, nil, nil
}
obj2, ok := t2.Value.(ast.Object)
if !ok {
return false, nil, nil
}
return true, obj1, obj2
}
func bothSets(t1, t2 *ast.Term) (bool, ast.Set, ast.Set) {
if (t1 == nil) || (t2 == nil) {
return false, nil, nil
}
set1, ok := t1.Value.(ast.Set)
if !ok {
return false, nil, nil
}
set2, ok := t2.Value.(ast.Set)
if !ok {
return false, nil, nil
}
return true, set1, set2
}
func bothArrays(t1, t2 *ast.Term) (bool, *ast.Array, *ast.Array) {
if (t1 == nil) || (t2 == nil) {
return false, nil, nil
}
array1, ok := t1.Value.(*ast.Array)
if !ok {
return false, nil, nil
}
array2, ok := t2.Value.(*ast.Array)
if !ok {
return false, nil, nil
}
return true, array1, array2
}
func arraySet(t1, t2 *ast.Term) (bool, *ast.Array, ast.Set) {
if (t1 == nil) || (t2 == nil) {
return false, nil, nil
}
array, ok := t1.Value.(*ast.Array)
if !ok {
return false, nil, nil
}
set, ok := t2.Value.(ast.Set)
if !ok {
return false, nil, nil
}
return true, array, set
}
// objectSubset implements the subset operation on a pair of objects.
//
// This function will try to recursively apply the subset operation where it
// can, such as if both super and sub have an object or set as the value
// associated with a key.
func objectSubset(super ast.Object, sub ast.Object) bool {
var superTerm *ast.Term
notSubset := sub.Until(func(key, subTerm *ast.Term) bool {
// This really wants to be a for loop, hence the somewhat
// weird internal structure. However, using Until() in this
// was is a performance optimization, as it avoids performing
// any key hashing on the sub-object.
superTerm = super.Get(key)
// subTerm can't be nil because we got it from Until(), so
// we only need to verify that super is non-nil.
if superTerm == nil {
return true // break, not a subset
}
if subTerm.Equal(superTerm) {
return false // continue
}
// If both of the terms are objects then we want to apply
// the subset operation recursively, otherwise we just compare
// them normally. If only one term is an object, then we
// do a normal comparison which will come up false.
if ok, superObj, subObj := bothObjects(superTerm, subTerm); ok {
return !objectSubset(superObj, subObj)
}
if ok, superSet, subSet := bothSets(superTerm, subTerm); ok {
return !setSubset(superSet, subSet)
}
if ok, superArray, subArray := bothArrays(superTerm, subTerm); ok {
return !arraySubset(superArray, subArray)
}
// We have already checked for exact equality, as well as for
// all of the types of nested subsets we care about, so if we
// get here it means this isn't a subset.
return true // break, not a subset
})
return !notSubset
}
// setSubset implements the subset operation on sets.
//
// Unlike in the object case, this is not recursive, we just compare values
// using ast.Set.Contains() because we have no well-defined way to "match up"
// objects that are in different sets.
func setSubset(super ast.Set, sub ast.Set) bool {
for _, elem := range sub.Slice() {
if !super.Contains(elem) {
return false
}
}
return true
}
// arraySubset implements the subset operation on arrays.
//
// This is defined to mean that the entire "sub" array must appear in
// the "super" array. For the same rationale as setSubset(), we do not attempt
// to recurse into values.
func arraySubset(super, sub *ast.Array) bool {
// Notice that this is essentially string search. The naive approach
// used here is O(n^2). This should probably be rewritten later to use
// Boyer-Moore or something.
if sub.Len() > super.Len() {
return false
}
if sub.Equal(super) {
return true
}
superCursor := 0
subCursor := 0
for {
if subCursor == sub.Len() {
return true
}
if superCursor+subCursor == super.Len() {
return false
}
superElem := super.Elem(superCursor + subCursor)
if superElem == nil {
return false
}
subElem := sub.Elem(subCursor)
if superElem.Value.Compare(subElem.Value) == 0 {
subCursor++
} else {
superCursor++
subCursor = 0
}
}
}
// arraySetSubset implements the subset operation on array and set.
//
// This is defined to mean that the entire "sub" set must appear in
// the "super" array with no consideration of ordering.
// For the same rationale as setSubset(), we do not attempt
// to recurse into values.
func arraySetSubset(super *ast.Array, sub ast.Set) bool {
unmatched := sub.Len()
return super.Until(func(t *ast.Term) bool {
if sub.Contains(t) {
unmatched--
}
if unmatched == 0 {
return true
}
return false
})
}
func builtinObjectSubset(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
superTerm := operands[0]
subTerm := operands[1]
if ok, superObj, subObj := bothObjects(superTerm, subTerm); ok {
// Both operands are objects.
return iter(ast.InternedTerm(objectSubset(superObj, subObj)))
}
if ok, superSet, subSet := bothSets(superTerm, subTerm); ok {
// Both operands are sets.
return iter(ast.InternedTerm(setSubset(superSet, subSet)))
}
if ok, superArray, subArray := bothArrays(superTerm, subTerm); ok {
// Both operands are sets.
return iter(ast.InternedTerm(arraySubset(superArray, subArray)))
}
if ok, superArray, subSet := arraySet(superTerm, subTerm); ok {
// Super operand is array and sub operand is set
return iter(ast.InternedTerm(arraySetSubset(superArray, subSet)))
}
return builtins.ErrOperand("both arguments object.subset must be of the same type or array and set")
}
func init() {
RegisterBuiltinFunc(ast.ObjectSubset.Name, builtinObjectSubset)
}
+45
View File
@@ -0,0 +1,45 @@
package topdown
import (
"bytes"
"strings"
"text/template"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func renderTemplate(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
preContentTerm, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
templateVariablesTerm, err := builtins.ObjectOperand(operands[1].Value, 2)
if err != nil {
return err
}
var templateVariables map[string]any
if err := ast.As(templateVariablesTerm, &templateVariables); err != nil {
return err
}
tmpl, err := template.New("template").Parse(string(preContentTerm))
if err != nil {
return err
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, templateVariables); err != nil {
return err
}
res := strings.ReplaceAll(buf.String(), "<no value>", "<undefined>")
return iter(ast.StringTerm(res))
}
func init() {
RegisterBuiltinFunc(ast.RenderTemplate.Name, renderTemplate)
}
@@ -0,0 +1,45 @@
// Copyright 2025 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
func builtinTemplateString(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
arr, err := builtins.ArrayOperand(operands[0].Value, 1)
if err != nil {
return err
}
buf := make([]string, arr.Len())
var count int
err = builtinPrintCrossProductOperands(bctx.Location, buf, arr, 0, func(buf []string) error {
count += 1
// Precautionary run-time assertion that template-strings can't produce multiple outputs; e.g. for custom relation type built-ins not known at compile-time.
if count > 1 {
return Halt{Err: &Error{
Code: ConflictErr,
Location: bctx.Location,
Message: "template-strings must not produce multiple outputs",
}}
}
return nil
})
if err != nil {
return err
}
return iter(ast.StringTerm(strings.Join(buf, "")))
}
func init() {
RegisterBuiltinFunc(ast.InternalTemplateString.Name, builtinTemplateString)
}
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2025 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import "github.com/open-policy-agent/opa/v1/ast"
const TestCaseOp Op = "TestCase"
func builtinTestCase(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
e := &Event{
Op: TestCaseOp,
QueryID: bctx.QueryID,
Node: ast.NewExpr([]*ast.Term{
ast.NewTerm(ast.InternalTestCase.Ref()),
ast.NewTerm(operands[0].Value),
}),
}
for _, tracer := range bctx.QueryTracers {
tracer.TraceEvent(*e)
}
return iter(ast.BooleanTerm(true))
}
func init() {
RegisterBuiltinFunc(ast.InternalTestCase.Name, builtinTestCase)
}
+341
View File
@@ -0,0 +1,341 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"encoding/json"
"errors"
"math"
"math/big"
"strconv"
"sync"
"time"
_ "time/tzdata" // this is needed to have LoadLocation when no filesystem tzdata is available
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
var tzCache map[string]*time.Location
var tzCacheMutex *sync.Mutex
// 1677-09-21T00:12:43.145224192-00:00
var minDateAllowedForNsConversion = time.Unix(0, math.MinInt64)
// 2262-04-11T23:47:16.854775807-00:00
var maxDateAllowedForNsConversion = time.Unix(0, math.MaxInt64)
func toSafeUnixNano(t time.Time, iter func(*ast.Term) error) error {
if t.Before(minDateAllowedForNsConversion) || t.After(maxDateAllowedForNsConversion) {
return errors.New("time outside of valid range")
}
return iter(ast.NewTerm(ast.Number(int64ToJSONNumber(t.UnixNano()))))
}
func builtinTimeNowNanos(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error {
return iter(bctx.Time)
}
func builtinTimeParseNanos(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
format, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
value, err := builtins.StringOperand(operands[1].Value, 2)
if err != nil {
return err
}
formatStr := string(format)
// look for the formatStr in our acceptedTimeFormats and
// use the constant instead if it matches
if f, ok := acceptedTimeFormats[formatStr]; ok {
formatStr = f
}
result, err := time.Parse(formatStr, string(value))
if err != nil {
return err
}
return toSafeUnixNano(result, iter)
}
func builtinTimeParseRFC3339Nanos(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
value, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
result, err := time.Parse(time.RFC3339, string(value))
if err != nil {
return err
}
return toSafeUnixNano(result, iter)
}
func builtinParseDurationNanos(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
duration, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
value, err := time.ParseDuration(string(duration))
if err != nil {
return err
}
return iter(ast.NumberTerm(int64ToJSONNumber(int64(value))))
}
// Represent exposed constants for formatting from the stdlib time pkg
var acceptedTimeFormats = map[string]string{
"ANSIC": time.ANSIC,
"UnixDate": time.UnixDate,
"RubyDate": time.RubyDate,
"RFC822": time.RFC822,
"RFC822Z": time.RFC822Z,
"RFC850": time.RFC850,
"RFC1123": time.RFC1123,
"RFC1123Z": time.RFC1123Z,
"RFC3339": time.RFC3339,
"RFC3339Nano": time.RFC3339Nano,
}
func builtinFormat(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t, layout, err := tzTime(operands[0].Value)
if err != nil {
return err
}
// Using RFC3339Nano time formatting as default
if layout == "" {
layout = time.RFC3339Nano
} else if layoutStr, ok := acceptedTimeFormats[layout]; ok {
// if we can find a constant specified, use the constant
layout = layoutStr
}
// otherwise try to treat the fmt string as a datetime fmt string
timestamp := t.Format(layout)
return iter(ast.StringTerm(timestamp))
}
func builtinDate(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t, _, err := tzTime(operands[0].Value)
if err != nil {
return err
}
year, month, day := t.Date()
return iter(ast.ArrayTerm(ast.InternedTerm(year), ast.InternedTerm(int(month)), ast.InternedTerm(day)))
}
func builtinClock(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t, _, err := tzTime(operands[0].Value)
if err != nil {
return err
}
hour, minute, second := t.Clock()
result := ast.NewArray(ast.InternedTerm(hour), ast.InternedTerm(minute), ast.InternedTerm(second))
return iter(ast.NewTerm(result))
}
func builtinWeekday(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t, _, err := tzTime(operands[0].Value)
if err != nil {
return err
}
weekday := t.Weekday().String()
return iter(ast.StringTerm(weekday))
}
func builtinAddDate(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t, _, err := tzTime(operands[0].Value)
if err != nil {
return err
}
years, err := builtins.IntOperand(operands[1].Value, 2)
if err != nil {
return err
}
months, err := builtins.IntOperand(operands[2].Value, 3)
if err != nil {
return err
}
days, err := builtins.IntOperand(operands[3].Value, 4)
if err != nil {
return err
}
result := t.AddDate(years, months, days)
return toSafeUnixNano(result, iter)
}
func builtinDiff(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
t1, _, err := tzTime(operands[0].Value)
if err != nil {
return err
}
t2, _, err := tzTime(operands[1].Value)
if err != nil {
return err
}
// The following implementation of this function is taken
// from https://github.com/icza/gox licensed under Apache 2.0.
// The only modification made is to variable names.
//
// For details, see https://stackoverflow.com/a/36531443/1705598
//
// Copyright 2021 icza
// BEGIN REDISTRIBUTION FROM APACHE 2.0 LICENSED PROJECT
if t1.Location() != t2.Location() {
t2 = t2.In(t1.Location())
}
if t1.After(t2) {
t1, t2 = t2, t1
}
y1, M1, d1 := t1.Date()
y2, M2, d2 := t2.Date()
h1, m1, s1 := t1.Clock()
h2, m2, s2 := t2.Clock()
year := y2 - y1
month := int(M2 - M1)
day := d2 - d1
hour := h2 - h1
min := m2 - m1
sec := s2 - s1
// Normalize negative values
if sec < 0 {
sec += 60
min--
}
if min < 0 {
min += 60
hour--
}
if hour < 0 {
hour += 24
day--
}
if day < 0 {
// Days in month:
t := time.Date(y1, M1, 32, 0, 0, 0, 0, time.UTC)
day += 32 - t.Day()
month--
}
if month < 0 {
month += 12
year--
}
// END REDISTRIBUTION FROM APACHE 2.0 LICENSED PROJECT
return iter(ast.ArrayTerm(ast.InternedTerm(year), ast.InternedTerm(month), ast.InternedTerm(day),
ast.InternedTerm(hour), ast.InternedTerm(min), ast.InternedTerm(sec)))
}
func tzTime(a ast.Value) (t time.Time, lay string, err error) {
var nVal ast.Value
loc := time.UTC
layout := ""
switch va := a.(type) {
case *ast.Array:
if va.Len() == 0 {
return time.Time{}, layout, builtins.NewOperandTypeErr(1, a, "either number (ns) or [number (ns), string (tz)]")
}
nVal, err = builtins.NumberOperand(va.Elem(0).Value, 1)
if err != nil {
return time.Time{}, layout, err
}
if va.Len() > 1 {
tzVal, err := builtins.StringOperand(va.Elem(1).Value, 1)
if err != nil {
return time.Time{}, layout, err
}
tzName := string(tzVal)
switch tzName {
case "", "UTC":
// loc is already UTC
case "Local":
loc = time.Local
default:
var ok bool
tzCacheMutex.Lock()
loc, ok = tzCache[tzName]
if !ok {
loc, err = time.LoadLocation(tzName)
if err != nil {
tzCacheMutex.Unlock()
return time.Time{}, layout, err
}
tzCache[tzName] = loc
}
tzCacheMutex.Unlock()
}
}
if va.Len() > 2 {
lay, err := builtins.StringOperand(va.Elem(2).Value, 1)
if err != nil {
return time.Time{}, layout, err
}
layout = string(lay)
}
case ast.Number:
nVal = a
default:
return time.Time{}, layout, builtins.NewOperandTypeErr(1, a, "either number (ns) or [number (ns), string (tz)]")
}
value, err := builtins.NumberOperand(nVal, 1)
if err != nil {
return time.Time{}, layout, err
}
f := builtins.NumberToFloat(value)
i64, acc := f.Int64()
if acc != big.Exact {
return time.Time{}, layout, errors.New("timestamp too big")
}
t = time.Unix(0, i64).In(loc)
return t, layout, nil
}
func int64ToJSONNumber(i int64) json.Number {
return json.Number(strconv.FormatInt(i, 10))
}
func init() {
RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos)
RegisterBuiltinFunc(ast.ParseRFC3339Nanos.Name, builtinTimeParseRFC3339Nanos)
RegisterBuiltinFunc(ast.ParseNanos.Name, builtinTimeParseNanos)
RegisterBuiltinFunc(ast.ParseDurationNanos.Name, builtinParseDurationNanos)
RegisterBuiltinFunc(ast.Format.Name, builtinFormat)
RegisterBuiltinFunc(ast.Date.Name, builtinDate)
RegisterBuiltinFunc(ast.Clock.Name, builtinClock)
RegisterBuiltinFunc(ast.Weekday.Name, builtinWeekday)
RegisterBuiltinFunc(ast.AddDate.Name, builtinAddDate)
RegisterBuiltinFunc(ast.Diff.Name, builtinDiff)
tzCacheMutex = &sync.Mutex{}
tzCache = make(map[string]*time.Location)
}
File diff suppressed because it is too large Load Diff
+897
View File
@@ -0,0 +1,897 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"bytes"
"fmt"
"io"
"slices"
"strings"
iStrs "github.com/open-policy-agent/opa/internal/strings"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
const (
minLocationWidth = 5 // len("query")
maxIdealLocationWidth = 64
columnPadding = 4
maxExprVarWidth = 32
maxPrettyExprVarWidth = 64
)
// Op defines the types of tracing events.
type Op string
const (
// EnterOp is emitted when a new query is about to be evaluated.
EnterOp Op = "Enter"
// ExitOp is emitted when a query has evaluated to true.
ExitOp Op = "Exit"
// EvalOp is emitted when an expression is about to be evaluated.
EvalOp Op = "Eval"
// RedoOp is emitted when an expression, rule, or query is being re-evaluated.
RedoOp Op = "Redo"
// SaveOp is emitted when an expression is saved instead of evaluated
// during partial evaluation.
SaveOp Op = "Save"
// FailOp is emitted when an expression evaluates to false.
FailOp Op = "Fail"
// DuplicateOp is emitted when a query has produced a duplicate value. The search
// will stop at the point where the duplicate was emitted and backtrack.
DuplicateOp Op = "Duplicate"
// NoteOp is emitted when an expression invokes a tracing built-in function.
NoteOp Op = "Note"
// IndexOp is emitted during an expression evaluation to represent lookup
// matches.
IndexOp Op = "Index"
// WasmOp is emitted when resolving a ref using an external
// Resolver.
WasmOp Op = "Wasm"
// UnifyOp is emitted when two terms are unified. Node will be set to an
// equality expression with the two terms. This Node will not have location
// info.
UnifyOp Op = "Unify"
FailedAssertionOp Op = "FailedAssertion"
)
// VarMetadata provides some user facing information about
// a variable in some policy.
type VarMetadata struct {
Name ast.Var `json:"name"`
Location *ast.Location `json:"location"`
}
// Event contains state associated with a tracing event.
type Event struct {
Op Op // Identifies type of event.
Node ast.Node // Contains AST node relevant to the event.
Location *ast.Location // The location of the Node this event relates to.
QueryID uint64 // Identifies the query this event belongs to.
ParentID uint64 // Identifies the parent query this event belongs to.
Locals *ast.ValueMap // Contains local variable bindings from the query context. Nil if variables were not included in the trace event.
LocalMetadata map[ast.Var]VarMetadata // Contains metadata for the local variable bindings. Nil if variables were not included in the trace event.
Message string // Contains message for Note events.
Ref *ast.Ref // Identifies the subject ref for the event. Only applies to Index and Wasm operations.
input *ast.Term
bindings *bindings
localVirtualCacheSnapshot *ast.ValueMap
}
func (evt *Event) WithInput(input *ast.Term) *Event {
evt.input = input
return evt
}
// HasRule returns true if the Event contains an ast.Rule.
func (evt *Event) HasRule() bool {
_, ok := evt.Node.(*ast.Rule)
return ok
}
// HasBody returns true if the Event contains an ast.Body.
func (evt *Event) HasBody() bool {
_, ok := evt.Node.(ast.Body)
return ok
}
// HasExpr returns true if the Event contains an ast.Expr.
func (evt *Event) HasExpr() bool {
_, ok := evt.Node.(*ast.Expr)
return ok
}
// Equal returns true if this event is equal to the other event.
func (evt *Event) Equal(other *Event) bool {
if evt.Op != other.Op {
return false
}
if evt.QueryID != other.QueryID {
return false
}
if evt.ParentID != other.ParentID {
return false
}
if !evt.equalNodes(other) {
return false
}
return evt.Locals.Equal(other.Locals)
}
func (evt *Event) String() string {
return fmt.Sprintf("%v %v %v (qid=%v, pqid=%v)", evt.Op, evt.Node, evt.Locals, evt.QueryID, evt.ParentID)
}
// Input returns the input object as it was at the event.
func (evt *Event) Input() *ast.Term {
return evt.input
}
// Plug plugs event bindings into the provided ast.Term. Because bindings are mutable, this only makes sense to do when
// the event is emitted rather than on recorded trace events as the bindings are going to be different by then.
func (evt *Event) Plug(term *ast.Term) *ast.Term {
return evt.bindings.Plug(term)
}
func (evt *Event) equalNodes(other *Event) bool {
switch a := evt.Node.(type) {
case ast.Body:
if b, ok := other.Node.(ast.Body); ok {
return a.Equal(b)
}
case *ast.Rule:
if b, ok := other.Node.(*ast.Rule); ok {
return a.Equal(b)
}
case *ast.Expr:
if b, ok := other.Node.(*ast.Expr); ok {
return a.Equal(b)
}
case nil:
return other.Node == nil
}
return false
}
// Tracer defines the interface for tracing in the top-down evaluation engine.
//
// Deprecated: Use QueryTracer instead.
type Tracer interface {
Enabled() bool
Trace(*Event)
}
// QueryTracer defines the interface for tracing in the top-down evaluation engine.
// The implementation can provide additional configuration to modify the tracing
// behavior for query evaluations.
type QueryTracer interface {
Enabled() bool
TraceEvent(Event)
Config() TraceConfig
}
// TraceConfig defines some common configuration for Tracer implementations
type TraceConfig struct {
PlugLocalVars bool // Indicate whether to plug local variable bindings before calling into the tracer.
}
// legacyTracer Implements the QueryTracer interface by wrapping an older Tracer instance.
type legacyTracer struct {
t Tracer
}
func (l *legacyTracer) Enabled() bool {
return l.t.Enabled()
}
func (*legacyTracer) Config() TraceConfig {
return TraceConfig{
PlugLocalVars: true, // For backwards compatibility old tracers will plug local variables
}
}
func (l *legacyTracer) TraceEvent(evt Event) {
l.t.Trace(&evt)
}
// WrapLegacyTracer will create a new QueryTracer which wraps an
// older Tracer instance.
func WrapLegacyTracer(tracer Tracer) QueryTracer {
return &legacyTracer{t: tracer}
}
// BufferTracer implements the Tracer and QueryTracer interface by
// simply buffering all events received.
type BufferTracer []*Event
// NewBufferTracer returns a new BufferTracer.
func NewBufferTracer() *BufferTracer {
return &BufferTracer{}
}
// Enabled always returns true if the BufferTracer is instantiated.
func (b *BufferTracer) Enabled() bool {
return b != nil
}
// Trace adds the event to the buffer.
//
// Deprecated: Use TraceEvent instead.
func (b *BufferTracer) Trace(evt *Event) {
*b = append(*b, evt)
}
// TraceEvent adds the event to the buffer.
func (b *BufferTracer) TraceEvent(evt Event) {
*b = append(*b, &evt)
}
// Config returns the Tracers standard configuration
func (*BufferTracer) Config() TraceConfig {
return TraceConfig{PlugLocalVars: true}
}
// PrettyTrace pretty prints the trace to the writer.
func PrettyTrace(w io.Writer, trace []*Event) {
PrettyTraceWithOpts(w, trace, PrettyTraceOptions{})
}
// PrettyTraceWithLocation prints the trace to the writer and includes location information
func PrettyTraceWithLocation(w io.Writer, trace []*Event) {
PrettyTraceWithOpts(w, trace, PrettyTraceOptions{Locations: true})
}
type PrettyTraceOptions struct {
Locations bool // Include location information
ExprVariables bool // Include variables found in the expression
LocalVariables bool // Include all local variables
}
type traceRow []string
func (r *traceRow) add(s string) {
*r = append(*r, s)
}
type traceTable struct {
rows []traceRow
maxWidths []int
}
func (t *traceTable) add(row traceRow) {
t.rows = append(t.rows, row)
for i := range row {
if i >= len(t.maxWidths) {
t.maxWidths = append(t.maxWidths, len(row[i]))
} else if len(row[i]) > t.maxWidths[i] {
t.maxWidths[i] = len(row[i])
}
}
}
func (t *traceTable) write(w io.Writer, padding int) {
for _, row := range t.rows {
for i, cell := range row {
width := t.maxWidths[i] + padding
if i < len(row)-1 {
_, _ = fmt.Fprintf(w, "%-*s ", width, cell)
} else {
_, _ = fmt.Fprintf(w, "%s", cell)
}
}
_, _ = fmt.Fprintln(w)
}
}
func PrettyTraceWithOpts(w io.Writer, trace []*Event, opts PrettyTraceOptions) {
depths := depths{}
// FIXME: Can we shorten each location as we process each trace event instead of beforehand?
filePathAliases, _ := getShortenedFileNames(trace)
table := traceTable{}
for _, event := range trace {
depth := depths.GetOrSet(event.QueryID, event.ParentID)
row := traceRow{}
if opts.Locations {
location := formatLocation(event, filePathAliases)
row.add(location)
}
row.add(formatEvent(event, depth))
if opts.ExprVariables {
vars := exprLocalVars(event)
keys := sortedKeys(vars)
buf := new(bytes.Buffer)
buf.WriteString("{")
for i, k := range keys {
if i > 0 {
buf.WriteString(", ")
}
_, _ = fmt.Fprintf(buf, "%v: %s", k, iStrs.Truncate(vars.Get(k).String(), maxExprVarWidth))
}
buf.WriteString("}")
row.add(buf.String())
}
if opts.LocalVariables {
if locals := event.Locals; locals != nil {
keys := sortedKeys(locals)
buf := new(bytes.Buffer)
buf.WriteString("{")
for i, k := range keys {
if i > 0 {
buf.WriteString(", ")
}
_, _ = fmt.Fprintf(buf, "%v: %s", k, iStrs.Truncate(locals.Get(k).String(), maxExprVarWidth))
}
buf.WriteString("}")
row.add(buf.String())
} else {
row.add("{}")
}
}
table.add(row)
}
table.write(w, columnPadding)
}
func sortedKeys(vm *ast.ValueMap) []ast.Value {
keys := make([]ast.Value, 0, vm.Len())
vm.Iter(func(k, _ ast.Value) bool {
keys = append(keys, k)
return false
})
slices.SortFunc(keys, func(a, b ast.Value) int {
return strings.Compare(a.String(), b.String())
})
return keys
}
func exprLocalVars(e *Event) *ast.ValueMap {
vars := ast.NewValueMap()
findVars := func(term *ast.Term) bool {
if name, ok := term.Value.(ast.Var); ok {
if meta, ok := e.LocalMetadata[name]; ok {
if val := e.Locals.Get(name); val != nil {
vars.Put(meta.Name, val)
}
}
}
return false
}
if r, ok := e.Node.(*ast.Rule); ok {
// We're only interested in vars in the head, not the body
ast.WalkTerms(r.Head, findVars)
return vars
}
// The local cache snapshot only contains a snapshot for those refs present in the event node,
// so they can all be added to the vars map.
e.localVirtualCacheSnapshot.Iter(func(k, v ast.Value) bool {
vars.Put(k, v)
return false
})
ast.WalkTerms(e.Node, findVars)
return vars
}
func formatEvent(event *Event, depth int) string {
padding := formatEventPadding(event, depth)
if event.Op == NoteOp {
return fmt.Sprintf("%v%v %q", padding, event.Op, event.Message)
}
var details any
if node, ok := event.Node.(*ast.Rule); ok {
details = ast.RulePath(node)
} else if event.Ref != nil {
details = event.Ref
} else {
details = rewrite(event).Node
}
template := "%v%v %v"
opts := []any{padding, event.Op, details}
if event.Message != "" {
template += " %v"
opts = append(opts, event.Message)
}
return fmt.Sprintf(template, opts...)
}
func formatEventPadding(event *Event, depth int) string {
spaces := formatEventSpaces(event, depth)
if spaces > 1 {
return strings.Repeat("| ", spaces-1)
}
return ""
}
func formatEventSpaces(event *Event, depth int) int {
switch event.Op {
case EnterOp:
return depth
case RedoOp:
if _, ok := event.Node.(*ast.Expr); !ok {
return depth
}
}
return depth + 1
}
// getShortenedFileNames will return a map of file paths to shortened aliases
// that were found in the trace. It also returns the longest location expected
func getShortenedFileNames(trace []*Event) (map[string]string, int) {
// Get a deduplicated list of all file paths
// and the longest file path size
fpAliases := map[string]string{}
var canShorten []string
longestLocation := 0
for _, event := range trace {
if event.Location != nil {
if event.Location.File != "" {
// length of "<name>:<row>"
curLen := len(event.Location.File) + numDigits10(event.Location.Row) + 1
if curLen > longestLocation {
longestLocation = curLen
}
if _, ok := fpAliases[event.Location.File]; ok {
continue
}
canShorten = append(canShorten, event.Location.File)
// Default to just alias their full path
fpAliases[event.Location.File] = event.Location.File
} else {
// length of "<min width>:<row>"
curLen := minLocationWidth + numDigits10(event.Location.Row) + 1
if curLen > longestLocation {
longestLocation = curLen
}
}
}
}
if len(canShorten) > 0 && longestLocation > maxIdealLocationWidth {
fpAliases, longestLocation = iStrs.TruncateFilePaths(maxIdealLocationWidth, longestLocation, canShorten...)
}
return fpAliases, longestLocation
}
func numDigits10(n int) int {
if n < 10 {
return 1
}
return numDigits10(n/10) + 1
}
func formatLocation(event *Event, fileAliases map[string]string) string {
location := event.Location
if location == nil {
return ""
}
if location.File == "" {
return fmt.Sprintf("query:%v", location.Row)
}
return fmt.Sprintf("%v:%v", fileAliases[location.File], location.Row)
}
// depths is a helper for computing the depth of an event. Events within the
// same query all have the same depth. The depth of query is
// depth(parent(query))+1.
type depths map[uint64]int
func (ds depths) GetOrSet(qid uint64, pqid uint64) int {
depth := ds[qid]
if depth == 0 {
depth = ds[pqid]
depth++
ds[qid] = depth
}
return depth
}
func builtinTrace(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return handleBuiltinErr(ast.Trace.Name, bctx.Location, err)
}
if !bctx.TraceEnabled {
return iter(ast.InternedTerm(true))
}
evt := Event{
Op: NoteOp,
Location: bctx.Location,
QueryID: bctx.QueryID,
ParentID: bctx.ParentID,
Message: string(str),
}
for i := range bctx.QueryTracers {
bctx.QueryTracers[i].TraceEvent(evt)
}
return iter(ast.InternedTerm(true))
}
func rewrite(event *Event) *Event {
cpy := *event
var node ast.Node
switch v := event.Node.(type) {
case *ast.Expr:
expr := v.Copy()
// Hide generated local vars in 'key' position that have not been
// rewritten.
if ev, ok := v.Terms.(*ast.Every); ok {
if kv, ok := ev.Key.Value.(ast.Var); ok {
if rw, ok := cpy.LocalMetadata[kv]; !ok || rw.Name.IsGenerated() {
expr.Terms.(*ast.Every).Key = nil
}
}
}
node = expr
case ast.Body:
node = v.Copy()
case *ast.Rule:
node = v.Copy()
}
_, _ = ast.TransformVars(node, func(v ast.Var) (ast.Value, error) {
if meta, ok := cpy.LocalMetadata[v]; ok {
return meta.Name, nil
}
return v, nil
})
cpy.Node = node
return &cpy
}
type varInfo struct {
VarMetadata
val ast.Value
exprLoc *ast.Location
col int // 0-indexed column
}
func (v varInfo) Value() string {
if v.val != nil {
return v.val.String()
}
return "undefined"
}
func (v varInfo) Title() string {
if v.exprLoc != nil && v.exprLoc.Text != nil {
return string(v.exprLoc.Text)
}
return string(v.Name)
}
func padLocationText(loc *ast.Location) string {
if loc == nil {
return ""
}
text := string(loc.Text)
if loc.Col == 0 {
return text
}
buf := new(bytes.Buffer)
j := 0
for i := 1; i < loc.Col; i++ {
if len(loc.Tabs) > 0 && j < len(loc.Tabs) && loc.Tabs[j] == i {
buf.WriteString("\t")
j++
} else {
buf.WriteString(" ")
}
}
buf.WriteString(text)
return buf.String()
}
type PrettyEventOpts struct {
PrettyVars bool
}
func walkTestTerms(x any, f func(*ast.Term) bool) {
var vis *ast.GenericVisitor
vis = ast.NewGenericVisitor(func(x any) bool {
switch x := x.(type) {
case ast.Call:
for _, t := range x[1:] {
vis.Walk(t)
}
return true
case *ast.Expr:
if x.IsCall() {
for _, o := range x.Operands() {
vis.Walk(o)
}
for i := range x.With {
vis.Walk(x.With[i])
}
return true
}
case *ast.Term:
return f(x)
case *ast.With:
vis.Walk(x.Value)
return true
}
return false
})
vis.Walk(x)
}
func PrettyEvent(w io.Writer, e *Event, opts PrettyEventOpts) error {
if !opts.PrettyVars {
_, _ = fmt.Fprintln(w, padLocationText(e.Location))
return nil
}
buf := new(bytes.Buffer)
exprVars := map[string]varInfo{}
findVars := func(unknownAreUndefined bool) func(term *ast.Term) bool {
return func(term *ast.Term) bool {
if term.Location == nil {
return false
}
switch v := term.Value.(type) {
case *ast.ArrayComprehension, *ast.SetComprehension, *ast.ObjectComprehension:
// we don't report on the internals of a comprehension, as it's already evaluated, and we won't have the local vars.
return true
case ast.Var:
var info *varInfo
if meta, ok := e.LocalMetadata[v]; ok {
info = &varInfo{
VarMetadata: meta,
val: e.Locals.Get(v),
exprLoc: term.Location,
}
} else if unknownAreUndefined {
info = &varInfo{
VarMetadata: VarMetadata{Name: v},
exprLoc: term.Location,
col: term.Location.Col,
}
}
if info != nil {
if v, exists := exprVars[info.Title()]; !exists || v.val == nil {
if term.Location != nil {
info.col = term.Location.Col
}
exprVars[info.Title()] = *info
}
}
}
return false
}
}
expr, ok := e.Node.(*ast.Expr)
if !ok || expr == nil {
return nil
}
base := expr.BaseCogeneratedExpr()
exprText := padLocationText(base.Location)
buf.WriteString(exprText)
e.localVirtualCacheSnapshot.Iter(func(k, v ast.Value) bool {
var info *varInfo
switch k := k.(type) {
case ast.Ref:
info = &varInfo{
VarMetadata: VarMetadata{Name: ast.Var(k.String())},
val: v,
exprLoc: k[0].Location,
col: k[0].Location.Col,
}
case *ast.ArrayComprehension:
info = &varInfo{
VarMetadata: VarMetadata{Name: ast.Var(k.String())},
val: v,
exprLoc: k.Term.Location,
col: k.Term.Location.Col,
}
case *ast.SetComprehension:
info = &varInfo{
VarMetadata: VarMetadata{Name: ast.Var(k.String())},
val: v,
exprLoc: k.Term.Location,
col: k.Term.Location.Col,
}
case *ast.ObjectComprehension:
info = &varInfo{
VarMetadata: VarMetadata{Name: ast.Var(k.String())},
val: v,
exprLoc: k.Key.Location,
col: k.Key.Location.Col,
}
}
if info != nil {
exprVars[info.Title()] = *info
}
return false
})
// If the expression is negated, we can't confidently assert that vars with unknown values are 'undefined',
// since the compiler might have opted out of the necessary rewrite.
walkTestTerms(expr, findVars(!expr.Negated))
coExprs := expr.CogeneratedExprs()
for _, coExpr := range coExprs {
// Only the current "co-expr" can have undefined vars, if we don't know the value for a var in any other co-expr,
// it's unknown, not undefined. A var can be unknown if it hasn't been assigned a value yet, because the co-expr
// hasn't been evaluated yet (the fail happened before it).
walkTestTerms(coExpr, findVars(false))
}
printPrettyVars(buf, exprVars)
_, _ = fmt.Fprint(w, buf.String())
return nil
}
func printPrettyVars(w *bytes.Buffer, exprVars map[string]varInfo) {
containsTabs := false
varRows := make(map[int]any)
for _, info := range exprVars {
if len(info.exprLoc.Tabs) > 0 {
containsTabs = true
}
varRows[info.exprLoc.Row] = nil
}
if containsTabs && len(varRows) > 1 {
// We can't (currently) reliably point to var locations when they are on different rows that contain tabs.
// So we'll just print them in alphabetical order instead.
byName := make([]varInfo, 0, len(exprVars))
for _, info := range exprVars {
byName = append(byName, info)
}
slices.SortStableFunc(byName, func(a, b varInfo) int {
return strings.Compare(a.Title(), b.Title())
})
w.WriteString("\n\nWhere:\n")
for _, info := range byName {
fmt.Fprintf(w, "\n%s: %s", info.Title(), iStrs.Truncate(info.Value(), maxPrettyExprVarWidth))
}
return
}
byCol := make([]varInfo, 0, len(exprVars))
for _, info := range exprVars {
byCol = append(byCol, info)
}
slices.SortFunc(byCol, func(a, b varInfo) int {
// sort first by column, then by reverse row (to present vars in the same order they appear in the expr)
if a.col == b.col {
if a.exprLoc.Row == b.exprLoc.Row {
return strings.Compare(a.Title(), b.Title())
}
return b.exprLoc.Row - a.exprLoc.Row
}
return a.col - b.col
})
if len(byCol) == 0 {
return
}
w.WriteString("\n")
printArrows(w, byCol, -1)
for i := len(byCol) - 1; i >= 0; i-- {
w.WriteString("\n")
printArrows(w, byCol, i)
}
}
func printArrows(w *bytes.Buffer, l []varInfo, printValueAt int) {
prevCol := 0
var slice []varInfo
if printValueAt >= 0 {
slice = l[:printValueAt+1]
} else {
slice = l
}
isFirst := true
for i, info := range slice {
isLast := i >= len(slice)-1
col := info.col
if !isLast && col == l[i+1].col {
// We're sharing the same column with another, subsequent var
continue
}
spaces := col - 1
if i > 0 && !isFirst {
spaces = (col - prevCol) - 1
}
for j := range spaces {
tab := false
if slices.Contains(info.exprLoc.Tabs, j+prevCol+1) {
w.WriteString("\t")
tab = true
}
if !tab {
w.WriteString(" ")
}
}
if isLast && printValueAt >= 0 {
valueStr := iStrs.Truncate(info.Value(), maxPrettyExprVarWidth)
if (i > 0 && col == l[i-1].col) || (i < len(l)-1 && col == l[i+1].col) {
// There is another var on this column, so we need to include the name to differentiate them.
fmt.Fprintf(w, "%s: %s", info.Title(), valueStr)
} else {
w.WriteString(valueStr)
}
} else {
w.WriteString("|")
}
prevCol = col
isFirst = false
}
}
func init() {
RegisterBuiltinFunc(ast.Trace.Name, builtinTrace)
}
+82
View File
@@ -0,0 +1,82 @@
// Copyright 2022 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
)
func builtinIsNumber(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Number:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsString(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.String:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsBoolean(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Boolean:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsArray(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case *ast.Array:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsSet(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Set:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsObject(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Object:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func builtinIsNull(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Null:
return iter(ast.InternedTerm(true))
default:
return iter(ast.InternedTerm(false))
}
}
func init() {
RegisterBuiltinFunc(ast.IsNumber.Name, builtinIsNumber)
RegisterBuiltinFunc(ast.IsString.Name, builtinIsString)
RegisterBuiltinFunc(ast.IsBoolean.Name, builtinIsBoolean)
RegisterBuiltinFunc(ast.IsArray.Name, builtinIsArray)
RegisterBuiltinFunc(ast.IsSet.Name, builtinIsSet)
RegisterBuiltinFunc(ast.IsObject.Name, builtinIsObject)
RegisterBuiltinFunc(ast.IsNull.Name, builtinIsNull)
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2018 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"errors"
"github.com/open-policy-agent/opa/v1/ast"
)
func builtinTypeName(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
switch operands[0].Value.(type) {
case ast.Null:
return iter(ast.InternedTerm("null"))
case ast.Boolean:
return iter(ast.InternedTerm("boolean"))
case ast.Number:
return iter(ast.InternedTerm("number"))
case ast.String:
return iter(ast.InternedTerm("string"))
case *ast.Array:
return iter(ast.InternedTerm("array"))
case ast.Object:
return iter(ast.InternedTerm("object"))
case ast.Set:
return iter(ast.InternedTerm("set"))
}
return errors.New("illegal value")
}
func init() {
RegisterBuiltinFunc(ast.TypeNameBuiltin.Name, builtinTypeName)
}
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2020 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/internal/uuid"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/topdown/builtins"
)
type uuidCachingKey string
func builtinUUIDRFC4122(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
var key = uuidCachingKey(operands[0].Value.String())
val, ok := bctx.Cache.Get(key)
if ok {
return iter(val.(*ast.Term))
}
s, err := uuid.New(bctx.Seed)
if err != nil {
return err
}
result := ast.StringTerm(s)
bctx.Cache.Put(key, result)
return iter(result)
}
func builtinUUIDParse(_ BuiltinContext, operands []*ast.Term, iter func(term *ast.Term) error) error {
str, err := builtins.StringOperand(operands[0].Value, 1)
if err != nil {
return err
}
parsed, err := uuid.Parse(string(str))
if err != nil {
return nil
}
val, err := ast.InterfaceToValue(parsed)
if err != nil {
return err
}
return iter(ast.NewTerm(val))
}
func init() {
RegisterBuiltinFunc(ast.UUIDRFC4122.Name, builtinUUIDRFC4122)
RegisterBuiltinFunc(ast.UUIDParse.Name, builtinUUIDParse)
}
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package topdown
import (
"github.com/open-policy-agent/opa/v1/ast"
)
func evalWalk(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
input := operands[0]
if pathIsWildcard(operands) {
// When the path assignment is a wildcard: walk(input, [_, value])
// we may skip the path construction entirely, and simply return
// same pointer in each iteration. This is a *much* more efficient
// path when only the values are needed.
return walkNoPath(ast.ArrayTerm(ast.InternedEmptyArray, input), iter)
}
filter := getOutputPath(operands)
return walk(filter, nil, input, iter)
}
func walk(filter, path *ast.Array, input *ast.Term, iter func(*ast.Term) error) error {
if filter == nil || filter.Len() == 0 {
if path == nil {
if err := iter(ast.ArrayTerm(ast.NewTerm(ast.InternedEmptyArrayValue), input)); err != nil {
return err
}
} else {
// Shallow copy, as while the array is modified, the elements are not
pathCopy := copyShallow(path)
// TODO(ae): I'd *really* like these terms to be retrieved from a sync.Pool, and
// returned after iter is called. However, all my atttempts to do this have failed
// as there seems to be something holding on to these references after the call,
// leading to modifications that entirely alter the results. Perhaps this is not
// possible to do, but if it is,it would be a huge performance win.
if err := iter(ast.ArrayTerm(ast.NewTerm(pathCopy), input)); err != nil {
return err
}
}
}
if filter != nil && filter.Len() > 0 {
key := filter.Elem(0)
filter = filter.Slice(1, -1)
if key.IsGround() {
if term := input.Get(key); term != nil {
return walk(filter, pathAppend(path, key), term, iter)
}
return nil
}
}
switch v := input.Value.(type) {
case *ast.Array:
for i := range v.Len() {
if err := walk(filter, pathAppend(path, ast.InternedTerm(i)), v.Elem(i), iter); err != nil {
return err
}
}
case ast.Object:
for _, k := range v.Keys() {
if err := walk(filter, pathAppend(path, k), v.Get(k), iter); err != nil {
return err
}
}
case ast.Set:
for _, elem := range v.Slice() {
if err := walk(filter, pathAppend(path, elem), elem, iter); err != nil {
return err
}
}
}
return nil
}
func walkNoPath(input *ast.Term, iter func(*ast.Term) error) error {
// Note: the path array is embedded in the input from the start here
// in order to avoid an extra allocation per iteration. This leads to
// a little convoluted code below in order to extract and set the value,
// but since walk is commonly used to traverse large data structures,
// the performance gain is worth it.
if err := iter(input); err != nil {
return err
}
inputArray := input.Value.(*ast.Array)
value := inputArray.Get(ast.InternedTerm(1)).Value
switch v := value.(type) {
case ast.Object:
for _, k := range v.Keys() {
inputArray.Set(1, v.Get(k))
if err := walkNoPath(input, iter); err != nil {
return err
}
}
case *ast.Array:
for i := range v.Len() {
inputArray.Set(1, v.Elem(i))
if err := walkNoPath(input, iter); err != nil {
return err
}
}
case ast.Set:
for _, elem := range v.Slice() {
inputArray.Set(1, elem)
if err := walkNoPath(input, iter); err != nil {
return err
}
}
}
return nil
}
func pathAppend(path *ast.Array, key *ast.Term) *ast.Array {
if path == nil {
return ast.NewArray(key)
}
return path.Append(key)
}
func getOutputPath(operands []*ast.Term) *ast.Array {
if len(operands) == 2 {
if arr, ok := operands[1].Value.(*ast.Array); ok && arr.Len() == 2 {
if path, ok := arr.Elem(0).Value.(*ast.Array); ok {
return path
}
}
}
return nil
}
func pathIsWildcard(operands []*ast.Term) bool {
if len(operands) == 2 {
if arr, ok := operands[1].Value.(*ast.Array); ok && arr.Len() == 2 {
if v, ok := arr.Elem(0).Value.(ast.Var); ok {
return v.IsWildcard()
}
}
}
return false
}
func copyShallow(arr *ast.Array) *ast.Array {
cpy := make([]*ast.Term, 0, arr.Len())
arr.Foreach(func(elem *ast.Term) {
cpy = append(cpy, elem)
})
return ast.NewArray(cpy...)
}
func init() {
RegisterBuiltinFunc(ast.WalkBuiltin.Name, evalWalk)
}