Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+6
View File
@@ -0,0 +1,6 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package storage exposes the policy engine's storage layer.
package storage
+121
View File
@@ -0,0 +1,121 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package storage
import (
"fmt"
)
const (
// InternalErr indicates an unknown, internal error has occurred.
InternalErr = "storage_internal_error"
// NotFoundErr indicates the path used in the storage operation does not
// locate a document.
NotFoundErr = "storage_not_found_error"
// WriteConflictErr indicates a write on the path enocuntered a conflicting
// value inside the transaction.
WriteConflictErr = "storage_write_conflict_error"
// InvalidPatchErr indicates an invalid patch/write was issued. The patch
// was rejected.
InvalidPatchErr = "storage_invalid_patch_error"
// InvalidTransactionErr indicates an invalid operation was performed
// inside of the transaction.
InvalidTransactionErr = "storage_invalid_txn_error"
// TriggersNotSupportedErr indicates the caller attempted to register a
// trigger against a store that does not support them.
TriggersNotSupportedErr = "storage_triggers_not_supported_error"
// WritesNotSupportedErr indicate the caller attempted to perform a write
// against a store that does not support them.
WritesNotSupportedErr = "storage_writes_not_supported_error"
// PolicyNotSupportedErr indicate the caller attempted to perform a policy
// management operation against a store that does not support them.
PolicyNotSupportedErr = "storage_policy_not_supported_error"
)
// Error is the error type returned by the storage layer.
type Error struct {
Code string `json:"code"`
Message string `json:"message"`
}
func (err *Error) Error() string {
if err.Message != "" {
return fmt.Sprintf("%v: %v", err.Code, err.Message)
}
return err.Code
}
// IsNotFound returns true if this error is a NotFoundErr.
func IsNotFound(err error) bool {
if err, ok := err.(*Error); ok {
return err.Code == NotFoundErr
}
return false
}
// IsWriteConflictError returns true if this error a WriteConflictErr.
func IsWriteConflictError(err error) bool {
switch err := err.(type) {
case *Error:
return err.Code == WriteConflictErr
}
return false
}
// IsInvalidPatch returns true if this error is a InvalidPatchErr.
func IsInvalidPatch(err error) bool {
switch err := err.(type) {
case *Error:
return err.Code == InvalidPatchErr
}
return false
}
// IsInvalidTransaction returns true if this error is a InvalidTransactionErr.
func IsInvalidTransaction(err error) bool {
switch err := err.(type) {
case *Error:
return err.Code == InvalidTransactionErr
}
return false
}
// IsIndexingNotSupported is a stub for backwards-compatibility.
//
// Deprecated: We no longer return IndexingNotSupported errors, so it is
// unnecessary to check for them.
func IsIndexingNotSupported(error) bool { return false }
func writeConflictError(path Path) *Error {
return &Error{
Code: WriteConflictErr,
Message: path.String(),
}
}
func triggersNotSupportedError() *Error {
return &Error{
Code: TriggersNotSupportedErr,
}
}
func writesNotSupportedError() *Error {
return &Error{
Code: WritesNotSupportedErr,
}
}
func policyNotSupportedError() *Error {
return &Error{
Code: PolicyNotSupportedErr,
}
}
+304
View File
@@ -0,0 +1,304 @@
// Copyright 2024 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package inmem
import (
"fmt"
"strconv"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/storage"
"github.com/open-policy-agent/opa/v1/storage/internal/errors"
"github.com/open-policy-agent/opa/v1/storage/internal/ptr"
)
type updateAST struct {
path storage.Path // data path modified by update
remove bool // indicates whether update removes the value at path
value ast.Value // value to add/replace at path (ignored if remove is true)
}
func (u *updateAST) Path() storage.Path {
return u.path
}
func (u *updateAST) Remove() bool {
return u.remove
}
func (u *updateAST) Set(v any) {
if v, ok := v.(ast.Value); ok {
u.value = v
} else {
panic("illegal value type") // FIXME: do conversion?
}
}
func (u *updateAST) Value() any {
return u.value
}
func (u *updateAST) Relative(path storage.Path) dataUpdate {
cpy := *u
cpy.path = cpy.path[len(path):]
return &cpy
}
func (u *updateAST) Apply(v any) any {
if len(u.path) == 0 {
return u.value
}
data, ok := v.(ast.Value)
if !ok {
panic(fmt.Errorf("illegal value type %T, expected ast.Value", v))
}
if u.remove {
newV, err := removeInAst(data, u.path)
if err != nil {
panic(err)
}
return newV
}
// If we're not removing, we're replacing (adds are turned into replaces during updateAST creation).
newV, err := setInAst(data, u.path, u.value)
if err != nil {
panic(err)
}
return newV
}
func newUpdateAST(data any, op storage.PatchOp, path storage.Path, idx int, value ast.Value) (*updateAST, error) {
switch data.(type) {
case ast.Null, ast.Boolean, ast.Number, ast.String:
return nil, errors.NotFoundErr
}
switch data := data.(type) {
case ast.Object:
return newUpdateObjectAST(data, op, path, idx, value)
case *ast.Array:
return newUpdateArrayAST(data, op, path, idx, value)
}
return nil, &storage.Error{
Code: storage.InternalErr,
Message: "invalid data value encountered",
}
}
func newUpdateArrayAST(data *ast.Array, op storage.PatchOp, path storage.Path, idx int, value ast.Value) (*updateAST, error) {
if idx == len(path)-1 {
if path[idx] == "-" || path[idx] == strconv.Itoa(data.Len()) {
if op != storage.AddOp {
return nil, errors.NewInvalidPatchError("%v: invalid patch path", path)
}
cpy := data.Append(ast.NewTerm(value))
return &updateAST{path[:len(path)-1], false, cpy}, nil
}
pos, err := ptr.ValidateASTArrayIndex(data, path[idx], path)
if err != nil {
return nil, err
}
switch op {
case storage.AddOp:
var results []*ast.Term
for i := range data.Len() {
if i == pos {
results = append(results, ast.NewTerm(value))
}
results = append(results, data.Elem(i))
}
return &updateAST{path[:len(path)-1], false, ast.NewArray(results...)}, nil
case storage.RemoveOp:
var results []*ast.Term
for i := range data.Len() {
if i != pos {
results = append(results, data.Elem(i))
}
}
return &updateAST{path[:len(path)-1], false, ast.NewArray(results...)}, nil
default:
var results []*ast.Term
for i := range data.Len() {
if i == pos {
results = append(results, ast.NewTerm(value))
} else {
results = append(results, data.Elem(i))
}
}
return &updateAST{path[:len(path)-1], false, ast.NewArray(results...)}, nil
}
}
pos, err := ptr.ValidateASTArrayIndex(data, path[idx], path)
if err != nil {
return nil, err
}
return newUpdateAST(data.Elem(pos).Value, op, path, idx+1, value)
}
func newUpdateObjectAST(data ast.Object, op storage.PatchOp, path storage.Path, idx int, value ast.Value) (*updateAST, error) {
key := ast.InternedTerm(path[idx])
val := data.Get(key)
if idx == len(path)-1 {
switch op {
case storage.ReplaceOp, storage.RemoveOp:
if val == nil {
return nil, errors.NotFoundErr
}
}
return &updateAST{path, op == storage.RemoveOp, value}, nil
}
if val != nil {
return newUpdateAST(val.Value, op, path, idx+1, value)
}
return nil, errors.NotFoundErr
}
// setInAst updates the value in the AST at the given path with the given value.
// Values can only be replaced in arrays, not added.
// Values for new keys can be added to objects
func setInAst(data ast.Value, path storage.Path, value ast.Value) (ast.Value, error) {
if len(path) == 0 {
return data, nil
}
switch data := data.(type) {
case ast.Object:
return setInAstObject(data, path, value)
case *ast.Array:
return setInAstArray(data, path, value)
default:
return nil, fmt.Errorf("illegal value type %T, expected ast.Object or ast.Array", data)
}
}
func setInAstObject(obj ast.Object, path storage.Path, value ast.Value) (ast.Value, error) {
key := ast.InternedTerm(path[0])
if len(path) == 1 {
obj.Insert(key, ast.NewTerm(value))
return obj, nil
}
child := obj.Get(key)
newChild, err := setInAst(child.Value, path[1:], value)
if err != nil {
return nil, err
}
obj.Insert(key, ast.NewTerm(newChild))
return obj, nil
}
func setInAstArray(arr *ast.Array, path storage.Path, value ast.Value) (ast.Value, error) {
idx, err := strconv.Atoi(path[0])
if err != nil {
return nil, fmt.Errorf("illegal array index %v: %v", path[0], err)
}
if idx < 0 || idx >= arr.Len() {
return arr, nil
}
if len(path) == 1 {
arr.Set(idx, ast.NewTerm(value))
return arr, nil
}
child := arr.Elem(idx)
newChild, err := setInAst(child.Value, path[1:], value)
if err != nil {
return nil, err
}
arr.Set(idx, ast.NewTerm(newChild))
return arr, nil
}
func removeInAst(value ast.Value, path storage.Path) (ast.Value, error) {
if len(path) == 0 {
return value, nil
}
switch value := value.(type) {
case ast.Object:
return removeInAstObject(value, path)
case *ast.Array:
return removeInAstArray(value, path)
default:
return nil, fmt.Errorf("illegal value type %T, expected ast.Object or ast.Array", value)
}
}
func removeInAstObject(obj ast.Object, path storage.Path) (ast.Value, error) {
key := ast.InternedTerm(path[0])
if len(path) == 1 {
var items [][2]*ast.Term
// Note: possibly expensive operation for large data.
obj.Foreach(func(k *ast.Term, v *ast.Term) {
if k.Equal(key) {
return
}
items = append(items, [2]*ast.Term{k, v})
})
return ast.NewObject(items...), nil
}
if child := obj.Get(key); child != nil {
updatedChild, err := removeInAst(child.Value, path[1:])
if err != nil {
return nil, err
}
obj.Insert(key, ast.NewTerm(updatedChild))
}
return obj, nil
}
func removeInAstArray(arr *ast.Array, path storage.Path) (ast.Value, error) {
idx, err := strconv.Atoi(path[0])
if err != nil {
// We expect the path to be valid at this point.
return arr, nil
}
if idx < 0 || idx >= arr.Len() {
return arr, err
}
if len(path) == 1 {
var elems []*ast.Term
// Note: possibly expensive operation for large data.
for i := range arr.Len() {
if i == idx {
continue
}
elems = append(elems, arr.Elem(i))
}
return ast.NewArray(elems...), nil
}
updatedChild, err := removeInAst(arr.Elem(idx).Value, path[1:])
if err != nil {
return nil, err
}
arr.Set(idx, ast.NewTerm(updatedChild))
return arr, nil
}
@@ -0,0 +1,463 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package inmem implements an in-memory version of the policy engine's storage
// layer.
//
// The in-memory store is used as the default storage layer implementation. The
// in-memory store supports multi-reader/single-writer concurrency with
// rollback.
//
// Callers should assume the in-memory store does not make copies of written
// data. Once data is written to the in-memory store, it should not be modified
// (outside of calling Store.Write). Furthermore, data read from the in-memory
// store should be treated as read-only.
package inmem
import (
"context"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"github.com/open-policy-agent/opa/internal/merge"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/storage"
"github.com/open-policy-agent/opa/v1/storage/internal/errors"
"github.com/open-policy-agent/opa/v1/util"
)
// New returns an empty in-memory store.
func New() storage.Store {
return NewWithOpts()
}
// NewWithOpts returns an empty in-memory store, with extra options passed.
func NewWithOpts(opts ...Opt) storage.Store {
s := &store{
triggers: map[*handle]storage.TriggerConfig{},
policies: map[string][]byte{},
roundTripOnWrite: true,
returnASTValuesOnRead: false,
}
for _, opt := range opts {
opt(s)
}
if s.returnASTValuesOnRead {
s.data = ast.NewObject()
s.roundTripOnWrite = false
} else {
s.data = map[string]any{}
}
return s
}
// NewFromObject returns a new in-memory store from the supplied data object.
func NewFromObject(data map[string]any) storage.Store {
return NewFromObjectWithOpts(data)
}
// NewFromObjectWithOpts returns a new in-memory store from the supplied data object, with the
// options passed.
func NewFromObjectWithOpts(data map[string]any, opts ...Opt) storage.Store {
db := NewWithOpts(opts...)
ctx := context.Background()
txn, err := db.NewTransaction(ctx, storage.WriteParams)
if err != nil {
panic(err)
}
if err := db.Write(ctx, txn, storage.AddOp, storage.RootPath, data); err != nil {
panic(err)
}
if err := db.Commit(ctx, txn); err != nil {
panic(err)
}
return db
}
// NewFromReader returns a new in-memory store from a reader that produces a
// JSON serialized object. This function is for test purposes.
func NewFromReader(r io.Reader) storage.Store {
return NewFromReaderWithOpts(r)
}
// NewFromReader returns a new in-memory store from a reader that produces a
// JSON serialized object, with extra options. This function is for test purposes.
func NewFromReaderWithOpts(r io.Reader, opts ...Opt) storage.Store {
var data map[string]any
if err := util.NewJSONDecoder(r).Decode(&data); err != nil {
panic(err)
}
return NewFromObjectWithOpts(data, opts...)
}
type store struct {
rmu sync.RWMutex // reader-writer lock
wmu sync.Mutex // writer lock
xid uint64 // last generated transaction id
data any // raw or AST data
policies map[string][]byte // raw policies
triggers map[*handle]storage.TriggerConfig // registered triggers
// roundTripOnWrite, if true, means that every call to Write round trips the
// data through JSON before adding the data to the store. Defaults to true.
roundTripOnWrite bool
// returnASTValuesOnRead, if true, means that the store will eagerly convert data to AST values,
// and return them on Read.
// FIXME: naming(?)
returnASTValuesOnRead bool
}
type handle struct {
db *store
}
func (db *store) NewTransaction(_ context.Context, params ...storage.TransactionParams) (storage.Transaction, error) {
txn := &transaction{
xid: atomic.AddUint64(&db.xid, uint64(1)),
db: db,
}
if len(params) > 0 {
txn.write = params[0].Write
txn.context = params[0].Context
}
if txn.write {
db.wmu.Lock()
} else {
db.rmu.RLock()
}
return txn, nil
}
// Truncate implements the storage.Store interface. This method must be called within a transaction.
func (db *store) Truncate(ctx context.Context, txn storage.Transaction, params storage.TransactionParams, it storage.Iterator) error {
var update *storage.Update
var err error
underlying, err := db.underlying(txn)
if err != nil {
return err
}
mergedData := map[string]any{}
for {
if update, err = it.Next(); err != nil {
break
}
if update.IsPolicy {
err = underlying.UpsertPolicy(strings.TrimLeft(update.Path.String(), "/"), update.Value)
if err != nil {
return err
}
} else {
var value any
if err = util.Unmarshal(update.Value, &value); err != nil {
return err
}
var key []string
dirpath := strings.TrimLeft(update.Path.String(), "/")
if len(dirpath) > 0 {
key = strings.Split(dirpath, "/")
}
if value != nil {
obj, err := mktree(key, value)
if err != nil {
return err
}
merged, ok := merge.InterfaceMaps(mergedData, obj)
if !ok {
return fmt.Errorf("failed to insert data file from path %s", filepath.Join(key...))
}
mergedData = merged
}
}
}
// err is known not to be nil at this point, as it getting assigned
// a non-nil value is the only way the loop above can exit.
if err != io.EOF {
return err
}
// For backwards compatibility, check if `RootOverwrite` was configured.
if params.RootOverwrite {
return underlying.Write(storage.AddOp, storage.RootPath, mergedData)
}
for _, root := range params.BasePaths {
newPath, ok := storage.ParsePathEscaped("/" + root)
if !ok {
return fmt.Errorf("storage path invalid: %v", newPath)
}
if value, ok := lookup(newPath, mergedData); ok {
if len(newPath) > 0 {
if err := storage.MakeDir(ctx, db, txn, newPath[:len(newPath)-1]); err != nil {
return err
}
}
if err := underlying.Write(storage.AddOp, newPath, value); err != nil {
return err
}
}
}
return nil
}
func (db *store) Commit(ctx context.Context, txn storage.Transaction) error {
underlying, err := db.underlying(txn)
if err != nil {
return err
}
if underlying.write {
db.rmu.Lock()
event := underlying.Commit()
db.runOnCommitTriggers(ctx, txn, event)
// Mark the transaction stale after executing triggers, so they can
// perform store operations if needed.
underlying.stale = true
db.rmu.Unlock()
db.wmu.Unlock()
} else {
db.rmu.RUnlock()
}
return nil
}
func (db *store) Abort(_ context.Context, txn storage.Transaction) {
underlying, err := db.underlying(txn)
if err != nil {
panic(err)
}
underlying.stale = true
if underlying.write {
db.wmu.Unlock()
} else {
db.rmu.RUnlock()
}
}
func (db *store) ListPolicies(_ context.Context, txn storage.Transaction) ([]string, error) {
underlying, err := db.underlying(txn)
if err != nil {
return nil, err
}
return underlying.ListPolicies(), nil
}
func (db *store) GetPolicy(_ context.Context, txn storage.Transaction, id string) ([]byte, error) {
underlying, err := db.underlying(txn)
if err != nil {
return nil, err
}
return underlying.GetPolicy(id)
}
func (db *store) UpsertPolicy(_ context.Context, txn storage.Transaction, id string, bs []byte) error {
underlying, err := db.underlying(txn)
if err != nil {
return err
}
return underlying.UpsertPolicy(id, bs)
}
func (db *store) DeletePolicy(_ context.Context, txn storage.Transaction, id string) error {
underlying, err := db.underlying(txn)
if err != nil {
return err
}
if _, err := underlying.GetPolicy(id); err != nil {
return err
}
return underlying.DeletePolicy(id)
}
func (db *store) Register(_ context.Context, txn storage.Transaction, config storage.TriggerConfig) (storage.TriggerHandle, error) {
underlying, err := db.underlying(txn)
if err != nil {
return nil, err
}
if !underlying.write {
return nil, &storage.Error{
Code: storage.InvalidTransactionErr,
Message: "triggers must be registered with a write transaction",
}
}
h := &handle{db}
db.triggers[h] = config
return h, nil
}
func (db *store) Read(_ context.Context, txn storage.Transaction, path storage.Path) (any, error) {
underlying, err := db.underlying(txn)
if err != nil {
return nil, err
}
return underlying.Read(path)
}
func (db *store) Write(_ context.Context, txn storage.Transaction, op storage.PatchOp, path storage.Path, value any) error {
underlying, err := db.underlying(txn)
if err != nil {
return err
}
if db.returnASTValuesOnRead || !util.NeedsRoundTrip(value) {
// Fast path when value is nil, bool, string or json.Number.
return underlying.Write(op, path, value)
}
val := util.Reference(value)
if db.roundTripOnWrite {
if err := util.RoundTrip(val); err != nil {
return err
}
}
return underlying.Write(op, path, *val)
}
func (h *handle) Unregister(_ context.Context, txn storage.Transaction) {
underlying, err := h.db.underlying(txn)
if err != nil {
panic(err)
}
if !underlying.write {
panic(&storage.Error{
Code: storage.InvalidTransactionErr,
Message: "triggers must be unregistered with a write transaction",
})
}
delete(h.db.triggers, h)
}
func (db *store) runOnCommitTriggers(ctx context.Context, txn storage.Transaction, event storage.TriggerEvent) {
// While it's unlikely, the API allows one trigger to be configured to want
// data conversion, and another that doesn't. So let's handle that properly.
var wantsDataConversion bool
if db.returnASTValuesOnRead && len(event.Data) > 0 {
for _, t := range db.triggers {
if !t.SkipDataConversion {
wantsDataConversion = true
break
}
}
}
var converted storage.TriggerEvent
if wantsDataConversion {
converted = storage.TriggerEvent{
Policy: event.Policy,
Data: make([]storage.DataEvent, 0, len(event.Data)),
Context: event.Context,
}
for _, dataEvent := range event.Data {
if astData, ok := dataEvent.Data.(ast.Value); ok {
jsn, err := ast.ValueToInterface(astData, illegalResolver{})
if err != nil {
panic(err)
}
converted.Data = append(converted.Data, storage.DataEvent{
Path: dataEvent.Path,
Data: jsn,
Removed: dataEvent.Removed,
})
}
}
}
for _, t := range db.triggers {
if wantsDataConversion && !t.SkipDataConversion {
t.OnCommit(ctx, txn, converted)
} else {
t.OnCommit(ctx, txn, event)
}
}
}
type illegalResolver struct{}
func (illegalResolver) Resolve(ref ast.Ref) (any, error) {
return nil, fmt.Errorf("illegal value: %v", ref)
}
func (db *store) underlying(txn storage.Transaction) (*transaction, error) {
underlying, ok := txn.(*transaction)
if !ok {
return nil, &storage.Error{
Code: storage.InvalidTransactionErr,
Message: fmt.Sprintf("unexpected transaction type %T", txn),
}
}
if underlying.db != db {
return nil, &storage.Error{
Code: storage.InvalidTransactionErr,
Message: "unknown transaction",
}
}
if underlying.stale {
return nil, &storage.Error{
Code: storage.InvalidTransactionErr,
Message: "stale transaction",
}
}
return underlying, nil
}
func mktree(path []string, value any) (map[string]any, error) {
if len(path) == 0 {
// For 0 length path the value is the full tree.
obj, ok := value.(map[string]any)
if !ok {
return nil, errors.RootMustBeObjectErr
}
return obj, nil
}
dir := map[string]any{}
for i := len(path) - 1; i > 0; i-- {
dir[path[i]] = value
value = dir
dir = map[string]any{}
}
dir[path[0]] = value
return dir, nil
}
func lookup(path storage.Path, data map[string]any) (any, bool) {
if len(path) == 0 {
return data, true
}
for i := range len(path) - 1 {
value, ok := data[path[i]]
if !ok {
return nil, false
}
obj, ok := value.(map[string]any)
if !ok {
return nil, false
}
data = obj
}
value, ok := data[path[len(path)-1]]
return value, ok
}
@@ -0,0 +1,37 @@
package inmem
// An Opt modifies store at instantiation.
type Opt func(*store)
// OptRoundTripOnWrite sets whether incoming objects written to store are
// round-tripped through JSON to ensure they are serializable to JSON.
//
// Callers should disable this if they can guarantee all objects passed to
// Write() are serializable to JSON. Failing to do so may result in undefined
// behavior, including panics.
//
// Usually, when only storing objects in the inmem store that have been read
// via encoding/json, this is safe to disable, and comes with an improvement
// in performance and memory use.
//
// If setting to false, callers should deep-copy any objects passed to Write()
// unless they can guarantee the objects will not be mutated after being written,
// and that mutations happening to the objects after they have been passed into
// Write() don't affect their logic.
func OptRoundTripOnWrite(enabled bool) Opt {
return func(s *store) {
s.roundTripOnWrite = enabled
}
}
// OptReturnASTValuesOnRead sets whether data values added to the store should be
// eagerly converted to AST values, which are then returned on read.
//
// When enabled, this feature does not sanity check data before converting it to AST values,
// which may result in panics if the data is not valid. Callers should ensure that passed data
// can be serialized to AST values; otherwise, it's recommended to also enable OptRoundTripOnWrite.
func OptReturnASTValuesOnRead(enabled bool) Opt {
return func(s *store) {
s.returnASTValuesOnRead = enabled
}
}
+539
View File
@@ -0,0 +1,539 @@
// Copyright 2017 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package inmem
import (
"container/list"
"encoding/json"
"slices"
"strconv"
"github.com/open-policy-agent/opa/internal/deepcopy"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/storage"
"github.com/open-policy-agent/opa/v1/storage/internal/errors"
"github.com/open-policy-agent/opa/v1/storage/internal/ptr"
)
// transaction implements the low-level read/write operations on the in-memory
// store and contains the state required for pending transactions.
//
// For write transactions, the struct contains a logical set of updates
// performed by write operations in the transaction. Each write operation
// compacts the set such that two updates never overlap:
//
// - If new update path is a prefix of existing update path, existing update is
// removed, new update is added.
//
// - If existing update path is a prefix of new update path, existing update is
// modified.
//
// - Otherwise, new update is added.
//
// Read transactions do not require any special handling and simply passthrough
// to the underlying store. Read transactions do not support upgrade.
type transaction struct {
db *store
updates *list.List
context *storage.Context
policies map[string]policyUpdate
xid uint64
write bool
stale bool
}
type policyUpdate struct {
value []byte
remove bool
}
func (txn *transaction) ID() uint64 {
return txn.xid
}
func (txn *transaction) Write(op storage.PatchOp, path storage.Path, value any) error {
if !txn.write {
return &storage.Error{Code: storage.InvalidTransactionErr, Message: "data write during read transaction"}
}
if txn.updates == nil {
txn.updates = list.New()
}
if len(path) == 0 {
return txn.updateRoot(op, value)
}
for curr := txn.updates.Front(); curr != nil; {
update := curr.Value.(dataUpdate)
// Check if new update masks existing update exactly. In this case, the
// existing update can be removed and no other updates have to be
// visited (because no two updates overlap.)
if update.Path().Equal(path) {
if update.Remove() {
if op != storage.AddOp {
return errors.NotFoundErr
}
}
// If the last update has the same path and value, we have nothing to do.
if txn.db.returnASTValuesOnRead {
if astValue, ok := update.Value().(ast.Value); ok {
if equalsValue(value, astValue) {
return nil
}
}
} else if comparableEquals(update.Value(), value) {
return nil
}
txn.updates.Remove(curr)
break
}
// Check if new update masks existing update. In this case, the
// existing update has to be removed but other updates may overlap, so
// we must continue.
if update.Path().HasPrefix(path) {
remove := curr
curr = curr.Next()
txn.updates.Remove(remove)
continue
}
// Check if new update modifies existing update. In this case, the
// existing update is mutated.
if path.HasPrefix(update.Path()) {
if update.Remove() {
return errors.NotFoundErr
}
suffix := path[len(update.Path()):]
newUpdate, err := txn.db.newUpdate(update.Value(), op, suffix, 0, value)
if err != nil {
return err
}
update.Set(newUpdate.Apply(update.Value()))
return nil
}
curr = curr.Next()
}
update, err := txn.db.newUpdate(txn.db.data, op, path, 0, value)
if err != nil {
return err
}
txn.updates.PushFront(update)
return nil
}
func comparableEquals(a, b any) bool {
switch a := a.(type) {
case nil:
return b == nil
case bool:
if vb, ok := b.(bool); ok {
return vb == a
}
case string:
if vs, ok := b.(string); ok {
return vs == a
}
case json.Number:
if vn, ok := b.(json.Number); ok {
return vn == a
}
}
return false
}
func (txn *transaction) updateRoot(op storage.PatchOp, value any) error {
if op == storage.RemoveOp {
return errors.RootCannotBeRemovedErr
}
var update any
if txn.db.returnASTValuesOnRead {
valueAST, err := ast.InterfaceToValue(value)
if err != nil {
return err
}
if _, ok := valueAST.(ast.Object); !ok {
return errors.RootMustBeObjectErr
}
update = &updateAST{
path: storage.RootPath,
remove: false,
value: valueAST,
}
} else {
if _, ok := value.(map[string]any); !ok {
return errors.RootMustBeObjectErr
}
update = &updateRaw{
path: storage.RootPath,
remove: false,
value: value,
}
}
txn.updates.Init()
txn.updates.PushFront(update)
return nil
}
func (txn *transaction) Commit() (result storage.TriggerEvent) {
result.Context = txn.context
if txn.updates != nil {
if len(txn.db.triggers) > 0 {
result.Data = slices.Grow(result.Data, txn.updates.Len())
}
for curr := txn.updates.Front(); curr != nil; curr = curr.Next() {
action := curr.Value.(dataUpdate)
txn.db.data = action.Apply(txn.db.data)
if len(txn.db.triggers) > 0 {
result.Data = append(result.Data, storage.DataEvent{
Path: action.Path(),
Data: action.Value(),
Removed: action.Remove(),
})
}
}
}
if len(txn.policies) > 0 && len(txn.db.triggers) > 0 {
result.Policy = slices.Grow(result.Policy, len(txn.policies))
}
for id, upd := range txn.policies {
if upd.remove {
delete(txn.db.policies, id)
} else {
txn.db.policies[id] = upd.value
}
if len(txn.db.triggers) > 0 {
result.Policy = append(result.Policy, storage.PolicyEvent{
ID: id,
Data: upd.value,
Removed: upd.remove,
})
}
}
return result
}
func pointer(v any, path storage.Path) (any, error) {
if v, ok := v.(ast.Value); ok {
return ptr.ValuePtr(v, path)
}
return ptr.Ptr(v, path)
}
func deepcpy(v any) any {
if v, ok := v.(ast.Value); ok {
var cpy ast.Value
switch data := v.(type) {
case ast.Object:
cpy = data.Copy()
case *ast.Array:
cpy = data.Copy()
}
return cpy
}
return deepcopy.DeepCopy(v)
}
func (txn *transaction) Read(path storage.Path) (any, error) {
if !txn.write || txn.updates == nil {
return pointer(txn.db.data, path)
}
var merge []dataUpdate
for curr := txn.updates.Front(); curr != nil; curr = curr.Next() {
upd := curr.Value.(dataUpdate)
if path.HasPrefix(upd.Path()) {
if upd.Remove() {
return nil, errors.NotFoundErr
}
return pointer(upd.Value(), path[len(upd.Path()):])
}
if upd.Path().HasPrefix(path) {
merge = append(merge, upd)
}
}
data, err := pointer(txn.db.data, path)
if err != nil {
return nil, err
}
if len(merge) == 0 {
return data, nil
}
cpy := deepcpy(data)
for _, update := range merge {
cpy = update.Relative(path).Apply(cpy)
}
return cpy, nil
}
func (txn *transaction) ListPolicies() (ids []string) {
for id := range txn.db.policies {
if _, ok := txn.policies[id]; !ok {
ids = append(ids, id)
}
}
for id, update := range txn.policies {
if !update.remove {
ids = append(ids, id)
}
}
return ids
}
func (txn *transaction) GetPolicy(id string) ([]byte, error) {
if txn.policies != nil {
if update, ok := txn.policies[id]; ok {
if !update.remove {
return update.value, nil
}
return nil, errors.NewNotFoundErrorf("policy id %q", id)
}
}
if exist, ok := txn.db.policies[id]; ok {
return exist, nil
}
return nil, errors.NewNotFoundErrorf("policy id %q", id)
}
func (txn *transaction) UpsertPolicy(id string, bs []byte) error {
return txn.updatePolicy(id, policyUpdate{bs, false})
}
func (txn *transaction) DeletePolicy(id string) error {
return txn.updatePolicy(id, policyUpdate{nil, true})
}
func (txn *transaction) updatePolicy(id string, update policyUpdate) error {
if !txn.write {
return &storage.Error{Code: storage.InvalidTransactionErr, Message: "policy write during read transaction"}
}
if txn.policies == nil {
txn.policies = map[string]policyUpdate{id: update}
} else {
txn.policies[id] = update
}
return nil
}
type dataUpdate interface {
Path() storage.Path
Remove() bool
Apply(any) any
Relative(path storage.Path) dataUpdate
Set(any)
Value() any
}
// update contains state associated with an update to be applied to the
// in-memory data store.
type updateRaw struct {
path storage.Path // data path modified by update
remove bool // indicates whether update removes the value at path
value any // value to add/replace at path (ignored if remove is true)
}
func equalsValue(a any, v ast.Value) bool {
if a, ok := a.(ast.Value); ok {
return a.Compare(v) == 0
}
switch a := a.(type) {
case nil:
return v == ast.NullValue
case bool:
if vb, ok := v.(ast.Boolean); ok {
return bool(vb) == a
}
case string:
if vs, ok := v.(ast.String); ok {
return string(vs) == a
}
}
return false
}
func (db *store) newUpdate(data any, op storage.PatchOp, path storage.Path, idx int, value any) (dataUpdate, error) {
if db.returnASTValuesOnRead {
astData, err := ast.InterfaceToValue(data)
if err != nil {
return nil, err
}
astValue, err := ast.InterfaceToValue(value)
if err != nil {
return nil, err
}
return newUpdateAST(astData, op, path, idx, astValue)
}
return newUpdateRaw(data, op, path, idx, value)
}
func newUpdateRaw(data any, op storage.PatchOp, path storage.Path, idx int, value any) (dataUpdate, error) {
switch data.(type) {
case nil, bool, json.Number, string:
return nil, errors.NotFoundErr
}
switch data := data.(type) {
case map[string]any:
return newUpdateObject(data, op, path, idx, value)
case []any:
return newUpdateArray(data, op, path, idx, value)
}
return nil, &storage.Error{
Code: storage.InternalErr,
Message: "invalid data value encountered",
}
}
func newUpdateArray(data []any, op storage.PatchOp, path storage.Path, idx int, value any) (dataUpdate, error) {
if idx == len(path)-1 {
if path[idx] == "-" || path[idx] == strconv.Itoa(len(data)) {
if op != storage.AddOp {
return nil, errors.NewInvalidPatchError("%v: invalid patch path", path)
}
cpy := make([]any, len(data)+1)
copy(cpy, data)
cpy[len(data)] = value
return &updateRaw{path[:len(path)-1], false, cpy}, nil
}
pos, err := ptr.ValidateArrayIndex(data, path[idx], path)
if err != nil {
return nil, err
}
switch op {
case storage.AddOp:
cpy := make([]any, len(data)+1)
copy(cpy[:pos], data[:pos])
copy(cpy[pos+1:], data[pos:])
cpy[pos] = value
return &updateRaw{path[:len(path)-1], false, cpy}, nil
case storage.RemoveOp:
cpy := make([]any, len(data)-1)
copy(cpy[:pos], data[:pos])
copy(cpy[pos:], data[pos+1:])
return &updateRaw{path[:len(path)-1], false, cpy}, nil
default:
cpy := make([]any, len(data))
copy(cpy, data)
cpy[pos] = value
return &updateRaw{path[:len(path)-1], false, cpy}, nil
}
}
pos, err := ptr.ValidateArrayIndex(data, path[idx], path)
if err != nil {
return nil, err
}
return newUpdateRaw(data[pos], op, path, idx+1, value)
}
func newUpdateObject(data map[string]any, op storage.PatchOp, path storage.Path, idx int, value any) (dataUpdate, error) {
if idx == len(path)-1 {
switch op {
case storage.ReplaceOp, storage.RemoveOp:
if _, ok := data[path[idx]]; !ok {
return nil, errors.NotFoundErr
}
}
return &updateRaw{path, op == storage.RemoveOp, value}, nil
}
if data, ok := data[path[idx]]; ok {
return newUpdateRaw(data, op, path, idx+1, value)
}
return nil, errors.NotFoundErr
}
func (u *updateRaw) Remove() bool {
return u.remove
}
func (u *updateRaw) Path() storage.Path {
return u.path
}
func (u *updateRaw) Apply(data any) any {
if len(u.path) == 0 {
return u.value
}
parent, err := ptr.Ptr(data, u.path[:len(u.path)-1])
if err != nil {
panic(err)
}
key := u.path[len(u.path)-1]
if u.remove {
obj := parent.(map[string]any)
delete(obj, key)
return data
}
switch parent := parent.(type) {
case map[string]any:
if parent == nil {
parent = make(map[string]any, 1)
}
parent[key] = u.value
case []any:
idx, err := strconv.Atoi(key)
if err != nil {
panic(err)
}
parent[idx] = u.value
}
return data
}
func (u *updateRaw) Set(v any) {
u.value = v
}
func (u *updateRaw) Value() any {
return u.value
}
func (u *updateRaw) Relative(path storage.Path) dataUpdate {
cpy := *u
cpy.path = cpy.path[len(path):]
return &cpy
}
+264
View File
@@ -0,0 +1,264 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package storage
import (
"context"
"github.com/open-policy-agent/opa/v1/metrics"
)
// Transaction defines the interface that identifies a consistent snapshot over
// the policy engine's storage layer.
type Transaction interface {
ID() uint64
}
// Store defines the interface for the storage layer's backend.
type Store interface {
Trigger
Policy
// NewTransaction is called create a new transaction in the store.
NewTransaction(context.Context, ...TransactionParams) (Transaction, error)
// Read is called to fetch a document referred to by path.
Read(context.Context, Transaction, Path) (any, error)
// Write is called to modify a document referred to by path.
Write(context.Context, Transaction, PatchOp, Path, any) error
// Commit is called to finish the transaction. If Commit returns an error, the
// transaction must be automatically aborted by the Store implementation.
Commit(context.Context, Transaction) error
// Truncate is called to make a copy of the underlying store, write documents in the new store
// by creating multiple transactions in the new store as needed and finally swapping
// over to the new storage instance. This method must be called within a transaction on the original store.
Truncate(context.Context, Transaction, TransactionParams, Iterator) error
// Abort is called to cancel the transaction.
Abort(context.Context, Transaction)
}
// MakeDirer defines the interface a Store could realize to override the
// generic MakeDir functionality in storage.MakeDir
type MakeDirer interface {
MakeDir(context.Context, Transaction, Path) error
}
// NonEmptyer allows a store implemention to override NonEmpty())
type NonEmptyer interface {
NonEmpty(context.Context, Transaction) func([]string) (bool, error)
}
// Closer is an optional interface that storage implementations can implement
// to perform cleanup operations when the store is being shut down.
// If a Store implements this interface, Close will be called during
// graceful shutdown of the OPA runtime.
type Closer interface {
Close(context.Context) error
}
// TransactionParams describes a new transaction.
type TransactionParams struct {
// BasePaths indicates the top-level paths where write operations will be performed in this transaction.
BasePaths []string
// RootOverwrite is deprecated. Use BasePaths instead.
RootOverwrite bool
// Write indicates if this transaction will perform any write operations.
Write bool
// Context contains key/value pairs passed to triggers.
Context *Context
}
// Context is a simple container for key/value pairs.
type Context struct {
values map[any]any
}
// NewContext returns a new context object.
func NewContext() *Context {
return &Context{
values: map[any]any{},
}
}
// Get returns the key value in the context.
func (ctx *Context) Get(key any) any {
if ctx == nil {
return nil
}
return ctx.values[key]
}
// Put adds a key/value pair to the context.
func (ctx *Context) Put(key, value any) {
ctx.values[key] = value
}
var metricsKey = struct{}{}
// WithMetrics allows passing metrics via the Context.
// It puts the metrics object in the ctx, and returns the same
// ctx (not a copy) for convenience.
func (ctx *Context) WithMetrics(m metrics.Metrics) *Context {
ctx.values[metricsKey] = m
return ctx
}
// Metrics() allows using a Context's metrics. Returns nil if metrics
// were not attached to the Context.
func (ctx *Context) Metrics() metrics.Metrics {
if m, ok := ctx.values[metricsKey]; ok {
if met, ok := m.(metrics.Metrics); ok {
return met
}
}
return nil
}
// WriteParams specifies the TransactionParams for a write transaction.
var WriteParams = TransactionParams{
Write: true,
}
// PatchOp is the enumeration of supposed modifications.
type PatchOp int
// Patch supports add, remove, and replace operations.
const (
AddOp PatchOp = iota
RemoveOp = iota
ReplaceOp = iota
)
// WritesNotSupported provides a default implementation of the write
// interface which may be used if the backend does not support writes.
type WritesNotSupported struct{}
func (WritesNotSupported) Write(context.Context, Transaction, PatchOp, Path, any) error {
return writesNotSupportedError()
}
// Policy defines the interface for policy module storage.
type Policy interface {
ListPolicies(context.Context, Transaction) ([]string, error)
GetPolicy(context.Context, Transaction, string) ([]byte, error)
UpsertPolicy(context.Context, Transaction, string, []byte) error
DeletePolicy(context.Context, Transaction, string) error
}
// PolicyNotSupported provides a default implementation of the policy interface
// which may be used if the backend does not support policy storage.
type PolicyNotSupported struct{}
// ListPolicies always returns a PolicyNotSupportedErr.
func (PolicyNotSupported) ListPolicies(context.Context, Transaction) ([]string, error) {
return nil, policyNotSupportedError()
}
// GetPolicy always returns a PolicyNotSupportedErr.
func (PolicyNotSupported) GetPolicy(context.Context, Transaction, string) ([]byte, error) {
return nil, policyNotSupportedError()
}
// UpsertPolicy always returns a PolicyNotSupportedErr.
func (PolicyNotSupported) UpsertPolicy(context.Context, Transaction, string, []byte) error {
return policyNotSupportedError()
}
// DeletePolicy always returns a PolicyNotSupportedErr.
func (PolicyNotSupported) DeletePolicy(context.Context, Transaction, string) error {
return policyNotSupportedError()
}
// PolicyEvent describes a change to a policy.
type PolicyEvent struct {
ID string
Data []byte
Removed bool
}
// DataEvent describes a change to a base data document.
type DataEvent struct {
Path Path
Data any
Removed bool
}
// TriggerEvent describes the changes that caused the trigger to be invoked.
type TriggerEvent struct {
Policy []PolicyEvent
Data []DataEvent
Context *Context
}
// IsZero returns true if the TriggerEvent indicates no changes occurred. This
// function is primarily for test purposes.
func (e TriggerEvent) IsZero() bool {
return !e.PolicyChanged() && !e.DataChanged()
}
// PolicyChanged returns true if the trigger was caused by a policy change.
func (e TriggerEvent) PolicyChanged() bool {
return len(e.Policy) > 0
}
// DataChanged returns true if the trigger was caused by a data change.
func (e TriggerEvent) DataChanged() bool {
return len(e.Data) > 0
}
// TriggerConfig contains the trigger registration configuration.
type TriggerConfig struct {
// SkipDataConversion when set to true, avoids converting data passed to
// trigger functions from the store to Go types, and instead passes the
// original representation (e.g., ast.Value).
SkipDataConversion bool
// OnCommit is invoked when a transaction is successfully committed. The
// callback is invoked with a handle to the write transaction that
// successfully committed before other clients see the changes.
OnCommit func(context.Context, Transaction, TriggerEvent)
}
// Trigger defines the interface that stores implement to register for change
// notifications when the store is changed.
type Trigger interface {
Register(context.Context, Transaction, TriggerConfig) (TriggerHandle, error)
}
// TriggersNotSupported provides default implementations of the Trigger
// interface which may be used if the backend does not support triggers.
type TriggersNotSupported struct{}
// Register always returns an error indicating triggers are not supported.
func (TriggersNotSupported) Register(context.Context, Transaction, TriggerConfig) (TriggerHandle, error) {
return nil, triggersNotSupportedError()
}
// TriggerHandle defines the interface that can be used to unregister triggers that have
// been registered on a Store.
type TriggerHandle interface {
Unregister(context.Context, Transaction)
}
// Iterator defines the interface that can be used to read files from a directory starting with
// files at the base of the directory, then sub-directories etc.
type Iterator interface {
Next() (*Update, error)
}
// Update contains information about a file
type Update struct {
Path Path
Value []byte
IsPolicy bool
}
@@ -0,0 +1,54 @@
// Copyright 2021 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package errors contains reusable error-related code for the storage layer.
package errors
import (
"fmt"
"github.com/open-policy-agent/opa/v1/storage"
)
const (
ArrayIndexTypeMsg = "array index must be integer"
DoesNotExistMsg = "document does not exist"
OutOfRangeMsg = "array index out of range"
RootMustBeObjectMsg = "root must be object"
RootCannotBeRemovedMsg = "root cannot be removed"
)
var (
NotFoundErr = &storage.Error{Code: storage.NotFoundErr, Message: DoesNotExistMsg}
RootMustBeObjectErr = &storage.Error{Code: storage.InvalidPatchErr, Message: RootMustBeObjectMsg}
RootCannotBeRemovedErr = &storage.Error{Code: storage.InvalidPatchErr, Message: RootCannotBeRemovedMsg}
)
func NewNotFoundErrorWithHint(path storage.Path, hint string) *storage.Error {
return &storage.Error{
Code: storage.NotFoundErr,
Message: path.String() + ": " + hint,
}
}
func NewNotFoundErrorf(f string, a ...any) *storage.Error {
return &storage.Error{
Code: storage.NotFoundErr,
Message: fmt.Sprintf(f, a...),
}
}
func NewWriteConflictError(p storage.Path) *storage.Error {
return &storage.Error{
Code: storage.WriteConflictErr,
Message: p.String(),
}
}
func NewInvalidPatchError(f string, a ...any) *storage.Error {
return &storage.Error{
Code: storage.InvalidPatchErr,
Message: fmt.Sprintf(f, a...),
}
}
@@ -0,0 +1,141 @@
// Copyright 2021 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
// Package ptr provides utilities for pointer operations using storage layer paths.
package ptr
import (
"strconv"
"github.com/open-policy-agent/opa/v1/ast"
"github.com/open-policy-agent/opa/v1/storage"
"github.com/open-policy-agent/opa/v1/storage/internal/errors"
)
func Ptr(data any, path storage.Path) (any, error) {
node := data
for i := range path {
key := path[i]
switch curr := node.(type) {
case map[string]any:
var ok bool
if node, ok = curr[key]; !ok {
return nil, errors.NotFoundErr
}
case []any:
pos, err := ValidateArrayIndex(curr, key, path)
if err != nil {
return nil, err
}
node = curr[pos]
default:
return nil, errors.NotFoundErr
}
}
return node, nil
}
func ValuePtr(data ast.Value, path storage.Path) (ast.Value, error) {
var keyTerm *ast.Term
defer func() {
if keyTerm != nil {
ast.TermPtrPool.Put(keyTerm)
}
}()
node := data
for i := range path {
key := path[i]
switch curr := node.(type) {
case ast.Object:
// Note(anders):
// This term is only created for the lookup, which is not great — especially
// considering the path likely was converted from a ref, where we had all
// the terms available already! Without chaging the storage API, our options
// for performant lookups are limitied to using interning or a pool. Prefer
// interning when possible, as that is zero alloc. Using the pool avoids at
// least allocating a new term for every lookup, but still requires an alloc
// for the string Value.
if ast.HasInternedValue(key) {
if val := curr.Get(ast.InternedTerm(key)); val != nil {
node = val.Value
} else {
return nil, errors.NotFoundErr
}
} else {
if keyTerm == nil {
keyTerm = ast.TermPtrPool.Get()
}
// 1 alloc
keyTerm.Value = ast.String(key)
if val := curr.Get(keyTerm); val != nil {
node = val.Value
} else {
return nil, errors.NotFoundErr
}
}
case *ast.Array:
pos, err := ValidateASTArrayIndex(curr, key, path)
if err != nil {
return nil, err
}
node = curr.Elem(pos).Value
default:
return nil, errors.NotFoundErr
}
}
return node, nil
}
func ValidateArrayIndex(arr []any, s string, path storage.Path) (int, error) {
idx, ok := isInt(s)
if !ok {
return 0, errors.NewNotFoundErrorWithHint(path, errors.ArrayIndexTypeMsg)
}
return inRange(idx, arr, path)
}
func ValidateASTArrayIndex(arr *ast.Array, s string, path storage.Path) (int, error) {
idx, ok := isInt(s)
if !ok {
return 0, errors.NewNotFoundErrorWithHint(path, errors.ArrayIndexTypeMsg)
}
return inRange(idx, arr, path)
}
// ValidateArrayIndexForWrite also checks that `s` is a valid way to address an
// array element like `ValidateArrayIndex`, but returns a `resource_conflict` error
// if it is not.
func ValidateArrayIndexForWrite(arr []any, s string, i int, path storage.Path) (int, error) {
idx, ok := isInt(s)
if !ok {
return 0, errors.NewWriteConflictError(path[:i-1])
}
return inRange(idx, arr, path)
}
func isInt(s string) (int, bool) {
idx, err := strconv.Atoi(s)
return idx, err == nil
}
func inRange(i int, arr any, path storage.Path) (int, error) {
var arrLen int
switch v := arr.(type) {
case []any:
arrLen = len(v)
case *ast.Array:
arrLen = v.Len()
}
if i < 0 || i >= arrLen {
return 0, errors.NewNotFoundErrorWithHint(path, errors.OutOfRangeMsg)
}
return i, nil
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package storage
import (
"errors"
"fmt"
"net/url"
"slices"
"strconv"
"strings"
"github.com/open-policy-agent/opa/v1/ast"
)
// RootPath refers to the root document in storage.
var RootPath = Path{}
// Path refers to a document in storage.
type Path []string
// ParsePath returns a new path for the given str.
func ParsePath(str string) (path Path, ok bool) {
if len(str) == 0 || str[0] != '/' {
return nil, false
}
if len(str) == 1 {
return Path{}, true
}
return strings.Split(str[1:], "/"), true
}
// ParsePathEscaped returns a new path for the given escaped str.
func ParsePathEscaped(str string) (path Path, ok bool) {
if path, ok = ParsePath(str); ok {
for i := range path {
if segment, err := url.PathUnescape(path[i]); err == nil {
path[i] = segment
} else {
return nil, false
}
}
}
return
}
// NewPathForRef returns a new path for the given ref.
func NewPathForRef(ref ast.Ref) (path Path, err error) {
if len(ref) == 0 {
return nil, errors.New("empty reference (indicates error in caller)")
}
if len(ref) == 1 {
return Path{}, nil
}
path = make(Path, 0, len(ref)-1)
for _, term := range ref[1:] {
switch v := term.Value.(type) {
case ast.String:
path = append(path, string(v))
case ast.Number:
path = append(path, v.String())
case ast.Boolean, ast.Null:
return nil, &Error{
Code: NotFoundErr,
Message: fmt.Sprintf("%v: does not exist", ref),
}
case *ast.Array, ast.Object, ast.Set:
return nil, fmt.Errorf("composites cannot be base document keys: %v", ref)
default:
return nil, fmt.Errorf("unresolved reference (indicates error in caller): %v", ref)
}
}
return path, nil
}
// Compare performs lexigraphical comparison on p and other and returns -1 if p
// is less than other, 0 if p is equal to other, or 1 if p is greater than
// other.
func (p Path) Compare(other Path) (cmp int) {
return slices.Compare(p, other)
}
// Equal returns true if p is the same as other.
func (p Path) Equal(other Path) bool {
return slices.Equal(p, other)
}
// HasPrefix returns true if p starts with other.
func (p Path) HasPrefix(other Path) bool {
return len(other) <= len(p) && p[:len(other)].Equal(other)
}
// Ref returns a ref that represents p rooted at head.
func (p Path) Ref(head *ast.Term) (ref ast.Ref) {
ref = make(ast.Ref, len(p)+1)
ref[0] = head
for i := range p {
idx, err := strconv.ParseInt(p[i], 10, 64)
if err == nil {
ref[i+1] = ast.UIntNumberTerm(uint64(idx))
} else {
ref[i+1] = ast.StringTerm(p[i])
}
}
return ref
}
func (p Path) String() string {
if len(p) == 0 {
return "/"
}
l := 0
for i := range p {
l += len(p[i]) + 1
}
sb := strings.Builder{}
sb.Grow(l)
for i := range p {
sb.WriteByte('/')
sb.WriteString(url.PathEscape(p[i]))
}
return sb.String()
}
// MustParsePath returns a new Path for s. If s cannot be parsed, this function
// will panic. This is mostly for test purposes.
func MustParsePath(s string) Path {
path, ok := ParsePath(s)
if !ok {
panic(s)
}
return path
}
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2016 The OPA Authors. All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.
package storage
import (
"context"
"github.com/open-policy-agent/opa/v1/ast"
)
// NewTransactionOrDie is a helper function to create a new transaction. If the
// storage layer cannot create a new transaction, this function will panic. This
// function should only be used for tests.
func NewTransactionOrDie(ctx context.Context, store Store, params ...TransactionParams) Transaction {
txn, err := store.NewTransaction(ctx, params...)
if err != nil {
panic(err)
}
return txn
}
// ReadOne is a convenience function to read a single value from the provided Store. It
// will create a new Transaction to perform the read with, and clean up after itself
// should an error occur.
func ReadOne(ctx context.Context, store Store, path Path) (any, error) {
txn, err := store.NewTransaction(ctx)
if err != nil {
return nil, err
}
defer store.Abort(ctx, txn)
return store.Read(ctx, txn, path)
}
// WriteOne is a convenience function to write a single value to the provided Store. It
// will create a new Transaction to perform the write with, and clean up after itself
// should an error occur.
func WriteOne(ctx context.Context, store Store, op PatchOp, path Path, value any) error {
txn, err := store.NewTransaction(ctx, WriteParams)
if err != nil {
return err
}
if err := store.Write(ctx, txn, op, path, value); err != nil {
store.Abort(ctx, txn)
return err
}
return store.Commit(ctx, txn)
}
// MakeDir inserts an empty object at path. If the parent path does not exist,
// MakeDir will create it recursively.
func MakeDir(ctx context.Context, store Store, txn Transaction, path Path) error {
// Allow the Store implementation to deal with this in its own way.
if md, ok := store.(MakeDirer); ok {
return md.MakeDir(ctx, txn, path)
}
if len(path) == 0 {
return nil
}
node, err := store.Read(ctx, txn, path)
if err != nil {
if !IsNotFound(err) {
return err
}
if err := MakeDir(ctx, store, txn, path[:len(path)-1]); err != nil {
return err
}
return store.Write(ctx, txn, AddOp, path, map[string]any{})
}
if _, ok := node.(map[string]any); ok {
return nil
}
if _, ok := node.(ast.Object); ok {
return nil
}
return writeConflictError(path)
}
// Txn is a convenience function that executes f inside a new transaction
// opened on the store. If the function returns an error, the transaction is
// aborted and the error is returned. Otherwise, the transaction is committed
// and the result of the commit is returned.
func Txn(ctx context.Context, store Store, params TransactionParams, f func(Transaction) error) error {
txn, err := store.NewTransaction(ctx, params)
if err != nil {
return err
}
if err := f(txn); err != nil {
store.Abort(ctx, txn)
return err
}
return store.Commit(ctx, txn)
}
// NonEmpty returns a function that tests if a path is non-empty. A
// path is non-empty if a Read on the path returns a value or a Read
// on any of the path prefixes returns a non-object value.
func NonEmpty(ctx context.Context, store Store, txn Transaction) func([]string) (bool, error) {
if md, ok := store.(NonEmptyer); ok {
return md.NonEmpty(ctx, txn)
}
return func(path []string) (bool, error) {
if _, err := store.Read(ctx, txn, Path(path)); err == nil {
return true, nil
} else if !IsNotFound(err) {
return false, err
}
for i := len(path) - 1; i > 0; i-- {
val, err := store.Read(ctx, txn, Path(path[:i]))
if err != nil && !IsNotFound(err) {
return false, err
} else if err == nil {
if _, ok := val.(map[string]any); ok {
return false, nil
}
if _, ok := val.(ast.Object); ok {
return false, nil
}
return true, nil
}
}
return false, nil
}
}