Initial QSfera import
This commit is contained in:
+97
@@ -0,0 +1,97 @@
|
||||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package level
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/go-kit/log"
|
||||
"github.com/go-kit/log/level"
|
||||
)
|
||||
|
||||
var logLevel = LevelInfo
|
||||
|
||||
// A Level is a logging priority. Higher levels are more important.
|
||||
type Level int
|
||||
|
||||
const (
|
||||
// LevelDebug logs are typically voluminous, and are usually disabled in
|
||||
// production.
|
||||
LevelDebug Level = iota
|
||||
// LevelInfo is the default logging priority.
|
||||
LevelInfo
|
||||
// LevelWarn logs are more important than Info, but don't need individual
|
||||
// human review.
|
||||
LevelWarn
|
||||
// LevelError logs are high-priority. If an application is running smoothly,
|
||||
// it shouldn't generate any error-level logs.
|
||||
LevelError
|
||||
)
|
||||
|
||||
var emptyLogger = &EmptyLogger{}
|
||||
|
||||
type EmptyLogger struct{}
|
||||
|
||||
func (l *EmptyLogger) Log(keyvals ...interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetLogLevel sets the log level.
|
||||
func SetLogLevel(level string) error {
|
||||
switch level {
|
||||
case "debug":
|
||||
logLevel = LevelDebug
|
||||
case "info":
|
||||
logLevel = LevelInfo
|
||||
case "warn":
|
||||
logLevel = LevelWarn
|
||||
case "error":
|
||||
logLevel = LevelError
|
||||
default:
|
||||
return fmt.Errorf("unrecognized log level %s", level)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Error returns a logger that includes a Key/ErrorValue pair.
|
||||
func Error(logger log.Logger) log.Logger {
|
||||
if logLevel <= LevelError {
|
||||
return level.Error(logger)
|
||||
}
|
||||
return emptyLogger
|
||||
}
|
||||
|
||||
// Warn returns a logger that includes a Key/WarnValue pair.
|
||||
func Warn(logger log.Logger) log.Logger {
|
||||
if logLevel <= LevelWarn {
|
||||
return level.Warn(logger)
|
||||
}
|
||||
return emptyLogger
|
||||
}
|
||||
|
||||
// Info returns a logger that includes a Key/InfoValue pair.
|
||||
func Info(logger log.Logger) log.Logger {
|
||||
if logLevel <= LevelInfo {
|
||||
return level.Info(logger)
|
||||
}
|
||||
return emptyLogger
|
||||
}
|
||||
|
||||
// Debug returns a logger that includes a Key/DebugValue pair.
|
||||
func Debug(logger log.Logger) log.Logger {
|
||||
if logLevel <= LevelDebug {
|
||||
return level.Debug(logger)
|
||||
}
|
||||
return emptyLogger
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import "fmt"
|
||||
|
||||
type ActionType string
|
||||
|
||||
const (
|
||||
ActionTypeMap ActionType = "map"
|
||||
ActionTypeDrop ActionType = "drop"
|
||||
ActionTypeDefault ActionType = ""
|
||||
)
|
||||
|
||||
func (t *ActionType) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var v string
|
||||
|
||||
if err := unmarshal(&v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch ActionType(v) {
|
||||
case ActionTypeDrop:
|
||||
*t = ActionTypeDrop
|
||||
case ActionTypeMap, ActionTypeDefault:
|
||||
*t = ActionTypeMap
|
||||
default:
|
||||
return fmt.Errorf("invalid action type %q", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// EscapeMetricName replaces invalid characters in the metric name with "_"
|
||||
// Valid characters are a-z, A-Z, 0-9, and _
|
||||
func EscapeMetricName(metricName string) string {
|
||||
metricLen := len(metricName)
|
||||
if metricLen == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
escaped := false
|
||||
var sb strings.Builder
|
||||
// If a metric starts with a digit, allocate the memory and prepend an
|
||||
// underscore.
|
||||
if metricName[0] >= '0' && metricName[0] <= '9' {
|
||||
escaped = true
|
||||
sb.Grow(metricLen + 1)
|
||||
sb.WriteByte('_')
|
||||
}
|
||||
|
||||
// This is an character replacement method optimized for this limited
|
||||
// use case. It is much faster than using a regex.
|
||||
offset := 0
|
||||
|
||||
var prevChar rune
|
||||
|
||||
for i, c := range metricName {
|
||||
// Seek forward, skipping valid characters until we find one that needs
|
||||
// to be replaced, then add all the characters we've seen so far to the
|
||||
// string.Builder.
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
||||
(c >= '0' && c <= '9') || (c == '_') {
|
||||
// Character is valid, so skip over it without doing anything.
|
||||
} else {
|
||||
// Double-dashes are allowed if there is a corresponding mapping.
|
||||
// For consistency, double-dashes should also be allowed in the default case.
|
||||
if c == '-' && prevChar == '-' {
|
||||
offset = i + utf8.RuneLen(c)
|
||||
continue
|
||||
}
|
||||
|
||||
if !escaped {
|
||||
// Up until now we've been lazy and avoided actually allocating
|
||||
// memory. Unfortunately we've now determined this string needs
|
||||
// escaping, so allocate the buffer for the whole string.
|
||||
escaped = true
|
||||
sb.Grow(metricLen)
|
||||
}
|
||||
sb.WriteString(metricName[offset:i])
|
||||
offset = i + utf8.RuneLen(c)
|
||||
sb.WriteByte('_')
|
||||
}
|
||||
|
||||
prevChar = c
|
||||
}
|
||||
|
||||
if !escaped {
|
||||
// This is the happy path where nothing had to be escaped, so we can
|
||||
// avoid doing anything.
|
||||
return metricName
|
||||
}
|
||||
|
||||
if offset < metricLen {
|
||||
sb.WriteString(metricName[offset:])
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
# FSM Mapping
|
||||
|
||||
## Overview
|
||||
|
||||
This package implements a fast and efficient algorithm for generic glob style
|
||||
string matching using a finite state machine (FSM).
|
||||
|
||||
### Source Hierachy
|
||||
|
||||
```
|
||||
'-- fsm
|
||||
'-- dump.go // functionality to dump the FSM to Dot file
|
||||
'-- formatter.go // format glob templates using captured * groups
|
||||
'-- fsm.go // manipulating and searching of FSM
|
||||
'-- minmax.go // min() max() function for interger
|
||||
```
|
||||
|
||||
## FSM Explained
|
||||
|
||||
Per [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine):
|
||||
|
||||
> A finite-state machine (FSM) or finite-state automaton (FSA, plural: automata),
|
||||
> finite automaton, or simply a state machine, is a mathematical model of
|
||||
> computation. It is an abstract machine that can be in exactly one of a finite
|
||||
> number of states at any given time. The FSM can change from one state to
|
||||
> another in response to some external inputs; the change from one state to
|
||||
> another is called a transition. An FSM is defined by a list of its states, its
|
||||
> initial state, and the conditions for each transition.
|
||||
|
||||
In our use case, each *state* is a substring after the input StatsD metric name is splitted by `.`.
|
||||
|
||||
### Add state to FSM
|
||||
|
||||
`func (f *FSM) AddState(match string, matchMetricType string,
|
||||
maxPossibleTransitions int, result interface{}) int`
|
||||
|
||||
At first, the FSM only contains three states, representing three possible metric types:
|
||||
|
||||
____ [gauge]
|
||||
/
|
||||
(start)---- [counter]
|
||||
\
|
||||
'--- [observer]
|
||||
|
||||
|
||||
Adding a rule `client.*.request.count` with type `counter` will make the FSM to be:
|
||||
|
||||
|
||||
____ [gauge]
|
||||
/
|
||||
(start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1}
|
||||
\
|
||||
'--- [observer]
|
||||
|
||||
`{R1}` is short for result 1, which is the match result for `client.*.request.count`.
|
||||
|
||||
Adding a rule `client.*.*.size` with type `counter` will make the FSM to be:
|
||||
|
||||
____ [gauge] __ [request] -- [count] -- {R1}
|
||||
/ /
|
||||
(start)---- [counter] -- [client] -- [*]
|
||||
\ \__ [*] -- [size] -- {R2}
|
||||
'--- [observer]
|
||||
|
||||
|
||||
### Finding a result state in FSM
|
||||
|
||||
`func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string)
|
||||
(*mappingState, []string)`
|
||||
|
||||
For example, when mapping `client.aaa.request.count` with `counter` type in the
|
||||
FSM, the `^1` to `^7` symbols indicate how FSM will traversal in its tree:
|
||||
|
||||
|
||||
____ [gauge] __ [request] -- [count] -- {R1}
|
||||
/ / ^5 ^6 ^7
|
||||
(start)---- [counter] -- [client] -- [*]
|
||||
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
|
||||
'--- [observer] ^4
|
||||
|
||||
|
||||
To map `client.bbb.request.size`, FSM will do a backtracking:
|
||||
|
||||
|
||||
____ [gauge] __ [request] -- [count] -- {R1}
|
||||
/ / ^5 ^6
|
||||
(start)---- [counter] -- [client] -- [*]
|
||||
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
|
||||
'--- [observer] ^4
|
||||
^7 ^8 ^9
|
||||
|
||||
|
||||
## Debugging
|
||||
|
||||
To see all the states of the current FSM, use `func (f *FSM) DumpFSM(w io.Writer)`
|
||||
to dump into a Dot file. The Dot file can be further renderer into image using:
|
||||
|
||||
```shell
|
||||
$ dot -Tpng dump.dot > dump.png
|
||||
```
|
||||
|
||||
In StatsD exporter, one could use the following:
|
||||
|
||||
```shell
|
||||
$ statsd_exporter --statsd.mapping-config=statsd.rules --debug.dump-fsm=dump.dot
|
||||
$ dot -Tpng dump.dot > dump.png
|
||||
```
|
||||
|
||||
For example, the following rules:
|
||||
|
||||
```yaml
|
||||
mappings:
|
||||
- match: client.*.request.count
|
||||
name: request_count
|
||||
match_metric_type: counter
|
||||
labels:
|
||||
client: $1
|
||||
|
||||
- match: client.*.*.size
|
||||
name: sizes
|
||||
match_metric_type: counter
|
||||
labels:
|
||||
client: $1
|
||||
direction: $2
|
||||
```
|
||||
|
||||
will be rendered as:
|
||||
|
||||

|
||||
|
||||
The `dot` program is part of [Graphviz](https://www.graphviz.org/) and is
|
||||
available in most of popular operating systems.
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fsm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// DumpFSM accepts a io.writer and write the current FSM into dot file format.
|
||||
func (f *FSM) DumpFSM(w io.Writer) {
|
||||
idx := 0
|
||||
states := make(map[int]*mappingState)
|
||||
states[idx] = f.root
|
||||
|
||||
w.Write([]byte("digraph g {\n"))
|
||||
w.Write([]byte("rankdir=LR\n")) // make it vertical
|
||||
w.Write([]byte("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n")) // remove label of node
|
||||
|
||||
for idx < len(states) {
|
||||
for field, transition := range states[idx].transitions {
|
||||
states[len(states)] = transition
|
||||
w.Write([]byte(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)))
|
||||
if idx == 0 {
|
||||
// color for metric types
|
||||
w.Write([]byte(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)))
|
||||
} else if transition.transitions == nil || len(transition.transitions) == 0 {
|
||||
// color for end state
|
||||
w.Write([]byte(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)))
|
||||
}
|
||||
}
|
||||
idx++
|
||||
}
|
||||
// color for start state
|
||||
w.Write([]byte(fmt.Sprintln("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];")))
|
||||
w.Write([]byte("}"))
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fsm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`)
|
||||
)
|
||||
|
||||
type TemplateFormatter struct {
|
||||
captureIndexes []int
|
||||
captureCount int
|
||||
fmtString string
|
||||
}
|
||||
|
||||
// NewTemplateFormatter instantiates a TemplateFormatter
|
||||
// from given template string and the maximum amount of captures.
|
||||
func NewTemplateFormatter(template string, captureCount int) *TemplateFormatter {
|
||||
matches := templateReplaceCaptureRE.FindAllStringSubmatch(template, -1)
|
||||
if len(matches) == 0 {
|
||||
// if no regex reference found, keep it as it is
|
||||
return &TemplateFormatter{captureCount: 0, fmtString: template}
|
||||
}
|
||||
|
||||
var indexes []int
|
||||
valueFormatter := template
|
||||
for _, match := range matches {
|
||||
idx, err := strconv.Atoi(match[len(match)-1])
|
||||
if err != nil || idx > captureCount || idx < 1 {
|
||||
// if index larger than captured count or using unsupported named capture group,
|
||||
// replace with empty string
|
||||
valueFormatter = strings.Replace(valueFormatter, match[0], "", -1)
|
||||
} else {
|
||||
valueFormatter = strings.Replace(valueFormatter, match[0], "%s", -1)
|
||||
// note: the regex reference variable $? starts from 1
|
||||
indexes = append(indexes, idx-1)
|
||||
}
|
||||
}
|
||||
return &TemplateFormatter{
|
||||
captureIndexes: indexes,
|
||||
captureCount: len(indexes),
|
||||
fmtString: valueFormatter,
|
||||
}
|
||||
}
|
||||
|
||||
// Format accepts a list containing captured strings and returns the formatted
|
||||
// string using the template stored in current TemplateFormatter.
|
||||
func (formatter *TemplateFormatter) Format(captures []string) string {
|
||||
if formatter.captureCount == 0 {
|
||||
// no label substitution, keep as it is
|
||||
return formatter.fmtString
|
||||
}
|
||||
indexes := formatter.captureIndexes
|
||||
vargs := make([]interface{}, formatter.captureCount)
|
||||
for i, idx := range indexes {
|
||||
vargs[i] = captures[idx]
|
||||
}
|
||||
return fmt.Sprintf(formatter.fmtString, vargs...)
|
||||
}
|
||||
+326
@@ -0,0 +1,326 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fsm
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-kit/log"
|
||||
|
||||
"github.com/prometheus/statsd_exporter/pkg/level"
|
||||
)
|
||||
|
||||
type mappingState struct {
|
||||
transitions map[string]*mappingState
|
||||
minRemainingLength int
|
||||
maxRemainingLength int
|
||||
// result* members are nil unless there's a metric ends with this state
|
||||
Result interface{}
|
||||
ResultPriority int
|
||||
}
|
||||
|
||||
type fsmBacktrackStackCursor struct {
|
||||
fieldIndex int
|
||||
captureIndex int
|
||||
currentCapture string
|
||||
state *mappingState
|
||||
prev *fsmBacktrackStackCursor
|
||||
next *fsmBacktrackStackCursor
|
||||
}
|
||||
|
||||
type FSM struct {
|
||||
root *mappingState
|
||||
metricTypes []string
|
||||
statesCount int
|
||||
BacktrackingNeeded bool
|
||||
OrderingDisabled bool
|
||||
}
|
||||
|
||||
// NewFSM creates a new FSM instance
|
||||
func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled bool) *FSM {
|
||||
fsm := FSM{}
|
||||
root := &mappingState{}
|
||||
root.transitions = make(map[string]*mappingState, len(metricTypes))
|
||||
|
||||
for _, field := range metricTypes {
|
||||
state := &mappingState{}
|
||||
(*state).transitions = make(map[string]*mappingState, maxPossibleTransitions)
|
||||
root.transitions[string(field)] = state
|
||||
}
|
||||
fsm.OrderingDisabled = orderingDisabled
|
||||
fsm.metricTypes = metricTypes
|
||||
fsm.statesCount = 0
|
||||
fsm.root = root
|
||||
return &fsm
|
||||
}
|
||||
|
||||
// AddState adds a mapping rule into the existing FSM.
|
||||
// The maxPossibleTransitions parameter sets the expected count of transitions left.
|
||||
// The result parameter sets the generic type to be returned when fsm found a match in GetMapping.
|
||||
func (f *FSM) AddState(match string, matchMetricType string, maxPossibleTransitions int, result interface{}) int {
|
||||
// first split by "."
|
||||
matchFields := strings.Split(match, ".")
|
||||
// fill into our FSM
|
||||
roots := []*mappingState{}
|
||||
// first state is the metric type
|
||||
if matchMetricType == "" {
|
||||
// if metricType not specified, connect the start state from all three types
|
||||
for _, metricType := range f.metricTypes {
|
||||
roots = append(roots, f.root.transitions[string(metricType)])
|
||||
}
|
||||
} else {
|
||||
roots = append(roots, f.root.transitions[matchMetricType])
|
||||
}
|
||||
var captureCount int
|
||||
var finalStates []*mappingState
|
||||
// iterating over different start state (different metric types)
|
||||
for _, root := range roots {
|
||||
captureCount = 0
|
||||
// for each start state, connect from start state to end state
|
||||
for i, field := range matchFields {
|
||||
state, prs := root.transitions[field]
|
||||
if !prs {
|
||||
// create a state if it's not exist in the fsm
|
||||
state = &mappingState{}
|
||||
(*state).transitions = make(map[string]*mappingState, maxPossibleTransitions)
|
||||
(*state).maxRemainingLength = len(matchFields) - i - 1
|
||||
(*state).minRemainingLength = len(matchFields) - i - 1
|
||||
root.transitions[field] = state
|
||||
// if this is last field, set result to currentMapping instance
|
||||
if i == len(matchFields)-1 {
|
||||
root.transitions[field].Result = result
|
||||
}
|
||||
} else {
|
||||
(*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength)
|
||||
(*state).minRemainingLength = min(len(matchFields)-i-1, (*state).minRemainingLength)
|
||||
}
|
||||
if field == "*" {
|
||||
captureCount++
|
||||
}
|
||||
|
||||
// goto next state
|
||||
root = state
|
||||
}
|
||||
finalStates = append(finalStates, root)
|
||||
}
|
||||
|
||||
for _, state := range finalStates {
|
||||
state.ResultPriority = f.statesCount
|
||||
}
|
||||
|
||||
f.statesCount++
|
||||
|
||||
return captureCount
|
||||
}
|
||||
|
||||
// GetMapping using the fsm to find matching rules according to given statsdMetric and statsdMetricType.
|
||||
// If it finds a match, the final state and the captured strings are returned;
|
||||
// if there's no match found, nil and a empty list will be returned.
|
||||
func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mappingState, []string) {
|
||||
matchFields := strings.Split(statsdMetric, ".")
|
||||
currentState := f.root.transitions[statsdMetricType]
|
||||
|
||||
// the cursor/pointer in the backtrack stack implemented as a double-linked list
|
||||
var backtrackCursor *fsmBacktrackStackCursor
|
||||
resumeFromBacktrack := false
|
||||
|
||||
// the return variable
|
||||
var finalState *mappingState
|
||||
|
||||
captures := make([]string, len(matchFields))
|
||||
finalCaptures := make([]string, len(matchFields))
|
||||
// keep track of captured group so we don't need to do append() on captures
|
||||
captureIdx := 0
|
||||
filedsCount := len(matchFields)
|
||||
i := 0
|
||||
var state *mappingState
|
||||
for { // the loop for backtracking
|
||||
for { // the loop for a single "depth only" search
|
||||
var present bool
|
||||
// if we resume from backtrack, we should skip this branch in this case
|
||||
// since the state that were saved at the end of this branch
|
||||
if !resumeFromBacktrack {
|
||||
if len(currentState.transitions) > 0 {
|
||||
field := matchFields[i]
|
||||
state, present = currentState.transitions[field]
|
||||
fieldsLeft := filedsCount - i - 1
|
||||
// also compare length upfront to avoid unnecessary loop or backtrack
|
||||
if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength {
|
||||
state, present = currentState.transitions["*"]
|
||||
if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength {
|
||||
break
|
||||
} else {
|
||||
captures[captureIdx] = field
|
||||
captureIdx++
|
||||
}
|
||||
} else if f.BacktrackingNeeded {
|
||||
// if backtracking is needed, also check for alternative transition, i.e. *
|
||||
altState, present := currentState.transitions["*"]
|
||||
if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength {
|
||||
} else {
|
||||
// push to backtracking stack
|
||||
newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState,
|
||||
fieldIndex: i,
|
||||
captureIndex: captureIdx, currentCapture: field,
|
||||
}
|
||||
// if this is not the first time, connect to the previous cursor
|
||||
if backtrackCursor != nil {
|
||||
backtrackCursor.next = &newCursor
|
||||
}
|
||||
backtrackCursor = &newCursor
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// no more transitions for this state
|
||||
break
|
||||
}
|
||||
} // backtrack will resume from here
|
||||
|
||||
// do we reach a final state?
|
||||
if state.Result != nil && i == filedsCount-1 {
|
||||
if f.OrderingDisabled {
|
||||
finalState = state
|
||||
return finalState, captures
|
||||
} else if finalState == nil || finalState.ResultPriority > state.ResultPriority {
|
||||
// if we care about ordering, try to find a result with highest prioity
|
||||
finalState = state
|
||||
// do a deep copy to preserve current captures
|
||||
copy(finalCaptures, captures)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
i++
|
||||
if i >= filedsCount {
|
||||
break
|
||||
}
|
||||
|
||||
resumeFromBacktrack = false
|
||||
currentState = state
|
||||
}
|
||||
if backtrackCursor == nil {
|
||||
// if we are not doing backtracking or all path has been travesaled
|
||||
break
|
||||
} else {
|
||||
// pop one from stack
|
||||
state = backtrackCursor.state
|
||||
currentState = state
|
||||
i = backtrackCursor.fieldIndex
|
||||
captureIdx = backtrackCursor.captureIndex + 1
|
||||
// put the * capture back
|
||||
captures[captureIdx-1] = backtrackCursor.currentCapture
|
||||
backtrackCursor = backtrackCursor.prev
|
||||
if backtrackCursor != nil {
|
||||
// deref for GC
|
||||
backtrackCursor.next = nil
|
||||
}
|
||||
resumeFromBacktrack = true
|
||||
}
|
||||
}
|
||||
return finalState, finalCaptures
|
||||
}
|
||||
|
||||
// TestIfNeedBacktracking tests if backtrack is needed for given list of mappings
|
||||
// and whether ordering is disabled.
|
||||
func TestIfNeedBacktracking(mappings []string, orderingDisabled bool, logger log.Logger) bool {
|
||||
backtrackingNeeded := false
|
||||
// A has * in rules, but there's other transisitions at the same state,
|
||||
// this makes A the cause of backtracking
|
||||
ruleByLength := make(map[int][]string)
|
||||
ruleREByLength := make(map[int][]*regexp.Regexp)
|
||||
|
||||
// first sort rules by length
|
||||
for _, mapping := range mappings {
|
||||
l := len(strings.Split(mapping, "."))
|
||||
ruleByLength[l] = append(ruleByLength[l], mapping)
|
||||
|
||||
metricRe := strings.Replace(mapping, ".", "\\.", -1)
|
||||
metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1)
|
||||
regex, err := regexp.Compile("^" + metricRe + "$")
|
||||
if err != nil {
|
||||
level.Warn(logger).Log("msg", "Invalid match, cannot compile regex in mapping", "mapping", mapping, "err", err)
|
||||
}
|
||||
// put into array no matter there's error or not, we will skip later if regex is nil
|
||||
ruleREByLength[l] = append(ruleREByLength[l], regex)
|
||||
}
|
||||
|
||||
for l, rules := range ruleByLength {
|
||||
if len(rules) == 1 {
|
||||
continue
|
||||
}
|
||||
rulesRE := ruleREByLength[l]
|
||||
for i1, r1 := range rules {
|
||||
currentRuleNeedBacktrack := false
|
||||
re1 := rulesRE[i1]
|
||||
if re1 == nil || !strings.Contains(r1, "*") {
|
||||
continue
|
||||
}
|
||||
// if rule r1 is A.B.C.*.E.*, is there a rule r2 is A.B.C.D.x.x or A.B.C.*.E.F ? (x is any string or *)
|
||||
// if such r2 exists, then to match r1 we will need backtracking
|
||||
for index := 0; index < len(r1); index++ {
|
||||
if r1[index] != '*' {
|
||||
continue
|
||||
}
|
||||
// translate the substring of r1 from 0 to the index of current * into regex
|
||||
// A.B.C.*.E.* will becomes ^A\.B\.C\. and ^A\.B\.C\.\*\.E\.
|
||||
reStr := strings.Replace(r1[:index], ".", "\\.", -1)
|
||||
reStr = strings.Replace(reStr, "*", "\\*", -1)
|
||||
re := regexp.MustCompile("^" + reStr)
|
||||
for i2, r2 := range rules {
|
||||
if i2 == i1 {
|
||||
continue
|
||||
}
|
||||
if len(re.FindStringSubmatchIndex(r2)) > 0 {
|
||||
currentRuleNeedBacktrack = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i2, r2 := range rules {
|
||||
if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 {
|
||||
// log if we care about ordering and the superset occurs before
|
||||
if !orderingDisabled && i1 < i2 {
|
||||
level.Warn(logger).Log("msg", "match is a super set of match but in a lower order, the first will never be matched", "first_match", r1, "second_match", r2)
|
||||
}
|
||||
currentRuleNeedBacktrack = false
|
||||
}
|
||||
}
|
||||
for i2, re2 := range rulesRE {
|
||||
if i2 == i1 || re2 == nil {
|
||||
continue
|
||||
}
|
||||
// if r1 is a subset of other rule, we don't need backtrack
|
||||
// because either we turned on ordering
|
||||
// or we disabled ordering and can't match it even with backtrack
|
||||
if len(re2.FindStringSubmatchIndex(r1)) > 0 {
|
||||
currentRuleNeedBacktrack = false
|
||||
}
|
||||
}
|
||||
|
||||
if currentRuleNeedBacktrack {
|
||||
level.Warn(logger).Log("msg", "backtracking required because of match. Performance may be degraded", "match", r1)
|
||||
backtrackingNeeded = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backtracking will always be needed if ordering of rules is not disabled
|
||||
// since transistions are stored in (unordered) map
|
||||
// note: don't move this branch to the beginning of this function
|
||||
// since we need logs for superset rules
|
||||
|
||||
return !orderingDisabled || backtrackingNeeded
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
+30
@@ -0,0 +1,30 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fsm
|
||||
|
||||
// min and max implementation for integer
|
||||
|
||||
func min(x, y int) int {
|
||||
if x < y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
func max(x, y int) int {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-kit/log"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/prometheus/statsd_exporter/pkg/level"
|
||||
"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
|
||||
)
|
||||
|
||||
var (
|
||||
// The first segment of a match cannot start with a number
|
||||
statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
|
||||
// The subsequent segments of a match can start with a number
|
||||
// See https://github.com/prometheus/statsd_exporter/issues/328
|
||||
statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
|
||||
templateReplaceRE = `(\$\{?\d+\}?)`
|
||||
|
||||
metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
|
||||
metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
|
||||
labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
|
||||
)
|
||||
|
||||
type MetricMapper struct {
|
||||
Registerer prometheus.Registerer
|
||||
Defaults mapperConfigDefaults `yaml:"defaults"`
|
||||
Mappings []MetricMapping `yaml:"mappings"`
|
||||
FSM *fsm.FSM
|
||||
doFSM bool
|
||||
doRegex bool
|
||||
cache MetricMapperCache
|
||||
mutex sync.RWMutex
|
||||
|
||||
MappingsCount prometheus.Gauge
|
||||
|
||||
Logger log.Logger
|
||||
}
|
||||
|
||||
type SummaryOptions struct {
|
||||
Quantiles []metricObjective `yaml:"quantiles"`
|
||||
MaxAge time.Duration `yaml:"max_age"`
|
||||
AgeBuckets uint32 `yaml:"age_buckets"`
|
||||
BufCap uint32 `yaml:"buf_cap"`
|
||||
}
|
||||
|
||||
type HistogramOptions struct {
|
||||
Buckets []float64 `yaml:"buckets"`
|
||||
}
|
||||
|
||||
type metricObjective struct {
|
||||
Quantile float64 `yaml:"quantile"`
|
||||
Error float64 `yaml:"error"`
|
||||
}
|
||||
|
||||
var defaultQuantiles = []metricObjective{
|
||||
{Quantile: 0.5, Error: 0.05},
|
||||
{Quantile: 0.9, Error: 0.01},
|
||||
{Quantile: 0.99, Error: 0.001},
|
||||
}
|
||||
|
||||
func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
|
||||
var n MetricMapper
|
||||
|
||||
if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(n.Defaults.HistogramOptions.Buckets) == 0 {
|
||||
n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
|
||||
}
|
||||
|
||||
if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
|
||||
n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
|
||||
}
|
||||
|
||||
if n.Defaults.MatchType == MatchTypeDefault {
|
||||
n.Defaults.MatchType = MatchTypeGlob
|
||||
}
|
||||
|
||||
remainingMappingsCount := len(n.Mappings)
|
||||
|
||||
n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
|
||||
remainingMappingsCount, n.Defaults.GlobDisableOrdering)
|
||||
|
||||
for i := range n.Mappings {
|
||||
remainingMappingsCount--
|
||||
|
||||
currentMapping := &n.Mappings[i]
|
||||
|
||||
// check that label is correct
|
||||
for k := range currentMapping.Labels {
|
||||
if !labelNameRE.MatchString(k) {
|
||||
return fmt.Errorf("invalid label key: %s", k)
|
||||
}
|
||||
}
|
||||
|
||||
if currentMapping.Name == "" {
|
||||
return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
|
||||
}
|
||||
|
||||
if !metricNameRE.MatchString(currentMapping.Name) {
|
||||
return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
|
||||
}
|
||||
|
||||
if currentMapping.MatchType == "" {
|
||||
currentMapping.MatchType = n.Defaults.MatchType
|
||||
}
|
||||
|
||||
if currentMapping.Action == "" {
|
||||
currentMapping.Action = ActionTypeMap
|
||||
}
|
||||
|
||||
if currentMapping.MatchType == MatchTypeGlob {
|
||||
n.doFSM = true
|
||||
if !metricLineRE.MatchString(currentMapping.Match) {
|
||||
return fmt.Errorf("invalid match: %s", currentMapping.Match)
|
||||
}
|
||||
|
||||
captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
|
||||
remainingMappingsCount, currentMapping)
|
||||
|
||||
currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)
|
||||
|
||||
labelKeys := make([]string, len(currentMapping.Labels))
|
||||
labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
|
||||
labelIndex := 0
|
||||
for label, valueExpr := range currentMapping.Labels {
|
||||
labelKeys[labelIndex] = label
|
||||
labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
|
||||
labelIndex++
|
||||
}
|
||||
currentMapping.labelFormatters = labelFormatters
|
||||
currentMapping.labelKeys = labelKeys
|
||||
} else {
|
||||
if regex, err := regexp.Compile(currentMapping.Match); err != nil {
|
||||
return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
|
||||
} else {
|
||||
currentMapping.regex = regex
|
||||
}
|
||||
n.doRegex = true
|
||||
}
|
||||
|
||||
if currentMapping.ObserverType == "" {
|
||||
currentMapping.ObserverType = n.Defaults.ObserverType
|
||||
}
|
||||
|
||||
if currentMapping.LegacyQuantiles != nil &&
|
||||
(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
|
||||
level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated. Please use quantiles in the summary_options hierarchy")
|
||||
}
|
||||
|
||||
if currentMapping.LegacyBuckets != nil &&
|
||||
(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
|
||||
level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated. Please use buckets in the histogram_options hierarchy")
|
||||
}
|
||||
|
||||
if currentMapping.SummaryOptions != nil &&
|
||||
currentMapping.LegacyQuantiles != nil &&
|
||||
currentMapping.SummaryOptions.Quantiles != nil {
|
||||
return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
|
||||
}
|
||||
|
||||
if currentMapping.HistogramOptions != nil &&
|
||||
currentMapping.LegacyBuckets != nil &&
|
||||
currentMapping.HistogramOptions.Buckets != nil {
|
||||
return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
|
||||
}
|
||||
|
||||
if currentMapping.ObserverType == ObserverTypeHistogram {
|
||||
if currentMapping.SummaryOptions != nil {
|
||||
return fmt.Errorf("cannot use histogram observer and summary options at the same time")
|
||||
}
|
||||
if currentMapping.HistogramOptions == nil {
|
||||
currentMapping.HistogramOptions = &HistogramOptions{}
|
||||
}
|
||||
if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
|
||||
currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
|
||||
}
|
||||
if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
|
||||
currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
|
||||
}
|
||||
}
|
||||
|
||||
if currentMapping.ObserverType == ObserverTypeSummary {
|
||||
if currentMapping.HistogramOptions != nil {
|
||||
return fmt.Errorf("cannot use summary observer and histogram options at the same time")
|
||||
}
|
||||
if currentMapping.SummaryOptions == nil {
|
||||
currentMapping.SummaryOptions = &SummaryOptions{}
|
||||
}
|
||||
if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
|
||||
currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
|
||||
}
|
||||
if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
|
||||
currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
|
||||
}
|
||||
if currentMapping.SummaryOptions.MaxAge == 0 {
|
||||
currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
|
||||
}
|
||||
if currentMapping.SummaryOptions.AgeBuckets == 0 {
|
||||
currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
|
||||
}
|
||||
if currentMapping.SummaryOptions.BufCap == 0 {
|
||||
currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
|
||||
}
|
||||
}
|
||||
|
||||
if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
|
||||
currentMapping.Ttl = n.Defaults.Ttl
|
||||
}
|
||||
}
|
||||
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
if m.Logger == nil {
|
||||
m.Logger = log.NewNopLogger()
|
||||
}
|
||||
|
||||
m.Defaults = n.Defaults
|
||||
m.Mappings = n.Mappings
|
||||
|
||||
// Reset the cache since this function can be used to reload config
|
||||
if m.cache != nil {
|
||||
m.cache.Reset()
|
||||
}
|
||||
|
||||
if n.doFSM {
|
||||
var mappings []string
|
||||
for _, mapping := range n.Mappings {
|
||||
if mapping.MatchType == MatchTypeGlob {
|
||||
mappings = append(mappings, mapping.Match)
|
||||
}
|
||||
}
|
||||
n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)
|
||||
|
||||
m.FSM = n.FSM
|
||||
m.doRegex = n.doRegex
|
||||
}
|
||||
m.doFSM = n.doFSM
|
||||
|
||||
if m.MappingsCount != nil {
|
||||
m.MappingsCount.Set(float64(len(n.Mappings)))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MetricMapper) InitFromFile(fileName string) error {
|
||||
mappingStr, err := os.ReadFile(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return m.InitFromYAMLString(string(mappingStr))
|
||||
}
|
||||
|
||||
// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
|
||||
// This cache MUST be thread-safe!
|
||||
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.cache = cache
|
||||
}
|
||||
|
||||
func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
|
||||
m.mutex.RLock()
|
||||
defer m.mutex.RUnlock()
|
||||
|
||||
// only use a cache if one is present
|
||||
if m.cache != nil {
|
||||
result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
|
||||
if cached {
|
||||
r := result.(MetricMapperCacheResult)
|
||||
return r.Mapping, r.Labels, r.Matched
|
||||
}
|
||||
}
|
||||
|
||||
// glob matching
|
||||
if m.doFSM {
|
||||
finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
|
||||
if finalState != nil && finalState.Result != nil {
|
||||
v := finalState.Result.(*MetricMapping)
|
||||
result := copyMetricMapping(v)
|
||||
result.Name = result.nameFormatter.Format(captures)
|
||||
|
||||
labels := prometheus.Labels{}
|
||||
for index, formatter := range result.labelFormatters {
|
||||
labels[result.labelKeys[index]] = formatter.Format(captures)
|
||||
}
|
||||
|
||||
r := MetricMapperCacheResult{
|
||||
Mapping: result,
|
||||
Matched: true,
|
||||
Labels: labels,
|
||||
}
|
||||
// add match to cache
|
||||
if m.cache != nil {
|
||||
m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
|
||||
}
|
||||
|
||||
return result, labels, true
|
||||
} else if !m.doRegex {
|
||||
// if there's no regex match type, return immediately
|
||||
// Add miss to cache
|
||||
if m.cache != nil {
|
||||
m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// regex matching
|
||||
for _, mapping := range m.Mappings {
|
||||
// if a rule don't have regex matching type, the regex field is unset
|
||||
if mapping.regex == nil {
|
||||
continue
|
||||
}
|
||||
matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
mapping.Name = string(mapping.regex.ExpandString(
|
||||
[]byte{},
|
||||
mapping.Name,
|
||||
statsdMetric,
|
||||
matches,
|
||||
))
|
||||
|
||||
if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
|
||||
continue
|
||||
}
|
||||
|
||||
labels := prometheus.Labels{}
|
||||
for label, valueExpr := range mapping.Labels {
|
||||
value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
|
||||
labels[label] = string(value)
|
||||
}
|
||||
|
||||
r := MetricMapperCacheResult{
|
||||
Mapping: &mapping,
|
||||
Matched: true,
|
||||
Labels: labels,
|
||||
}
|
||||
// Add Match to cache
|
||||
if m.cache != nil {
|
||||
m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
|
||||
}
|
||||
|
||||
return &mapping, labels, true
|
||||
}
|
||||
|
||||
// Add Miss to cache
|
||||
if m.cache != nil {
|
||||
m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
// make a shallow copy so that we do not overwrite name
|
||||
// as multiple names can be matched by same mapping
|
||||
func copyMetricMapping(in *MetricMapping) *MetricMapping {
|
||||
out := *in
|
||||
return &out
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
// Copyright 2019 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type CacheMetrics struct {
|
||||
CacheLength prometheus.Gauge
|
||||
CacheGetsTotal prometheus.Counter
|
||||
CacheHitsTotal prometheus.Counter
|
||||
}
|
||||
|
||||
func NewCacheMetrics(reg prometheus.Registerer) *CacheMetrics {
|
||||
var m CacheMetrics
|
||||
|
||||
m.CacheLength = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "statsd_metric_mapper_cache_length",
|
||||
Help: "The count of unique metrics currently cached.",
|
||||
},
|
||||
)
|
||||
m.CacheGetsTotal = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "statsd_metric_mapper_cache_gets_total",
|
||||
Help: "The count of total metric cache gets.",
|
||||
},
|
||||
)
|
||||
m.CacheHitsTotal = prometheus.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "statsd_metric_mapper_cache_hits_total",
|
||||
Help: "The count of total metric cache hits.",
|
||||
},
|
||||
)
|
||||
|
||||
if reg != nil {
|
||||
reg.MustRegister(m.CacheLength)
|
||||
reg.MustRegister(m.CacheGetsTotal)
|
||||
reg.MustRegister(m.CacheHitsTotal)
|
||||
}
|
||||
return &m
|
||||
}
|
||||
|
||||
type MetricMapperCacheResult struct {
|
||||
Mapping *MetricMapping
|
||||
Matched bool
|
||||
Labels prometheus.Labels
|
||||
}
|
||||
|
||||
// MetricMapperCache MUST be thread-safe and should be instrumented with CacheMetrics
|
||||
type MetricMapperCache interface {
|
||||
// Get a cached result
|
||||
Get(metricKey string) (interface{}, bool)
|
||||
// Add a statsd MetricMapperResult to the cache
|
||||
Add(metricKey string, result interface{}) // Add an item to the cache
|
||||
// Reset clears the cache for config reloads
|
||||
Reset()
|
||||
}
|
||||
|
||||
func formatKey(metricString string, metricType MetricType) string {
|
||||
return string(metricType) + "." + metricString
|
||||
}
|
||||
Generated
Vendored
+72
@@ -0,0 +1,72 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import "time"
|
||||
|
||||
type mapperConfigDefaults struct {
|
||||
ObserverType ObserverType `yaml:"observer_type"`
|
||||
MatchType MatchType `yaml:"match_type"`
|
||||
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
|
||||
Ttl time.Duration `yaml:"ttl"`
|
||||
SummaryOptions SummaryOptions `yaml:"summary_options"`
|
||||
HistogramOptions HistogramOptions `yaml:"histogram_options"`
|
||||
}
|
||||
|
||||
// mapperConfigDefaultsAlias is used to unmarshal the yaml config into mapperConfigDefaults and allows deprecated fields
|
||||
type mapperConfigDefaultsAlias struct {
|
||||
ObserverType ObserverType `yaml:"observer_type"`
|
||||
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs
|
||||
Buckets []float64 `yaml:"buckets"` // DEPRECATED - field only present to preserve backwards compatibility in configs
|
||||
Quantiles []metricObjective `yaml:"quantiles"` // DEPRECATED - field only present to preserve backwards compatibility in configs
|
||||
MatchType MatchType `yaml:"match_type"`
|
||||
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
|
||||
Ttl time.Duration `yaml:"ttl"`
|
||||
SummaryOptions SummaryOptions `yaml:"summary_options"`
|
||||
HistogramOptions HistogramOptions `yaml:"histogram_options"`
|
||||
}
|
||||
|
||||
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
|
||||
// observer_type will override timer_type
|
||||
func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var tmp mapperConfigDefaultsAlias
|
||||
if err := unmarshal(&tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy defaults
|
||||
d.ObserverType = tmp.ObserverType
|
||||
d.MatchType = tmp.MatchType
|
||||
d.GlobDisableOrdering = tmp.GlobDisableOrdering
|
||||
d.Ttl = tmp.Ttl
|
||||
d.SummaryOptions = tmp.SummaryOptions
|
||||
d.HistogramOptions = tmp.HistogramOptions
|
||||
|
||||
// Use deprecated TimerType if necessary
|
||||
if tmp.ObserverType == "" {
|
||||
d.ObserverType = tmp.TimerType
|
||||
}
|
||||
|
||||
// Use deprecated quantiles if necessary
|
||||
if len(tmp.SummaryOptions.Quantiles) == 0 && len(tmp.Quantiles) > 0 {
|
||||
d.SummaryOptions = SummaryOptions{Quantiles: tmp.Quantiles}
|
||||
}
|
||||
|
||||
// Use deprecated buckets if necessary
|
||||
if len(tmp.HistogramOptions.Buckets) == 0 && len(tmp.Buckets) > 0 {
|
||||
d.HistogramOptions = HistogramOptions{Buckets: tmp.Buckets}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either xpress or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
|
||||
)
|
||||
|
||||
type MetricMapping struct {
|
||||
Match string `yaml:"match"`
|
||||
Name string `yaml:"name"`
|
||||
nameFormatter *fsm.TemplateFormatter
|
||||
regex *regexp.Regexp
|
||||
Labels prometheus.Labels `yaml:"labels"`
|
||||
labelKeys []string
|
||||
labelFormatters []*fsm.TemplateFormatter
|
||||
ObserverType ObserverType `yaml:"observer_type"`
|
||||
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty
|
||||
LegacyBuckets []float64 `yaml:"buckets"`
|
||||
LegacyQuantiles []metricObjective `yaml:"quantiles"`
|
||||
MatchType MatchType `yaml:"match_type"`
|
||||
HelpText string `yaml:"help"`
|
||||
Action ActionType `yaml:"action"`
|
||||
MatchMetricType MetricType `yaml:"match_metric_type"`
|
||||
Ttl time.Duration `yaml:"ttl"`
|
||||
SummaryOptions *SummaryOptions `yaml:"summary_options"`
|
||||
HistogramOptions *HistogramOptions `yaml:"histogram_options"`
|
||||
}
|
||||
|
||||
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
|
||||
// observer_type will override timer_type
|
||||
func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
type MetricMappingAlias MetricMapping
|
||||
var tmp MetricMappingAlias
|
||||
if err := unmarshal(&tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy defaults
|
||||
m.Match = tmp.Match
|
||||
m.Name = tmp.Name
|
||||
m.Labels = tmp.Labels
|
||||
m.ObserverType = tmp.ObserverType
|
||||
m.LegacyBuckets = tmp.LegacyBuckets
|
||||
m.LegacyQuantiles = tmp.LegacyQuantiles
|
||||
m.MatchType = tmp.MatchType
|
||||
m.HelpText = tmp.HelpText
|
||||
m.Action = tmp.Action
|
||||
m.MatchMetricType = tmp.MatchMetricType
|
||||
m.Ttl = tmp.Ttl
|
||||
m.SummaryOptions = tmp.SummaryOptions
|
||||
m.HistogramOptions = tmp.HistogramOptions
|
||||
|
||||
// Use deprecated TimerType if necessary
|
||||
if tmp.ObserverType == "" {
|
||||
m.ObserverType = tmp.TimerType
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import "fmt"
|
||||
|
||||
type MatchType string
|
||||
|
||||
const (
|
||||
MatchTypeGlob MatchType = "glob"
|
||||
MatchTypeRegex MatchType = "regex"
|
||||
MatchTypeDefault MatchType = ""
|
||||
)
|
||||
|
||||
func (t *MatchType) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var v string
|
||||
if err := unmarshal(&v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch MatchType(v) {
|
||||
case MatchTypeRegex:
|
||||
*t = MatchTypeRegex
|
||||
case MatchTypeGlob, MatchTypeDefault:
|
||||
*t = MatchTypeGlob
|
||||
default:
|
||||
return fmt.Errorf("invalid match type %q", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import "fmt"
|
||||
|
||||
type MetricType string
|
||||
|
||||
const (
|
||||
MetricTypeCounter MetricType = "counter"
|
||||
MetricTypeGauge MetricType = "gauge"
|
||||
MetricTypeObserver MetricType = "observer"
|
||||
MetricTypeTimer MetricType = "timer" // DEPRECATED
|
||||
)
|
||||
|
||||
func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var v string
|
||||
if err := unmarshal(&v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch MetricType(v) {
|
||||
case MetricTypeCounter:
|
||||
*m = MetricTypeCounter
|
||||
case MetricTypeGauge:
|
||||
*m = MetricTypeGauge
|
||||
case MetricTypeObserver:
|
||||
*m = MetricTypeObserver
|
||||
case MetricTypeTimer:
|
||||
*m = MetricTypeObserver
|
||||
default:
|
||||
return fmt.Errorf("invalid metric type '%s'", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mapper
|
||||
|
||||
import "fmt"
|
||||
|
||||
type ObserverType string
|
||||
|
||||
const (
|
||||
ObserverTypeHistogram ObserverType = "histogram"
|
||||
ObserverTypeSummary ObserverType = "summary"
|
||||
ObserverTypeDefault ObserverType = ""
|
||||
)
|
||||
|
||||
func (t *ObserverType) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var v string
|
||||
if err := unmarshal(&v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch ObserverType(v) {
|
||||
case ObserverTypeHistogram:
|
||||
*t = ObserverTypeHistogram
|
||||
case ObserverTypeSummary, ObserverTypeDefault:
|
||||
*t = ObserverTypeSummary
|
||||
default:
|
||||
return fmt.Errorf("invalid observer type '%s'", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user