Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2021 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package level
import (
"fmt"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
)
var logLevel = LevelInfo
// A Level is a logging priority. Higher levels are more important.
type Level int
const (
// LevelDebug logs are typically voluminous, and are usually disabled in
// production.
LevelDebug Level = iota
// LevelInfo is the default logging priority.
LevelInfo
// LevelWarn logs are more important than Info, but don't need individual
// human review.
LevelWarn
// LevelError logs are high-priority. If an application is running smoothly,
// it shouldn't generate any error-level logs.
LevelError
)
var emptyLogger = &EmptyLogger{}
type EmptyLogger struct{}
func (l *EmptyLogger) Log(keyvals ...interface{}) error {
return nil
}
// SetLogLevel sets the log level.
func SetLogLevel(level string) error {
switch level {
case "debug":
logLevel = LevelDebug
case "info":
logLevel = LevelInfo
case "warn":
logLevel = LevelWarn
case "error":
logLevel = LevelError
default:
return fmt.Errorf("unrecognized log level %s", level)
}
return nil
}
// Error returns a logger that includes a Key/ErrorValue pair.
func Error(logger log.Logger) log.Logger {
if logLevel <= LevelError {
return level.Error(logger)
}
return emptyLogger
}
// Warn returns a logger that includes a Key/WarnValue pair.
func Warn(logger log.Logger) log.Logger {
if logLevel <= LevelWarn {
return level.Warn(logger)
}
return emptyLogger
}
// Info returns a logger that includes a Key/InfoValue pair.
func Info(logger log.Logger) log.Logger {
if logLevel <= LevelInfo {
return level.Info(logger)
}
return emptyLogger
}
// Debug returns a logger that includes a Key/DebugValue pair.
func Debug(logger log.Logger) log.Logger {
if logLevel <= LevelDebug {
return level.Debug(logger)
}
return emptyLogger
}
@@ -0,0 +1,42 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "fmt"
type ActionType string
const (
ActionTypeMap ActionType = "map"
ActionTypeDrop ActionType = "drop"
ActionTypeDefault ActionType = ""
)
func (t *ActionType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch ActionType(v) {
case ActionTypeDrop:
*t = ActionTypeDrop
case ActionTypeMap, ActionTypeDefault:
*t = ActionTypeMap
default:
return fmt.Errorf("invalid action type %q", v)
}
return nil
}
@@ -0,0 +1,86 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"strings"
"unicode/utf8"
)
// EscapeMetricName replaces invalid characters in the metric name with "_"
// Valid characters are a-z, A-Z, 0-9, and _
func EscapeMetricName(metricName string) string {
metricLen := len(metricName)
if metricLen == 0 {
return ""
}
escaped := false
var sb strings.Builder
// If a metric starts with a digit, allocate the memory and prepend an
// underscore.
if metricName[0] >= '0' && metricName[0] <= '9' {
escaped = true
sb.Grow(metricLen + 1)
sb.WriteByte('_')
}
// This is an character replacement method optimized for this limited
// use case. It is much faster than using a regex.
offset := 0
var prevChar rune
for i, c := range metricName {
// Seek forward, skipping valid characters until we find one that needs
// to be replaced, then add all the characters we've seen so far to the
// string.Builder.
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') || (c == '_') {
// Character is valid, so skip over it without doing anything.
} else {
// Double-dashes are allowed if there is a corresponding mapping.
// For consistency, double-dashes should also be allowed in the default case.
if c == '-' && prevChar == '-' {
offset = i + utf8.RuneLen(c)
continue
}
if !escaped {
// Up until now we've been lazy and avoided actually allocating
// memory. Unfortunately we've now determined this string needs
// escaping, so allocate the buffer for the whole string.
escaped = true
sb.Grow(metricLen)
}
sb.WriteString(metricName[offset:i])
offset = i + utf8.RuneLen(c)
sb.WriteByte('_')
}
prevChar = c
}
if !escaped {
// This is the happy path where nothing had to be escaped, so we can
// avoid doing anything.
return metricName
}
if offset < metricLen {
sb.WriteString(metricName[offset:])
}
return sb.String()
}
@@ -0,0 +1,132 @@
# FSM Mapping
## Overview
This package implements a fast and efficient algorithm for generic glob style
string matching using a finite state machine (FSM).
### Source Hierachy
```
'-- fsm
'-- dump.go // functionality to dump the FSM to Dot file
'-- formatter.go // format glob templates using captured * groups
'-- fsm.go // manipulating and searching of FSM
'-- minmax.go // min() max() function for interger
```
## FSM Explained
Per [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine):
> A finite-state machine (FSM) or finite-state automaton (FSA, plural: automata),
> finite automaton, or simply a state machine, is a mathematical model of
> computation. It is an abstract machine that can be in exactly one of a finite
> number of states at any given time. The FSM can change from one state to
> another in response to some external inputs; the change from one state to
> another is called a transition. An FSM is defined by a list of its states, its
> initial state, and the conditions for each transition.
In our use case, each *state* is a substring after the input StatsD metric name is splitted by `.`.
### Add state to FSM
`func (f *FSM) AddState(match string, matchMetricType string,
maxPossibleTransitions int, result interface{}) int`
At first, the FSM only contains three states, representing three possible metric types:
____ [gauge]
/
(start)---- [counter]
\
'--- [observer]
Adding a rule `client.*.request.count` with type `counter` will make the FSM to be:
____ [gauge]
/
(start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1}
\
'--- [observer]
`{R1}` is short for result 1, which is the match result for `client.*.request.count`.
Adding a rule `client.*.*.size` with type `counter` will make the FSM to be:
____ [gauge] __ [request] -- [count] -- {R1}
/ /
(start)---- [counter] -- [client] -- [*]
\ \__ [*] -- [size] -- {R2}
'--- [observer]
### Finding a result state in FSM
`func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string)
(*mappingState, []string)`
For example, when mapping `client.aaa.request.count` with `counter` type in the
FSM, the `^1` to `^7` symbols indicate how FSM will traversal in its tree:
____ [gauge] __ [request] -- [count] -- {R1}
/ / ^5 ^6 ^7
(start)---- [counter] -- [client] -- [*]
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
'--- [observer] ^4
To map `client.bbb.request.size`, FSM will do a backtracking:
____ [gauge] __ [request] -- [count] -- {R1}
/ / ^5 ^6
(start)---- [counter] -- [client] -- [*]
^1 \ ^2 ^3 \__ [*] -- [size] -- {R2}
'--- [observer] ^4
^7 ^8 ^9
## Debugging
To see all the states of the current FSM, use `func (f *FSM) DumpFSM(w io.Writer)`
to dump into a Dot file. The Dot file can be further renderer into image using:
```shell
$ dot -Tpng dump.dot > dump.png
```
In StatsD exporter, one could use the following:
```shell
$ statsd_exporter --statsd.mapping-config=statsd.rules --debug.dump-fsm=dump.dot
$ dot -Tpng dump.dot > dump.png
```
For example, the following rules:
```yaml
mappings:
- match: client.*.request.count
name: request_count
match_metric_type: counter
labels:
client: $1
- match: client.*.*.size
name: sizes
match_metric_type: counter
labels:
client: $1
direction: $2
```
will be rendered as:
![FSM](fsm.png)
The `dot` program is part of [Graphviz](https://www.graphviz.org/) and is
available in most of popular operating systems.
@@ -0,0 +1,48 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
import (
"fmt"
"io"
)
// DumpFSM accepts a io.writer and write the current FSM into dot file format.
func (f *FSM) DumpFSM(w io.Writer) {
idx := 0
states := make(map[int]*mappingState)
states[idx] = f.root
w.Write([]byte("digraph g {\n"))
w.Write([]byte("rankdir=LR\n")) // make it vertical
w.Write([]byte("node [ label=\"\",style=filled,fillcolor=white,shape=circle ]\n")) // remove label of node
for idx < len(states) {
for field, transition := range states[idx].transitions {
states[len(states)] = transition
w.Write([]byte(fmt.Sprintf("%d -> %d [label = \"%s\"];\n", idx, len(states)-1, field)))
if idx == 0 {
// color for metric types
w.Write([]byte(fmt.Sprintf("%d [color=\"#D6B656\",fillcolor=\"#FFF2CC\"];\n", len(states)-1)))
} else if transition.transitions == nil || len(transition.transitions) == 0 {
// color for end state
w.Write([]byte(fmt.Sprintf("%d [color=\"#82B366\",fillcolor=\"#D5E8D4\"];\n", len(states)-1)))
}
}
idx++
}
// color for start state
w.Write([]byte(fmt.Sprintln("0 [color=\"#a94442\",fillcolor=\"#f2dede\"];")))
w.Write([]byte("}"))
}
@@ -0,0 +1,76 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
import (
"fmt"
"regexp"
"strconv"
"strings"
)
var (
templateReplaceCaptureRE = regexp.MustCompile(`\$\{?([a-zA-Z0-9_\$]+)\}?`)
)
type TemplateFormatter struct {
captureIndexes []int
captureCount int
fmtString string
}
// NewTemplateFormatter instantiates a TemplateFormatter
// from given template string and the maximum amount of captures.
func NewTemplateFormatter(template string, captureCount int) *TemplateFormatter {
matches := templateReplaceCaptureRE.FindAllStringSubmatch(template, -1)
if len(matches) == 0 {
// if no regex reference found, keep it as it is
return &TemplateFormatter{captureCount: 0, fmtString: template}
}
var indexes []int
valueFormatter := template
for _, match := range matches {
idx, err := strconv.Atoi(match[len(match)-1])
if err != nil || idx > captureCount || idx < 1 {
// if index larger than captured count or using unsupported named capture group,
// replace with empty string
valueFormatter = strings.Replace(valueFormatter, match[0], "", -1)
} else {
valueFormatter = strings.Replace(valueFormatter, match[0], "%s", -1)
// note: the regex reference variable $? starts from 1
indexes = append(indexes, idx-1)
}
}
return &TemplateFormatter{
captureIndexes: indexes,
captureCount: len(indexes),
fmtString: valueFormatter,
}
}
// Format accepts a list containing captured strings and returns the formatted
// string using the template stored in current TemplateFormatter.
func (formatter *TemplateFormatter) Format(captures []string) string {
if formatter.captureCount == 0 {
// no label substitution, keep as it is
return formatter.fmtString
}
indexes := formatter.captureIndexes
vargs := make([]interface{}, formatter.captureCount)
for i, idx := range indexes {
vargs[i] = captures[idx]
}
return fmt.Sprintf(formatter.fmtString, vargs...)
}
@@ -0,0 +1,326 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
import (
"regexp"
"strings"
"github.com/go-kit/log"
"github.com/prometheus/statsd_exporter/pkg/level"
)
type mappingState struct {
transitions map[string]*mappingState
minRemainingLength int
maxRemainingLength int
// result* members are nil unless there's a metric ends with this state
Result interface{}
ResultPriority int
}
type fsmBacktrackStackCursor struct {
fieldIndex int
captureIndex int
currentCapture string
state *mappingState
prev *fsmBacktrackStackCursor
next *fsmBacktrackStackCursor
}
type FSM struct {
root *mappingState
metricTypes []string
statesCount int
BacktrackingNeeded bool
OrderingDisabled bool
}
// NewFSM creates a new FSM instance
func NewFSM(metricTypes []string, maxPossibleTransitions int, orderingDisabled bool) *FSM {
fsm := FSM{}
root := &mappingState{}
root.transitions = make(map[string]*mappingState, len(metricTypes))
for _, field := range metricTypes {
state := &mappingState{}
(*state).transitions = make(map[string]*mappingState, maxPossibleTransitions)
root.transitions[string(field)] = state
}
fsm.OrderingDisabled = orderingDisabled
fsm.metricTypes = metricTypes
fsm.statesCount = 0
fsm.root = root
return &fsm
}
// AddState adds a mapping rule into the existing FSM.
// The maxPossibleTransitions parameter sets the expected count of transitions left.
// The result parameter sets the generic type to be returned when fsm found a match in GetMapping.
func (f *FSM) AddState(match string, matchMetricType string, maxPossibleTransitions int, result interface{}) int {
// first split by "."
matchFields := strings.Split(match, ".")
// fill into our FSM
roots := []*mappingState{}
// first state is the metric type
if matchMetricType == "" {
// if metricType not specified, connect the start state from all three types
for _, metricType := range f.metricTypes {
roots = append(roots, f.root.transitions[string(metricType)])
}
} else {
roots = append(roots, f.root.transitions[matchMetricType])
}
var captureCount int
var finalStates []*mappingState
// iterating over different start state (different metric types)
for _, root := range roots {
captureCount = 0
// for each start state, connect from start state to end state
for i, field := range matchFields {
state, prs := root.transitions[field]
if !prs {
// create a state if it's not exist in the fsm
state = &mappingState{}
(*state).transitions = make(map[string]*mappingState, maxPossibleTransitions)
(*state).maxRemainingLength = len(matchFields) - i - 1
(*state).minRemainingLength = len(matchFields) - i - 1
root.transitions[field] = state
// if this is last field, set result to currentMapping instance
if i == len(matchFields)-1 {
root.transitions[field].Result = result
}
} else {
(*state).maxRemainingLength = max(len(matchFields)-i-1, (*state).maxRemainingLength)
(*state).minRemainingLength = min(len(matchFields)-i-1, (*state).minRemainingLength)
}
if field == "*" {
captureCount++
}
// goto next state
root = state
}
finalStates = append(finalStates, root)
}
for _, state := range finalStates {
state.ResultPriority = f.statesCount
}
f.statesCount++
return captureCount
}
// GetMapping using the fsm to find matching rules according to given statsdMetric and statsdMetricType.
// If it finds a match, the final state and the captured strings are returned;
// if there's no match found, nil and a empty list will be returned.
func (f *FSM) GetMapping(statsdMetric string, statsdMetricType string) (*mappingState, []string) {
matchFields := strings.Split(statsdMetric, ".")
currentState := f.root.transitions[statsdMetricType]
// the cursor/pointer in the backtrack stack implemented as a double-linked list
var backtrackCursor *fsmBacktrackStackCursor
resumeFromBacktrack := false
// the return variable
var finalState *mappingState
captures := make([]string, len(matchFields))
finalCaptures := make([]string, len(matchFields))
// keep track of captured group so we don't need to do append() on captures
captureIdx := 0
filedsCount := len(matchFields)
i := 0
var state *mappingState
for { // the loop for backtracking
for { // the loop for a single "depth only" search
var present bool
// if we resume from backtrack, we should skip this branch in this case
// since the state that were saved at the end of this branch
if !resumeFromBacktrack {
if len(currentState.transitions) > 0 {
field := matchFields[i]
state, present = currentState.transitions[field]
fieldsLeft := filedsCount - i - 1
// also compare length upfront to avoid unnecessary loop or backtrack
if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength {
state, present = currentState.transitions["*"]
if !present || fieldsLeft > state.maxRemainingLength || fieldsLeft < state.minRemainingLength {
break
} else {
captures[captureIdx] = field
captureIdx++
}
} else if f.BacktrackingNeeded {
// if backtracking is needed, also check for alternative transition, i.e. *
altState, present := currentState.transitions["*"]
if !present || fieldsLeft > altState.maxRemainingLength || fieldsLeft < altState.minRemainingLength {
} else {
// push to backtracking stack
newCursor := fsmBacktrackStackCursor{prev: backtrackCursor, state: altState,
fieldIndex: i,
captureIndex: captureIdx, currentCapture: field,
}
// if this is not the first time, connect to the previous cursor
if backtrackCursor != nil {
backtrackCursor.next = &newCursor
}
backtrackCursor = &newCursor
}
}
} else {
// no more transitions for this state
break
}
} // backtrack will resume from here
// do we reach a final state?
if state.Result != nil && i == filedsCount-1 {
if f.OrderingDisabled {
finalState = state
return finalState, captures
} else if finalState == nil || finalState.ResultPriority > state.ResultPriority {
// if we care about ordering, try to find a result with highest prioity
finalState = state
// do a deep copy to preserve current captures
copy(finalCaptures, captures)
}
break
}
i++
if i >= filedsCount {
break
}
resumeFromBacktrack = false
currentState = state
}
if backtrackCursor == nil {
// if we are not doing backtracking or all path has been travesaled
break
} else {
// pop one from stack
state = backtrackCursor.state
currentState = state
i = backtrackCursor.fieldIndex
captureIdx = backtrackCursor.captureIndex + 1
// put the * capture back
captures[captureIdx-1] = backtrackCursor.currentCapture
backtrackCursor = backtrackCursor.prev
if backtrackCursor != nil {
// deref for GC
backtrackCursor.next = nil
}
resumeFromBacktrack = true
}
}
return finalState, finalCaptures
}
// TestIfNeedBacktracking tests if backtrack is needed for given list of mappings
// and whether ordering is disabled.
func TestIfNeedBacktracking(mappings []string, orderingDisabled bool, logger log.Logger) bool {
backtrackingNeeded := false
// A has * in rules, but there's other transisitions at the same state,
// this makes A the cause of backtracking
ruleByLength := make(map[int][]string)
ruleREByLength := make(map[int][]*regexp.Regexp)
// first sort rules by length
for _, mapping := range mappings {
l := len(strings.Split(mapping, "."))
ruleByLength[l] = append(ruleByLength[l], mapping)
metricRe := strings.Replace(mapping, ".", "\\.", -1)
metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1)
regex, err := regexp.Compile("^" + metricRe + "$")
if err != nil {
level.Warn(logger).Log("msg", "Invalid match, cannot compile regex in mapping", "mapping", mapping, "err", err)
}
// put into array no matter there's error or not, we will skip later if regex is nil
ruleREByLength[l] = append(ruleREByLength[l], regex)
}
for l, rules := range ruleByLength {
if len(rules) == 1 {
continue
}
rulesRE := ruleREByLength[l]
for i1, r1 := range rules {
currentRuleNeedBacktrack := false
re1 := rulesRE[i1]
if re1 == nil || !strings.Contains(r1, "*") {
continue
}
// if rule r1 is A.B.C.*.E.*, is there a rule r2 is A.B.C.D.x.x or A.B.C.*.E.F ? (x is any string or *)
// if such r2 exists, then to match r1 we will need backtracking
for index := 0; index < len(r1); index++ {
if r1[index] != '*' {
continue
}
// translate the substring of r1 from 0 to the index of current * into regex
// A.B.C.*.E.* will becomes ^A\.B\.C\. and ^A\.B\.C\.\*\.E\.
reStr := strings.Replace(r1[:index], ".", "\\.", -1)
reStr = strings.Replace(reStr, "*", "\\*", -1)
re := regexp.MustCompile("^" + reStr)
for i2, r2 := range rules {
if i2 == i1 {
continue
}
if len(re.FindStringSubmatchIndex(r2)) > 0 {
currentRuleNeedBacktrack = true
break
}
}
}
for i2, r2 := range rules {
if i2 != i1 && len(re1.FindStringSubmatchIndex(r2)) > 0 {
// log if we care about ordering and the superset occurs before
if !orderingDisabled && i1 < i2 {
level.Warn(logger).Log("msg", "match is a super set of match but in a lower order, the first will never be matched", "first_match", r1, "second_match", r2)
}
currentRuleNeedBacktrack = false
}
}
for i2, re2 := range rulesRE {
if i2 == i1 || re2 == nil {
continue
}
// if r1 is a subset of other rule, we don't need backtrack
// because either we turned on ordering
// or we disabled ordering and can't match it even with backtrack
if len(re2.FindStringSubmatchIndex(r1)) > 0 {
currentRuleNeedBacktrack = false
}
}
if currentRuleNeedBacktrack {
level.Warn(logger).Log("msg", "backtracking required because of match. Performance may be degraded", "match", r1)
backtrackingNeeded = true
}
}
}
// backtracking will always be needed if ordering of rules is not disabled
// since transistions are stored in (unordered) map
// note: don't move this branch to the beginning of this function
// since we need logs for superset rules
return !orderingDisabled || backtrackingNeeded
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

@@ -0,0 +1,30 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fsm
// min and max implementation for integer
func min(x, y int) int {
if x < y {
return x
}
return y
}
func max(x, y int) int {
if x > y {
return x
}
return y
}
@@ -0,0 +1,386 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"fmt"
"os"
"regexp"
"sync"
"time"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"gopkg.in/yaml.v2"
"github.com/prometheus/statsd_exporter/pkg/level"
"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)
var (
// The first segment of a match cannot start with a number
statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
// The subsequent segments of a match can start with a number
// See https://github.com/prometheus/statsd_exporter/issues/328
statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
templateReplaceRE = `(\$\{?\d+\}?)`
metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
)
type MetricMapper struct {
Registerer prometheus.Registerer
Defaults mapperConfigDefaults `yaml:"defaults"`
Mappings []MetricMapping `yaml:"mappings"`
FSM *fsm.FSM
doFSM bool
doRegex bool
cache MetricMapperCache
mutex sync.RWMutex
MappingsCount prometheus.Gauge
Logger log.Logger
}
type SummaryOptions struct {
Quantiles []metricObjective `yaml:"quantiles"`
MaxAge time.Duration `yaml:"max_age"`
AgeBuckets uint32 `yaml:"age_buckets"`
BufCap uint32 `yaml:"buf_cap"`
}
type HistogramOptions struct {
Buckets []float64 `yaml:"buckets"`
}
type metricObjective struct {
Quantile float64 `yaml:"quantile"`
Error float64 `yaml:"error"`
}
var defaultQuantiles = []metricObjective{
{Quantile: 0.5, Error: 0.05},
{Quantile: 0.9, Error: 0.01},
{Quantile: 0.99, Error: 0.001},
}
func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
var n MetricMapper
if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
return err
}
if len(n.Defaults.HistogramOptions.Buckets) == 0 {
n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
}
if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
}
if n.Defaults.MatchType == MatchTypeDefault {
n.Defaults.MatchType = MatchTypeGlob
}
remainingMappingsCount := len(n.Mappings)
n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
remainingMappingsCount, n.Defaults.GlobDisableOrdering)
for i := range n.Mappings {
remainingMappingsCount--
currentMapping := &n.Mappings[i]
// check that label is correct
for k := range currentMapping.Labels {
if !labelNameRE.MatchString(k) {
return fmt.Errorf("invalid label key: %s", k)
}
}
if currentMapping.Name == "" {
return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
}
if !metricNameRE.MatchString(currentMapping.Name) {
return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
}
if currentMapping.MatchType == "" {
currentMapping.MatchType = n.Defaults.MatchType
}
if currentMapping.Action == "" {
currentMapping.Action = ActionTypeMap
}
if currentMapping.MatchType == MatchTypeGlob {
n.doFSM = true
if !metricLineRE.MatchString(currentMapping.Match) {
return fmt.Errorf("invalid match: %s", currentMapping.Match)
}
captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
remainingMappingsCount, currentMapping)
currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)
labelKeys := make([]string, len(currentMapping.Labels))
labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
labelIndex := 0
for label, valueExpr := range currentMapping.Labels {
labelKeys[labelIndex] = label
labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
labelIndex++
}
currentMapping.labelFormatters = labelFormatters
currentMapping.labelKeys = labelKeys
} else {
if regex, err := regexp.Compile(currentMapping.Match); err != nil {
return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
} else {
currentMapping.regex = regex
}
n.doRegex = true
}
if currentMapping.ObserverType == "" {
currentMapping.ObserverType = n.Defaults.ObserverType
}
if currentMapping.LegacyQuantiles != nil &&
(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated. Please use quantiles in the summary_options hierarchy")
}
if currentMapping.LegacyBuckets != nil &&
(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated. Please use buckets in the histogram_options hierarchy")
}
if currentMapping.SummaryOptions != nil &&
currentMapping.LegacyQuantiles != nil &&
currentMapping.SummaryOptions.Quantiles != nil {
return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
}
if currentMapping.HistogramOptions != nil &&
currentMapping.LegacyBuckets != nil &&
currentMapping.HistogramOptions.Buckets != nil {
return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
}
if currentMapping.ObserverType == ObserverTypeHistogram {
if currentMapping.SummaryOptions != nil {
return fmt.Errorf("cannot use histogram observer and summary options at the same time")
}
if currentMapping.HistogramOptions == nil {
currentMapping.HistogramOptions = &HistogramOptions{}
}
if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
}
if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
}
}
if currentMapping.ObserverType == ObserverTypeSummary {
if currentMapping.HistogramOptions != nil {
return fmt.Errorf("cannot use summary observer and histogram options at the same time")
}
if currentMapping.SummaryOptions == nil {
currentMapping.SummaryOptions = &SummaryOptions{}
}
if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
}
if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
}
if currentMapping.SummaryOptions.MaxAge == 0 {
currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
}
if currentMapping.SummaryOptions.AgeBuckets == 0 {
currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
}
if currentMapping.SummaryOptions.BufCap == 0 {
currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
}
}
if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
currentMapping.Ttl = n.Defaults.Ttl
}
}
m.mutex.Lock()
defer m.mutex.Unlock()
if m.Logger == nil {
m.Logger = log.NewNopLogger()
}
m.Defaults = n.Defaults
m.Mappings = n.Mappings
// Reset the cache since this function can be used to reload config
if m.cache != nil {
m.cache.Reset()
}
if n.doFSM {
var mappings []string
for _, mapping := range n.Mappings {
if mapping.MatchType == MatchTypeGlob {
mappings = append(mappings, mapping.Match)
}
}
n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)
m.FSM = n.FSM
m.doRegex = n.doRegex
}
m.doFSM = n.doFSM
if m.MappingsCount != nil {
m.MappingsCount.Set(float64(len(n.Mappings)))
}
return nil
}
func (m *MetricMapper) InitFromFile(fileName string) error {
mappingStr, err := os.ReadFile(fileName)
if err != nil {
return err
}
return m.InitFromYAMLString(string(mappingStr))
}
// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
// This cache MUST be thread-safe!
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.cache = cache
}
func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
m.mutex.RLock()
defer m.mutex.RUnlock()
// only use a cache if one is present
if m.cache != nil {
result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
if cached {
r := result.(MetricMapperCacheResult)
return r.Mapping, r.Labels, r.Matched
}
}
// glob matching
if m.doFSM {
finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
if finalState != nil && finalState.Result != nil {
v := finalState.Result.(*MetricMapping)
result := copyMetricMapping(v)
result.Name = result.nameFormatter.Format(captures)
labels := prometheus.Labels{}
for index, formatter := range result.labelFormatters {
labels[result.labelKeys[index]] = formatter.Format(captures)
}
r := MetricMapperCacheResult{
Mapping: result,
Matched: true,
Labels: labels,
}
// add match to cache
if m.cache != nil {
m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
}
return result, labels, true
} else if !m.doRegex {
// if there's no regex match type, return immediately
// Add miss to cache
if m.cache != nil {
m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
}
return nil, nil, false
}
}
// regex matching
for _, mapping := range m.Mappings {
// if a rule don't have regex matching type, the regex field is unset
if mapping.regex == nil {
continue
}
matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
if len(matches) == 0 {
continue
}
mapping.Name = string(mapping.regex.ExpandString(
[]byte{},
mapping.Name,
statsdMetric,
matches,
))
if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
continue
}
labels := prometheus.Labels{}
for label, valueExpr := range mapping.Labels {
value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
labels[label] = string(value)
}
r := MetricMapperCacheResult{
Mapping: &mapping,
Matched: true,
Labels: labels,
}
// Add Match to cache
if m.cache != nil {
m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
}
return &mapping, labels, true
}
// Add Miss to cache
if m.cache != nil {
m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
}
return nil, nil, false
}
// make a shallow copy so that we do not overwrite name
// as multiple names can be matched by same mapping
func copyMetricMapping(in *MetricMapping) *MetricMapping {
out := *in
return &out
}
@@ -0,0 +1,74 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"github.com/prometheus/client_golang/prometheus"
)
type CacheMetrics struct {
CacheLength prometheus.Gauge
CacheGetsTotal prometheus.Counter
CacheHitsTotal prometheus.Counter
}
func NewCacheMetrics(reg prometheus.Registerer) *CacheMetrics {
var m CacheMetrics
m.CacheLength = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "statsd_metric_mapper_cache_length",
Help: "The count of unique metrics currently cached.",
},
)
m.CacheGetsTotal = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_metric_mapper_cache_gets_total",
Help: "The count of total metric cache gets.",
},
)
m.CacheHitsTotal = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_metric_mapper_cache_hits_total",
Help: "The count of total metric cache hits.",
},
)
if reg != nil {
reg.MustRegister(m.CacheLength)
reg.MustRegister(m.CacheGetsTotal)
reg.MustRegister(m.CacheHitsTotal)
}
return &m
}
type MetricMapperCacheResult struct {
Mapping *MetricMapping
Matched bool
Labels prometheus.Labels
}
// MetricMapperCache MUST be thread-safe and should be instrumented with CacheMetrics
type MetricMapperCache interface {
// Get a cached result
Get(metricKey string) (interface{}, bool)
// Add a statsd MetricMapperResult to the cache
Add(metricKey string, result interface{}) // Add an item to the cache
// Reset clears the cache for config reloads
Reset()
}
func formatKey(metricString string, metricType MetricType) string {
return string(metricType) + "." + metricString
}
@@ -0,0 +1,72 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "time"
type mapperConfigDefaults struct {
ObserverType ObserverType `yaml:"observer_type"`
MatchType MatchType `yaml:"match_type"`
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
Ttl time.Duration `yaml:"ttl"`
SummaryOptions SummaryOptions `yaml:"summary_options"`
HistogramOptions HistogramOptions `yaml:"histogram_options"`
}
// mapperConfigDefaultsAlias is used to unmarshal the yaml config into mapperConfigDefaults and allows deprecated fields
type mapperConfigDefaultsAlias struct {
ObserverType ObserverType `yaml:"observer_type"`
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs
Buckets []float64 `yaml:"buckets"` // DEPRECATED - field only present to preserve backwards compatibility in configs
Quantiles []metricObjective `yaml:"quantiles"` // DEPRECATED - field only present to preserve backwards compatibility in configs
MatchType MatchType `yaml:"match_type"`
GlobDisableOrdering bool `yaml:"glob_disable_ordering"`
Ttl time.Duration `yaml:"ttl"`
SummaryOptions SummaryOptions `yaml:"summary_options"`
HistogramOptions HistogramOptions `yaml:"histogram_options"`
}
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
// observer_type will override timer_type
func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error {
var tmp mapperConfigDefaultsAlias
if err := unmarshal(&tmp); err != nil {
return err
}
// Copy defaults
d.ObserverType = tmp.ObserverType
d.MatchType = tmp.MatchType
d.GlobDisableOrdering = tmp.GlobDisableOrdering
d.Ttl = tmp.Ttl
d.SummaryOptions = tmp.SummaryOptions
d.HistogramOptions = tmp.HistogramOptions
// Use deprecated TimerType if necessary
if tmp.ObserverType == "" {
d.ObserverType = tmp.TimerType
}
// Use deprecated quantiles if necessary
if len(tmp.SummaryOptions.Quantiles) == 0 && len(tmp.Quantiles) > 0 {
d.SummaryOptions = SummaryOptions{Quantiles: tmp.Quantiles}
}
// Use deprecated buckets if necessary
if len(tmp.HistogramOptions.Buckets) == 0 && len(tmp.Buckets) > 0 {
d.HistogramOptions = HistogramOptions{Buckets: tmp.Buckets}
}
return nil
}
@@ -0,0 +1,76 @@
// Copyright 2020 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either xpress or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import (
"regexp"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)
type MetricMapping struct {
Match string `yaml:"match"`
Name string `yaml:"name"`
nameFormatter *fsm.TemplateFormatter
regex *regexp.Regexp
Labels prometheus.Labels `yaml:"labels"`
labelKeys []string
labelFormatters []*fsm.TemplateFormatter
ObserverType ObserverType `yaml:"observer_type"`
TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty
LegacyBuckets []float64 `yaml:"buckets"`
LegacyQuantiles []metricObjective `yaml:"quantiles"`
MatchType MatchType `yaml:"match_type"`
HelpText string `yaml:"help"`
Action ActionType `yaml:"action"`
MatchMetricType MetricType `yaml:"match_metric_type"`
Ttl time.Duration `yaml:"ttl"`
SummaryOptions *SummaryOptions `yaml:"summary_options"`
HistogramOptions *HistogramOptions `yaml:"histogram_options"`
}
// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys
// observer_type will override timer_type
func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error {
type MetricMappingAlias MetricMapping
var tmp MetricMappingAlias
if err := unmarshal(&tmp); err != nil {
return err
}
// Copy defaults
m.Match = tmp.Match
m.Name = tmp.Name
m.Labels = tmp.Labels
m.ObserverType = tmp.ObserverType
m.LegacyBuckets = tmp.LegacyBuckets
m.LegacyQuantiles = tmp.LegacyQuantiles
m.MatchType = tmp.MatchType
m.HelpText = tmp.HelpText
m.Action = tmp.Action
m.MatchMetricType = tmp.MatchMetricType
m.Ttl = tmp.Ttl
m.SummaryOptions = tmp.SummaryOptions
m.HistogramOptions = tmp.HistogramOptions
// Use deprecated TimerType if necessary
if tmp.ObserverType == "" {
m.ObserverType = tmp.TimerType
}
return nil
}
@@ -0,0 +1,41 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "fmt"
type MatchType string
const (
MatchTypeGlob MatchType = "glob"
MatchTypeRegex MatchType = "regex"
MatchTypeDefault MatchType = ""
)
func (t *MatchType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch MatchType(v) {
case MatchTypeRegex:
*t = MatchTypeRegex
case MatchTypeGlob, MatchTypeDefault:
*t = MatchTypeGlob
default:
return fmt.Errorf("invalid match type %q", v)
}
return nil
}
@@ -0,0 +1,46 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "fmt"
type MetricType string
const (
MetricTypeCounter MetricType = "counter"
MetricTypeGauge MetricType = "gauge"
MetricTypeObserver MetricType = "observer"
MetricTypeTimer MetricType = "timer" // DEPRECATED
)
func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch MetricType(v) {
case MetricTypeCounter:
*m = MetricTypeCounter
case MetricTypeGauge:
*m = MetricTypeGauge
case MetricTypeObserver:
*m = MetricTypeObserver
case MetricTypeTimer:
*m = MetricTypeObserver
default:
return fmt.Errorf("invalid metric type '%s'", v)
}
return nil
}
@@ -0,0 +1,41 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mapper
import "fmt"
type ObserverType string
const (
ObserverTypeHistogram ObserverType = "histogram"
ObserverTypeSummary ObserverType = "summary"
ObserverTypeDefault ObserverType = ""
)
func (t *ObserverType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch ObserverType(v) {
case ObserverTypeHistogram:
*t = ObserverTypeHistogram
case ObserverTypeSummary, ObserverTypeDefault:
*t = ObserverTypeSummary
default:
return fmt.Errorf("invalid observer type '%s'", v)
}
return nil
}