510 lines
13 KiB
Go
510 lines
13 KiB
Go
package graphite
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/influxdata/telegraf"
|
|
"github.com/influxdata/telegraf/metric"
|
|
)
|
|
|
|
// Minimum and maximum supported dates for timestamps.
|
|
var (
|
|
MinDate = time.Date(1901, 12, 13, 0, 0, 0, 0, time.UTC)
|
|
MaxDate = time.Date(2038, 1, 19, 0, 0, 0, 0, time.UTC)
|
|
)
|
|
|
|
// Parser encapsulates a Graphite Parser.
|
|
type GraphiteParser struct {
|
|
Separator string
|
|
Templates []string
|
|
DefaultTags map[string]string
|
|
|
|
matcher *matcher
|
|
}
|
|
|
|
func (p *GraphiteParser) SetDefaultTags(tags map[string]string) {
|
|
p.DefaultTags = tags
|
|
}
|
|
|
|
func NewGraphiteParser(
|
|
separator string,
|
|
templates []string,
|
|
defaultTags map[string]string,
|
|
) (*GraphiteParser, error) {
|
|
var err error
|
|
|
|
if separator == "" {
|
|
separator = DefaultSeparator
|
|
}
|
|
p := &GraphiteParser{
|
|
Separator: separator,
|
|
Templates: templates,
|
|
}
|
|
|
|
if defaultTags != nil {
|
|
p.DefaultTags = defaultTags
|
|
}
|
|
|
|
matcher := newMatcher()
|
|
p.matcher = matcher
|
|
defaultTemplate, _ := NewTemplate("measurement*", nil, p.Separator)
|
|
matcher.AddDefaultTemplate(defaultTemplate)
|
|
|
|
tmplts := parsedTemplates{}
|
|
for _, pattern := range p.Templates {
|
|
tmplt := parsedTemplate{}
|
|
tmplt.template = pattern
|
|
// Format is [filter] <template> [tag1=value1,tag2=value2]
|
|
parts := strings.Fields(pattern)
|
|
if len(parts) < 1 {
|
|
continue
|
|
} else if len(parts) >= 2 {
|
|
if strings.Contains(parts[1], "=") {
|
|
tmplt.template = parts[0]
|
|
tmplt.tagstring = parts[1]
|
|
} else {
|
|
tmplt.filter = parts[0]
|
|
tmplt.template = parts[1]
|
|
if len(parts) > 2 {
|
|
tmplt.tagstring = parts[2]
|
|
}
|
|
}
|
|
}
|
|
tmplts = append(tmplts, tmplt)
|
|
}
|
|
|
|
sort.Sort(tmplts)
|
|
for _, tmplt := range tmplts {
|
|
if err := p.addToMatcher(tmplt); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
return p, fmt.Errorf("exec input parser config is error: %s ", err.Error())
|
|
} else {
|
|
return p, nil
|
|
}
|
|
}
|
|
|
|
func (p *GraphiteParser) addToMatcher(tmplt parsedTemplate) error {
|
|
// Parse out the default tags specific to this template
|
|
tags := map[string]string{}
|
|
if tmplt.tagstring != "" {
|
|
for _, kv := range strings.Split(tmplt.tagstring, ",") {
|
|
parts := strings.Split(kv, "=")
|
|
tags[parts[0]] = parts[1]
|
|
}
|
|
}
|
|
|
|
tmpl, err := NewTemplate(tmplt.template, tags, p.Separator)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.matcher.Add(tmplt.filter, tmpl)
|
|
return nil
|
|
}
|
|
|
|
func (p *GraphiteParser) Parse(buf []byte) ([]telegraf.Metric, error) {
|
|
// parse even if the buffer begins with a newline
|
|
buf = bytes.TrimPrefix(buf, []byte("\n"))
|
|
// add newline to end if not exists:
|
|
if len(buf) > 0 && !bytes.HasSuffix(buf, []byte("\n")) {
|
|
buf = append(buf, []byte("\n")...)
|
|
}
|
|
|
|
metrics := make([]telegraf.Metric, 0)
|
|
|
|
var errStr string
|
|
buffer := bytes.NewBuffer(buf)
|
|
reader := bufio.NewReader(buffer)
|
|
for {
|
|
// Read up to the next newline.
|
|
buf, err := reader.ReadBytes('\n')
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil && err != io.EOF {
|
|
return metrics, err
|
|
}
|
|
|
|
// Trim the buffer, even though there should be no padding
|
|
line := strings.TrimSpace(string(buf))
|
|
metric, err := p.ParseLine(line)
|
|
|
|
if err == nil {
|
|
metrics = append(metrics, metric)
|
|
} else {
|
|
errStr += err.Error() + "\n"
|
|
}
|
|
}
|
|
|
|
if errStr != "" {
|
|
return metrics, fmt.Errorf(strings.TrimSpace(errStr))
|
|
}
|
|
return metrics, nil
|
|
}
|
|
|
|
// Parse performs Graphite parsing of a single line.
|
|
func (p *GraphiteParser) ParseLine(line string) (telegraf.Metric, error) {
|
|
// Break into 3 fields (name, value, timestamp).
|
|
fields := strings.Fields(line)
|
|
if len(fields) != 2 && len(fields) != 3 {
|
|
return nil, fmt.Errorf("received %q which doesn't have required fields", line)
|
|
}
|
|
|
|
// decode the name and tags
|
|
template := p.matcher.Match(fields[0])
|
|
measurement, tags, field, err := template.Apply(fields[0])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Could not extract measurement, use the raw value
|
|
if measurement == "" {
|
|
measurement = fields[0]
|
|
}
|
|
|
|
// Parse value.
|
|
v, err := strconv.ParseFloat(fields[1], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf(`field "%s" value: %s`, fields[0], err)
|
|
}
|
|
|
|
if math.IsNaN(v) || math.IsInf(v, 0) {
|
|
return nil, &UnsupposedValueError{Field: fields[0], Value: v}
|
|
}
|
|
|
|
fieldValues := map[string]interface{}{}
|
|
if field != "" {
|
|
fieldValues[field] = v
|
|
} else {
|
|
fieldValues["value"] = v
|
|
}
|
|
|
|
// If no 3rd field, use now as timestamp
|
|
timestamp := time.Now().UTC()
|
|
|
|
if len(fields) == 3 {
|
|
// Parse timestamp.
|
|
unixTime, err := strconv.ParseFloat(fields[2], 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf(`field "%s" time: %s`, fields[0], err)
|
|
}
|
|
|
|
// -1 is a special value that gets converted to current UTC time
|
|
// See https://github.com/graphite-project/carbon/issues/54
|
|
if unixTime != float64(-1) {
|
|
// Check if we have fractional seconds
|
|
timestamp = time.Unix(int64(unixTime), int64((unixTime-math.Floor(unixTime))*float64(time.Second)))
|
|
if timestamp.Before(MinDate) || timestamp.After(MaxDate) {
|
|
return nil, fmt.Errorf("timestamp out of range")
|
|
}
|
|
}
|
|
}
|
|
// Set the default tags on the point if they are not already set
|
|
for k, v := range p.DefaultTags {
|
|
if _, ok := tags[k]; !ok {
|
|
tags[k] = v
|
|
}
|
|
}
|
|
|
|
return metric.New(measurement, tags, fieldValues, timestamp)
|
|
}
|
|
|
|
// ApplyTemplate extracts the template fields from the given line and
|
|
// returns the measurement name and tags.
|
|
func (p *GraphiteParser) ApplyTemplate(line string) (string, map[string]string, string, error) {
|
|
// Break line into fields (name, value, timestamp), only name is used
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 0 {
|
|
return "", make(map[string]string), "", nil
|
|
}
|
|
// decode the name and tags
|
|
template := p.matcher.Match(fields[0])
|
|
name, tags, field, err := template.Apply(fields[0])
|
|
|
|
// Set the default tags on the point if they are not already set
|
|
for k, v := range p.DefaultTags {
|
|
if _, ok := tags[k]; !ok {
|
|
tags[k] = v
|
|
}
|
|
}
|
|
|
|
return name, tags, field, err
|
|
}
|
|
|
|
// template represents a pattern and tags to map a graphite metric string to a influxdb Point
|
|
type template struct {
|
|
tags []string
|
|
defaultTags map[string]string
|
|
greedyField bool
|
|
greedyMeasurement bool
|
|
separator string
|
|
}
|
|
|
|
// NewTemplate returns a new template ensuring it has a measurement
|
|
// specified.
|
|
func NewTemplate(pattern string, defaultTags map[string]string, separator string) (*template, error) {
|
|
tags := strings.Split(pattern, ".")
|
|
hasMeasurement := false
|
|
template := &template{tags: tags, defaultTags: defaultTags, separator: separator}
|
|
|
|
for _, tag := range tags {
|
|
if strings.HasPrefix(tag, "measurement") {
|
|
hasMeasurement = true
|
|
}
|
|
if tag == "measurement*" {
|
|
template.greedyMeasurement = true
|
|
} else if tag == "field*" {
|
|
template.greedyField = true
|
|
}
|
|
}
|
|
|
|
if !hasMeasurement {
|
|
return nil, fmt.Errorf("no measurement specified for template. %q", pattern)
|
|
}
|
|
|
|
return template, nil
|
|
}
|
|
|
|
// Apply extracts the template fields from the given line and returns the measurement
|
|
// name and tags
|
|
func (t *template) Apply(line string) (string, map[string]string, string, error) {
|
|
fields := strings.Split(line, ".")
|
|
var (
|
|
measurement []string
|
|
tags = make(map[string][]string)
|
|
field []string
|
|
)
|
|
|
|
// Set any default tags
|
|
for k, v := range t.defaultTags {
|
|
tags[k] = append(tags[k], v)
|
|
}
|
|
|
|
// See if an invalid combination has been specified in the template:
|
|
for _, tag := range t.tags {
|
|
if tag == "measurement*" {
|
|
t.greedyMeasurement = true
|
|
} else if tag == "field*" {
|
|
t.greedyField = true
|
|
}
|
|
}
|
|
if t.greedyField && t.greedyMeasurement {
|
|
return "", nil, "",
|
|
fmt.Errorf("either 'field*' or 'measurement*' can be used in each "+
|
|
"template (but not both together): %q",
|
|
strings.Join(t.tags, t.separator))
|
|
}
|
|
|
|
for i, tag := range t.tags {
|
|
if i >= len(fields) {
|
|
continue
|
|
}
|
|
if tag == "" {
|
|
continue
|
|
}
|
|
|
|
switch tag {
|
|
case "measurement":
|
|
measurement = append(measurement, fields[i])
|
|
case "field":
|
|
field = append(field, fields[i])
|
|
case "field*":
|
|
field = append(field, fields[i:]...)
|
|
break
|
|
case "measurement*":
|
|
measurement = append(measurement, fields[i:]...)
|
|
break
|
|
default:
|
|
tags[tag] = append(tags[tag], fields[i])
|
|
}
|
|
}
|
|
|
|
// Convert to map of strings.
|
|
outtags := make(map[string]string)
|
|
for k, values := range tags {
|
|
outtags[k] = strings.Join(values, t.separator)
|
|
}
|
|
|
|
return strings.Join(measurement, t.separator), outtags, strings.Join(field, t.separator), nil
|
|
}
|
|
|
|
// matcher determines which template should be applied to a given metric
|
|
// based on a filter tree.
|
|
type matcher struct {
|
|
root *node
|
|
defaultTemplate *template
|
|
}
|
|
|
|
func newMatcher() *matcher {
|
|
return &matcher{
|
|
root: &node{},
|
|
}
|
|
}
|
|
|
|
// Add inserts the template in the filter tree based the given filter
|
|
func (m *matcher) Add(filter string, template *template) {
|
|
if filter == "" {
|
|
m.AddDefaultTemplate(template)
|
|
return
|
|
}
|
|
m.root.Insert(filter, template)
|
|
}
|
|
|
|
func (m *matcher) AddDefaultTemplate(template *template) {
|
|
m.defaultTemplate = template
|
|
}
|
|
|
|
// Match returns the template that matches the given graphite line
|
|
func (m *matcher) Match(line string) *template {
|
|
tmpl := m.root.Search(line)
|
|
if tmpl != nil {
|
|
return tmpl
|
|
}
|
|
|
|
return m.defaultTemplate
|
|
}
|
|
|
|
// node is an item in a sorted k-ary tree. Each child is sorted by its value.
|
|
// The special value of "*", is always last.
|
|
type node struct {
|
|
value string
|
|
children nodes
|
|
template *template
|
|
}
|
|
|
|
func (n *node) insert(values []string, template *template) {
|
|
// Add the end, set the template
|
|
if len(values) == 0 {
|
|
n.template = template
|
|
return
|
|
}
|
|
|
|
// See if the the current element already exists in the tree. If so, insert the
|
|
// into that sub-tree
|
|
for _, v := range n.children {
|
|
if v.value == values[0] {
|
|
v.insert(values[1:], template)
|
|
return
|
|
}
|
|
}
|
|
|
|
// New element, add it to the tree and sort the children
|
|
newNode := &node{value: values[0]}
|
|
n.children = append(n.children, newNode)
|
|
sort.Sort(&n.children)
|
|
|
|
// Now insert the rest of the tree into the new element
|
|
newNode.insert(values[1:], template)
|
|
}
|
|
|
|
// Insert inserts the given string template into the tree. The filter string is separated
|
|
// on "." and each part is used as the path in the tree.
|
|
func (n *node) Insert(filter string, template *template) {
|
|
n.insert(strings.Split(filter, "."), template)
|
|
}
|
|
|
|
func (n *node) search(lineParts []string) *template {
|
|
// Nothing to search
|
|
if len(lineParts) == 0 || len(n.children) == 0 {
|
|
return n.template
|
|
}
|
|
|
|
// If last element is a wildcard, don't include in this search since it's sorted
|
|
// to the end but lexicographically it would not always be and sort.Search assumes
|
|
// the slice is sorted.
|
|
length := len(n.children)
|
|
if n.children[length-1].value == "*" {
|
|
length--
|
|
}
|
|
|
|
// Find the index of child with an exact match
|
|
i := sort.Search(length, func(i int) bool {
|
|
return n.children[i].value >= lineParts[0]
|
|
})
|
|
|
|
// Found an exact match, so search that child sub-tree
|
|
if i < len(n.children) && n.children[i].value == lineParts[0] {
|
|
return n.children[i].search(lineParts[1:])
|
|
}
|
|
// Not an exact match, see if we have a wildcard child to search
|
|
if n.children[len(n.children)-1].value == "*" {
|
|
return n.children[len(n.children)-1].search(lineParts[1:])
|
|
}
|
|
return n.template
|
|
}
|
|
|
|
func (n *node) Search(line string) *template {
|
|
return n.search(strings.Split(line, "."))
|
|
}
|
|
|
|
type nodes []*node
|
|
|
|
// Less returns a boolean indicating whether the filter at position j
|
|
// is less than the filter at position k. Filters are order by string
|
|
// comparison of each component parts. A wildcard value "*" is never
|
|
// less than a non-wildcard value.
|
|
//
|
|
// For example, the filters:
|
|
// "*.*"
|
|
// "servers.*"
|
|
// "servers.localhost"
|
|
// "*.localhost"
|
|
//
|
|
// Would be sorted as:
|
|
// "servers.localhost"
|
|
// "servers.*"
|
|
// "*.localhost"
|
|
// "*.*"
|
|
func (n *nodes) Less(j, k int) bool {
|
|
if (*n)[j].value == "*" && (*n)[k].value != "*" {
|
|
return false
|
|
}
|
|
|
|
if (*n)[j].value != "*" && (*n)[k].value == "*" {
|
|
return true
|
|
}
|
|
|
|
return (*n)[j].value < (*n)[k].value
|
|
}
|
|
|
|
func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] }
|
|
func (n *nodes) Len() int { return len(*n) }
|
|
|
|
type parsedTemplate struct {
|
|
template string
|
|
filter string
|
|
tagstring string
|
|
}
|
|
type parsedTemplates []parsedTemplate
|
|
|
|
func (e parsedTemplates) Less(j, k int) bool {
|
|
if len(e[j].filter) == 0 && len(e[k].filter) == 0 {
|
|
nj := len(strings.Split(e[j].template, "."))
|
|
nk := len(strings.Split(e[k].template, "."))
|
|
return nj < nk
|
|
}
|
|
if len(e[j].filter) == 0 {
|
|
return true
|
|
}
|
|
if len(e[k].filter) == 0 {
|
|
return false
|
|
}
|
|
|
|
nj := len(strings.Split(e[j].template, "."))
|
|
nk := len(strings.Split(e[k].template, "."))
|
|
return nj < nk
|
|
}
|
|
func (e parsedTemplates) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
|
|
func (e parsedTemplates) Len() int { return len(e) }
|