graphite parser, handle multiple templates empty filter

Previously, the graphite parser would simply overwrite any template that
had an identical filter to a previous template. This included the empty
filter.

Now we will still overwrite, but first we will sort to make sure that
the most "specific" template always matches.

closes #1731
This commit is contained in:
Cameron Sparr 2016-10-10 15:35:15 +01:00
parent 2bbb6aa6f2
commit e96f7a9b12
4 changed files with 114 additions and 20 deletions

View File

@ -58,6 +58,7 @@ continue sending logs to /var/log/telegraf/telegraf.log.
- [#1854](https://github.com/influxdata/telegraf/pull/1853): SQL Server waitstats truncation bug. - [#1854](https://github.com/influxdata/telegraf/pull/1853): SQL Server waitstats truncation bug.
- [#1810](https://github.com/influxdata/telegraf/issues/1810): Fix logparser common log format: numbers in ident. - [#1810](https://github.com/influxdata/telegraf/issues/1810): Fix logparser common log format: numbers in ident.
- [#1793](https://github.com/influxdata/telegraf/pull/1793): Fix JSON Serialization in OpenTSDB output. - [#1793](https://github.com/influxdata/telegraf/pull/1793): Fix JSON Serialization in OpenTSDB output.
- [#1731](https://github.com/influxdata/telegraf/issues/1731): Fix Graphite template ordering, use most specific.
## v1.0.1 [2016-09-26] ## v1.0.1 [2016-09-26]

View File

@ -232,6 +232,16 @@ us.west.cpu.load 100
=> cpu.load,region=us.west value=100 => cpu.load,region=us.west value=100
``` ```
Multiple templates can also be specified, but these should be differentiated
using _filters_ (see below for more details)
```toml
templates = [
"*.*.* region.region.measurement", # <- all 3-part measurements will match this one.
"*.*.*.* region.region.host.measurement", # <- all 4-part measurements will match this one.
]
```
#### Field Templates: #### Field Templates:
The field keyword tells Telegraf to give the metric that field name. The field keyword tells Telegraf to give the metric that field name.

View File

@ -57,38 +57,34 @@ func NewGraphiteParser(
defaultTemplate, _ := NewTemplate("measurement*", nil, p.Separator) defaultTemplate, _ := NewTemplate("measurement*", nil, p.Separator)
matcher.AddDefaultTemplate(defaultTemplate) matcher.AddDefaultTemplate(defaultTemplate)
tmplts := parsedTemplates{}
for _, pattern := range p.Templates { for _, pattern := range p.Templates {
template := pattern tmplt := parsedTemplate{}
filter := "" tmplt.template = pattern
// Format is [filter] <template> [tag1=value1,tag2=value2] // Format is [filter] <template> [tag1=value1,tag2=value2]
parts := strings.Fields(pattern) parts := strings.Fields(pattern)
if len(parts) < 1 { if len(parts) < 1 {
continue continue
} else if len(parts) >= 2 { } else if len(parts) >= 2 {
if strings.Contains(parts[1], "=") { if strings.Contains(parts[1], "=") {
template = parts[0] tmplt.template = parts[0]
tmplt.tagstring = parts[1]
} else { } else {
filter = parts[0] tmplt.filter = parts[0]
template = parts[1] tmplt.template = parts[1]
if len(parts) > 2 {
tmplt.tagstring = parts[2]
}
} }
} }
tmplts = append(tmplts, tmplt)
}
// Parse out the default tags specific to this template sort.Sort(tmplts)
tags := map[string]string{} for _, tmplt := range tmplts {
if strings.Contains(parts[len(parts)-1], "=") { if err := p.addToMatcher(tmplt); err != nil {
tagStrs := strings.Split(parts[len(parts)-1], ",") return nil, err
for _, kv := range tagStrs {
parts := strings.Split(kv, "=")
tags[parts[0]] = parts[1]
}
} }
tmpl, err1 := NewTemplate(template, tags, p.Separator)
if err1 != nil {
err = err1
break
}
matcher.Add(filter, tmpl)
} }
if err != nil { if err != nil {
@ -98,6 +94,24 @@ func NewGraphiteParser(
} }
} }
func (p *GraphiteParser) addToMatcher(tmplt parsedTemplate) error {
// Parse out the default tags specific to this template
tags := map[string]string{}
if tmplt.tagstring != "" {
for _, kv := range strings.Split(tmplt.tagstring, ",") {
parts := strings.Split(kv, "=")
tags[parts[0]] = parts[1]
}
}
tmpl, err := NewTemplate(tmplt.template, tags, p.Separator)
if err != nil {
return err
}
p.matcher.Add(tmplt.filter, tmpl)
return nil
}
func (p *GraphiteParser) Parse(buf []byte) ([]telegraf.Metric, error) { func (p *GraphiteParser) Parse(buf []byte) ([]telegraf.Metric, error) {
// parse even if the buffer begins with a newline // parse even if the buffer begins with a newline
buf = bytes.TrimPrefix(buf, []byte("\n")) buf = bytes.TrimPrefix(buf, []byte("\n"))
@ -465,3 +479,30 @@ func (n *nodes) Less(j, k int) bool {
func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] } func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] }
func (n *nodes) Len() int { return len(*n) } func (n *nodes) Len() int { return len(*n) }
type parsedTemplate struct {
template string
filter string
tagstring string
}
type parsedTemplates []parsedTemplate
func (e parsedTemplates) Less(j, k int) bool {
if len(e[j].filter) == 0 && len(e[k].filter) == 0 {
nj := len(strings.Split(e[j].template, "."))
nk := len(strings.Split(e[k].template, "."))
return nj < nk
}
if len(e[j].filter) == 0 {
return true
}
if len(e[k].filter) == 0 {
return false
}
nj := len(strings.Split(e[j].template, "."))
nk := len(strings.Split(e[k].template, "."))
return nj < nk
}
func (e parsedTemplates) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
func (e parsedTemplates) Len() int { return len(e) }

View File

@ -747,6 +747,48 @@ func TestApplyTemplateGreedyField(t *testing.T) {
} }
} }
func TestApplyTemplateOverSpecific(t *testing.T) {
p, err := NewGraphiteParser(
".",
[]string{
"measurement.host.metric.metric.metric",
},
nil,
)
assert.NoError(t, err)
measurement, tags, _, err := p.ApplyTemplate("net.server001.a.b 2")
assert.Equal(t, "net", measurement)
assert.Equal(t,
map[string]string{"host": "server001", "metric": "a.b"},
tags)
}
func TestApplyTemplateMostSpecificTemplate(t *testing.T) {
p, err := NewGraphiteParser(
".",
[]string{
"measurement.host.metric",
"measurement.host.metric.metric.metric",
"measurement.host.metric.metric",
},
nil,
)
assert.NoError(t, err)
measurement, tags, _, err := p.ApplyTemplate("net.server001.a.b.c 2")
assert.Equal(t, "net", measurement)
assert.Equal(t,
map[string]string{"host": "server001", "metric": "a.b.c"},
tags)
measurement, tags, _, err = p.ApplyTemplate("net.server001.a.b 2")
assert.Equal(t, "net", measurement)
assert.Equal(t,
map[string]string{"host": "server001", "metric": "a.b"},
tags)
}
// Test Helpers // Test Helpers
func errstr(err error) string { func errstr(err error) string {
if err != nil { if err != nil {