dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implement telegraf's own full metric type

Browse Source 
main reasons behind this:
- make adding/removing tags cheap
- make adding/removing fields cheap
- make parsing cheaper
- make parse -> decorate -> write out bytes metric flow much faster

Refactor serializer to use byte buffer
This commit is contained in:
Cameron Sparr
2016-11-22 12:51:57 +00:00
parent 332f678afb
commit db7a4b24b6
40 changed files with 1376 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/parsers/graphite/parser.go
View File

@@ -12,6 +12,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
// Minimum and maximum supported dates for timestamps.
@@ -216,7 +217,7 @@ func (p *GraphiteParser) ParseLine(line string) (telegraf.Metric, error) {
}
}
return telegraf.NewMetric(measurement, tags, fieldValues, timestamp)
return metric.New(measurement, tags, fieldValues, timestamp)
}
// ApplyTemplate extracts the template fields from the given line and

27
plugins/parsers/graphite/parser_test.go
View File

@@ -7,6 +7,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/stretchr/testify/assert"
)
@@ -369,7 +370,7 @@ func TestFilterMatchDefault(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("miss.servers.localhost.cpu_load",
exp, err := metric.New("miss.servers.localhost.cpu_load",
map[string]string{},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -387,7 +388,7 @@ func TestFilterMatchMultipleMeasurement(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu.cpu_load.10",
exp, err := metric.New("cpu.cpu_load.10",
map[string]string{"host": "localhost"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -406,7 +407,7 @@ func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) {
)
assert.NoError(t, err)
exp, err := telegraf.NewMetric("cpu_cpu_load_10",
exp, err := metric.New("cpu_cpu_load_10",
map[string]string{"host": "localhost"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -424,7 +425,7 @@ func TestFilterMatchSingle(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -441,7 +442,7 @@ func TestParseNoMatch(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("servers.localhost.memory.VmallocChunk",
exp, err := metric.New("servers.localhost.memory.VmallocChunk",
map[string]string{},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -459,7 +460,7 @@ func TestFilterMatchWildcard(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -479,7 +480,7 @@ func TestFilterMatchExactBeforeWildcard(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -504,7 +505,7 @@ func TestFilterMatchMostLongestFilter(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost", "resource": "cpu"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -528,7 +529,7 @@ func TestFilterMatchMultipleWildcards(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "server01"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -550,7 +551,7 @@ func TestParseDefaultTags(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -571,7 +572,7 @@ func TestParseDefaultTemplateTags(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -592,7 +593,7 @@ func TestParseDefaultTemplateTagsOverridGlobal(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))
@@ -615,7 +616,7 @@ func TestParseTemplateWhitespace(t *testing.T) {
t.Fatalf("unexpected error creating parser, got %v", err)
}
exp, err := telegraf.NewMetric("cpu_load",
exp, err := metric.New("cpu_load",
map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
map[string]interface{}{"value": float64(11)},
time.Unix(1435077219, 0))

21
plugins/parsers/influx/parser.go
View File

@@ -6,8 +6,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/influxdb/models"
"github.com/influxdata/telegraf/metric"
)
// InfluxParser is an object for Parsing incoming metrics.
@@ -19,18 +18,16 @@ type InfluxParser struct {
func (p *InfluxParser) ParseWithDefaultTime(buf []byte, t time.Time) ([]telegraf.Metric, error) {
// parse even if the buffer begins with a newline
buf = bytes.TrimPrefix(buf, []byte("\n"))
points, err := models.ParsePointsWithPrecision(buf, t, "n")
metrics := make([]telegraf.Metric, len(points))
for i, point := range points {
for k, v := range p.DefaultTags {
// only set the default tag if it doesn't already exist:
if tmp := point.Tags().GetString(k); tmp == "" {
point.AddTag(k, v)
metrics, err := metric.ParseWithDefaultTime(buf, t)
if len(p.DefaultTags) > 0 {
for _, m := range metrics {
for k, v := range p.DefaultTags {
// only set the default tag if it doesn't already exist:
if !m.HasTag(k) {
m.AddTag(k, v)
}
}
}
// Ignore error here because it's impossible that a model.Point
// wouldn't parse into client.Point properly
metrics[i] = telegraf.NewMetricFromPoint(point)
}
return metrics, err
}

24
plugins/parsers/influx/parser_test.go
View File

@@ -14,15 +14,14 @@ var (
ms []telegraf.Metric
writer = ioutil.Discard
metrics500 []byte
exptime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).UnixNano()
)
var exptime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
const (
validInflux = "cpu_load_short,cpu=cpu0 value=10 1257894000000000000"
validInflux = "cpu_load_short,cpu=cpu0 value=10 1257894000000000000\n"
validInfluxNewline = "\ncpu_load_short,cpu=cpu0 value=10 1257894000000000000\n"
invalidInflux = "I don't think this is line protocol"
invalidInflux2 = "{\"a\": 5, \"b\": {\"c\": 6}}"
invalidInflux = "I don't think this is line protocol\n"
invalidInflux2 = "{\"a\": 5, \"b\": {\"c\": 6}}\n"
)
const influxMulti = `
@@ -57,7 +56,7 @@ func TestParseValidInflux(t *testing.T) {
assert.Equal(t, map[string]string{
"cpu": "cpu0",
}, metrics[0].Tags())
assert.Equal(t, exptime, metrics[0].Time())
assert.Equal(t, exptime, metrics[0].Time().UnixNano())
metrics, err = parser.Parse([]byte(validInfluxNewline))
assert.NoError(t, err)
@@ -69,7 +68,7 @@ func TestParseValidInflux(t *testing.T) {
assert.Equal(t, map[string]string{
"cpu": "cpu0",
}, metrics[0].Tags())
assert.Equal(t, exptime, metrics[0].Time())
assert.Equal(t, exptime, metrics[0].Time().UnixNano())
}
func TestParseLineValidInflux(t *testing.T) {
@@ -84,7 +83,7 @@ func TestParseLineValidInflux(t *testing.T) {
assert.Equal(t, map[string]string{
"cpu": "cpu0",
}, metric.Tags())
assert.Equal(t, exptime, metric.Time())
assert.Equal(t, exptime, metric.Time().UnixNano())
metric, err = parser.ParseLine(validInfluxNewline)
assert.NoError(t, err)
@@ -95,7 +94,7 @@ func TestParseLineValidInflux(t *testing.T) {
assert.Equal(t, map[string]string{
"cpu": "cpu0",
}, metric.Tags())
assert.Equal(t, exptime, metric.Time())
assert.Equal(t, exptime, metric.Time().UnixNano())
}
func TestParseMultipleValid(t *testing.T) {
@@ -229,11 +228,8 @@ func BenchmarkParseAddTagWrite(b *testing.B) {
panic("500 metrics not parsed!!")
}
for _, tmp := range ms {
tags := tmp.Tags()
tags["host"] = "localhost"
tmp, _ = telegraf.NewMetric(tmp.Name(), tags, tmp.Fields(), tmp.Time())
writer.Write([]byte(tmp.String()))
writer.Write([]byte{'\n'})
tmp.AddTag("host", "localhost")
writer.Write(tmp.Serialize())
}
}
}

3
plugins/parsers/json/parser.go
View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
type JSONParser struct {
@@ -57,7 +58,7 @@ func (p *JSONParser) parseObject(metrics []telegraf.Metric, jsonOut map[string]i
return nil, err
}
metric, err := telegraf.NewMetric(p.MetricName, tags, f.Fields, time.Now().UTC())
metric, err := metric.New(p.MetricName, tags, f.Fields, time.Now().UTC())
if err != nil {
return nil, err

3
plugins/parsers/nagios/parser.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
type NagiosParser struct {
@@ -90,7 +91,7 @@ func (p *NagiosParser) Parse(buf []byte) ([]telegraf.Metric, error) {
fields["max"] = perf[0][7]
}
// Create metric
metric, err := telegraf.NewMetric(fieldName, tags, fields, time.Now().UTC())
metric, err := metric.New(fieldName, tags, fields, time.Now().UTC())
if err != nil {
return nil, err
}

3
plugins/parsers/value/parser.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
type ValueParser struct {
@@ -46,7 +47,7 @@ func (v *ValueParser) Parse(buf []byte) ([]telegraf.Metric, error) {
}
fields := map[string]interface{}{"value": value}
metric, err := telegraf.NewMetric(v.MetricName, v.DefaultTags,
metric, err := metric.New(v.MetricName, v.DefaultTags,
fields, time.Now().UTC())
if err != nil {
return nil, err