dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implement telegraf's own full metric type

Browse Source 
main reasons behind this:
- make adding/removing tags cheap
- make adding/removing fields cheap
- make parsing cheaper
- make parse -> decorate -> write out bytes metric flow much faster

Refactor serializer to use byte buffer
This commit is contained in:
Cameron Sparr
2016-11-22 12:51:57 +00:00
parent 332f678afb
commit db7a4b24b6
40 changed files with 1376 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
plugins/serializers/graphite/graphite.go
View File

@@ -20,8 +20,8 @@ type GraphiteSerializer struct {
Template string
}
func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
out := []string{}
func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]byte, error) {
out := []byte{}
// Convert UnixNano to Unix timestamps
timestamp := metric.UnixNano() / 1000000000
@@ -34,12 +34,12 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error)
for fieldName, value := range metric.Fields() {
// Convert value to string
valueS := fmt.Sprintf("%#v", value)
point := fmt.Sprintf("%s %s %d",
point := []byte(fmt.Sprintf("%s %s %d\n",
// insert "field" section of template
sanitizedChars.Replace(InsertField(bucket, fieldName)),
sanitizedChars.Replace(valueS),
timestamp)
out = append(out, point)
timestamp))
out = append(out, point...)
}
return out, nil
}

41
plugins/serializers/graphite/graphite_test.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
var defaultTags = map[string]string{
@@ -29,19 +30,19 @@ const (
)
func TestGraphiteTags(t *testing.T) {
m1, _ := telegraf.NewMetric(
m1, _ := metric.New(
"mymeasurement",
map[string]string{"host": "192.168.0.1"},
map[string]interface{}{"value": float64(3.14)},
time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC),
)
m2, _ := telegraf.NewMetric(
m2, _ := metric.New(
"mymeasurement",
map[string]string{"host": "192.168.0.1", "afoo": "first", "bfoo": "second"},
map[string]interface{}{"value": float64(3.14)},
time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC),
)
m3, _ := telegraf.NewMetric(
m3, _ := metric.New(
"mymeasurement",
map[string]string{"afoo": "first", "bfoo": "second"},
map[string]interface{}{"value": float64(3.14)},
@@ -67,7 +68,7 @@ func TestSerializeMetricNoHost(t *testing.T) {
"usage_idle": float64(91.5),
"usage_busy": float64(8.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{}
@@ -94,7 +95,7 @@ func TestSerializeMetricHost(t *testing.T) {
"usage_idle": float64(91.5),
"usage_busy": float64(8.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{}
@@ -121,7 +122,7 @@ func TestSerializeValueField(t *testing.T) {
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{}
@@ -145,7 +146,7 @@ func TestSerializeValueField2(t *testing.T) {
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
@@ -171,7 +172,7 @@ func TestSerializeFieldWithSpaces(t *testing.T) {
fields := map[string]interface{}{
`field\ with\ spaces`: float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
@@ -197,7 +198,7 @@ func TestSerializeTagWithSpaces(t *testing.T) {
fields := map[string]interface{}{
`field_with_spaces`: float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
@@ -223,7 +224,7 @@ func TestSerializeValueField3(t *testing.T) {
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
@@ -249,7 +250,7 @@ func TestSerializeValueField5(t *testing.T) {
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
@@ -275,7 +276,7 @@ func TestSerializeMetricPrefix(t *testing.T) {
"usage_idle": float64(91.5),
"usage_busy": float64(8.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Prefix: "prefix"}
@@ -300,7 +301,7 @@ func TestSerializeBucketNameNoHost(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), "", "")
@@ -314,7 +315,7 @@ func TestSerializeBucketNameHost(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), "", "")
@@ -328,7 +329,7 @@ func TestSerializeBucketNamePrefix(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), "", "prefix")
@@ -342,7 +343,7 @@ func TestTemplate1(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), template1, "")
@@ -356,7 +357,7 @@ func TestTemplate2(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), template2, "")
@@ -370,7 +371,7 @@ func TestTemplate3(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), template3, "")
@@ -384,7 +385,7 @@ func TestTemplate4(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), template4, "")
@@ -398,7 +399,7 @@ func TestTemplate6(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
m, err := metric.New("cpu", defaultTags, fields, now)
assert.NoError(t, err)
mS := SerializeBucketName(m.Name(), m.Tags(), template6, "")

4
plugins/serializers/influx/influx.go
View File

@@ -7,6 +7,6 @@ import (
type InfluxSerializer struct {
}
func (s *InfluxSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
return []string{metric.String()}, nil
func (s *InfluxSerializer) Serialize(metric telegraf.Metric) ([]byte, error) {
return metric.Serialize(), nil
}

7
plugins/serializers/influx/influx_test.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
func TestSerializeMetricFloat(t *testing.T) {
@@ -18,7 +19,7 @@ func TestSerializeMetricFloat(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := InfluxSerializer{}
@@ -37,7 +38,7 @@ func TestSerializeMetricInt(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": int64(90),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := InfluxSerializer{}
@@ -56,7 +57,7 @@ func TestSerializeMetricString(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": "foobar",
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := InfluxSerializer{}

10
plugins/serializers/json/json.go
View File

@@ -9,9 +9,7 @@ import (
type JsonSerializer struct {
}
func (s *JsonSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
out := []string{}
func (s *JsonSerializer) Serialize(metric telegraf.Metric) ([]byte, error) {
m := make(map[string]interface{})
m["tags"] = metric.Tags()
m["fields"] = metric.Fields()
@@ -19,9 +17,9 @@ func (s *JsonSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
m["timestamp"] = metric.UnixNano() / 1000000000
serialized, err := ejson.Marshal(m)
if err != nil {
return []string{}, err
return []byte{}, err
}
out = append(out, string(serialized))
serialized = append(serialized, '\n')
return out, nil
return serialized, nil
}

9
plugins/serializers/json/json_test.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
)
func TestSerializeMetricFloat(t *testing.T) {
@@ -18,7 +19,7 @@ func TestSerializeMetricFloat(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := JsonSerializer{}
@@ -36,7 +37,7 @@ func TestSerializeMetricInt(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": int64(90),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := JsonSerializer{}
@@ -55,7 +56,7 @@ func TestSerializeMetricString(t *testing.T) {
fields := map[string]interface{}{
"usage_idle": "foobar",
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := JsonSerializer{}
@@ -75,7 +76,7 @@ func TestSerializeMultiFields(t *testing.T) {
"usage_idle": int64(90),
"usage_total": 8559615,
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := metric.New("cpu", tags, fields, now)
assert.NoError(t, err)
s := JsonSerializer{}

6
plugins/serializers/registry.go
View File

@@ -18,8 +18,10 @@ type SerializerOutput interface {
// Serializer is an interface defining functions that a serializer plugin must
// satisfy.
type Serializer interface {
// Serialize takes a single telegraf metric and turns it into a string.
Serialize(metric telegraf.Metric) ([]string, error)
// Serialize takes a single telegraf metric and turns it into a byte buffer.
// separate metrics should be separated by a newline, and there should be
// a newline at the end of the buffer.
Serialize(metric telegraf.Metric) ([]byte, error)
}
// Config is a struct that covers the data types needed for all serializer types,