dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Create a template system for the graphite serializer

Browse Source 
closes #925
closes #879
This commit is contained in:
Cameron Sparr
2016-04-08 16:04:45 -06:00
parent 27fe4f7062
commit f5878eafb9
12 changed files with 401 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
plugins/serializers/graphite/graphite.go
View File

@@ -8,11 +8,16 @@ import (
"github.com/influxdata/telegraf"
)
const DEFAULT_TEMPLATE = "host.tags.measurement.field"
var fieldDeleter = strings.NewReplacer(".FIELDNAME", "", "FIELDNAME.", "")
type GraphiteSerializer struct {
Prefix string
Prefix string
Template string
}
var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_")
var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_", "..", ".")
func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
out := []string{}
@@ -20,65 +25,95 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error)
// Convert UnixNano to Unix timestamps
timestamp := metric.UnixNano() / 1000000000
for field_name, value := range metric.Fields() {
// Convert value
value_str := fmt.Sprintf("%#v", value)
// Write graphite metric
var graphitePoint string
graphitePoint = fmt.Sprintf("%s %s %d",
s.SerializeBucketName(metric, field_name),
value_str,
bucket := s.SerializeBucketName(metric.Name(), metric.Tags())
for fieldName, value := range metric.Fields() {
// Convert value to string
valueS := fmt.Sprintf("%#v", value)
point := fmt.Sprintf("%s %s %d",
// insert "field" section of template
InsertField(bucket, fieldName),
valueS,
timestamp)
out = append(out, graphitePoint)
out = append(out, point)
}
return out, nil
}
func (s *GraphiteSerializer) SerializeBucketName(metric telegraf.Metric, field_name string) string {
// Get the metric name
name := metric.Name()
// Convert UnixNano to Unix timestamps
tag_str := buildTags(metric)
// Write graphite metric
var serializedBucketName string
if name == field_name {
serializedBucketName = fmt.Sprintf("%s.%s",
tag_str,
strings.Replace(name, ".", "_", -1))
} else {
serializedBucketName = fmt.Sprintf("%s.%s.%s",
tag_str,
strings.Replace(name, ".", "_", -1),
strings.Replace(field_name, ".", "_", -1))
// SerializeBucketName will take the given measurement name and tags and
// produce a graphite bucket. It will use the GraphiteSerializer.Template
// to generate this, or DEFAULT_TEMPLATE.
//
// NOTE: SerializeBucketName replaces the "field" portion of the template with
// FIELDNAME. It is up to the user to replace this. This is so that
// SerializeBucketName can be called just once per measurement, rather than
// once per field. See GraphiteSerializer.InsertField() function.
func (s *GraphiteSerializer) SerializeBucketName(
measurement string,
tags map[string]string,
) string {
if s.Template == "" {
s.Template = DEFAULT_TEMPLATE
}
if s.Prefix != "" {
serializedBucketName = fmt.Sprintf("%s.%s", s.Prefix, serializedBucketName)
tagsCopy := make(map[string]string)
for k, v := range tags {
tagsCopy[k] = v
}
return serializedBucketName
var out []string
templateParts := strings.Split(s.Template, ".")
for _, templatePart := range templateParts {
switch templatePart {
case "measurement":
out = append(out, measurement)
case "tags":
// we will replace this later
out = append(out, "TAGS")
case "field":
// user of SerializeBucketName needs to replace this
out = append(out, "FIELDNAME")
default:
// This is a tag being applied
if tagvalue, ok := tagsCopy[templatePart]; ok {
out = append(out, strings.Replace(tagvalue, ".", "_", -1))
delete(tagsCopy, templatePart)
}
}
}
// insert remaining tags into output name
for i, templatePart := range out {
if templatePart == "TAGS" {
out[i] = buildTags(tagsCopy)
break
}
}
if s.Prefix == "" {
return sanitizedChars.Replace(strings.Join(out, "."))
}
return sanitizedChars.Replace(s.Prefix + "." + strings.Join(out, "."))
}
func buildTags(metric telegraf.Metric) string {
// InsertField takes the bucket string from SerializeBucketName and replaces the
// FIELDNAME portion. If fieldName == "value", it will simply delete the
// FIELDNAME portion.
func InsertField(bucket, fieldName string) string {
// if the field name is "value", then dont use it
if fieldName == "value" {
return fieldDeleter.Replace(bucket)
}
return strings.Replace(bucket, "FIELDNAME", fieldName, 1)
}
func buildTags(tags map[string]string) string {
var keys []string
tags := metric.Tags()
for k := range tags {
if k == "host" {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
var tag_str string
if host, ok := tags["host"]; ok {
if len(keys) > 0 {
tag_str = strings.Replace(host, ".", "_", -1) + "."
} else {
tag_str = strings.Replace(host, ".", "_", -1)
}
}
for i, k := range keys {
tag_value := strings.Replace(tags[k], ".", "_", -1)
if i == 0 {
@@ -87,5 +122,5 @@ func buildTags(metric telegraf.Metric) string {
tag_str += "." + tag_value
}
}
return sanitizedChars.Replace(tag_str)
return tag_str
}

217
plugins/serializers/graphite/graphite_test.go
View File

@@ -11,6 +11,23 @@ import (
"github.com/influxdata/telegraf"
)
var defaultTags = map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
const (
template1 = "tags.measurement.field"
template2 = "host.measurement.field"
template3 = "host.tags.field"
template4 = "host.tags.measurement"
// this template explicitly uses all tag keys, so "tags" should be empty
template5 = "host.datacenter.cpu.tags.measurement.field"
// this template has non-existent tag keys
template6 = "foo.host.cpu.bar.tags.measurement.field"
)
func TestGraphiteTags(t *testing.T) {
m1, _ := telegraf.NewMetric(
"mymeasurement",
@@ -31,12 +48,12 @@ func TestGraphiteTags(t *testing.T) {
time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC),
)
tags1 := buildTags(m1)
tags2 := buildTags(m2)
tags3 := buildTags(m3)
tags1 := buildTags(m1.Tags())
tags2 := buildTags(m2.Tags())
tags3 := buildTags(m3.Tags())
assert.Equal(t, "192_168_0_1", tags1)
assert.Equal(t, "192_168_0_1.first.second", tags2)
assert.Equal(t, "first.second.192_168_0_1", tags2)
assert.Equal(t, "first.second", tags3)
}
@@ -93,6 +110,82 @@ func TestSerializeMetricHost(t *testing.T) {
assert.Equal(t, expS, mS)
}
// test that a field named "value" gets ignored.
func TestSerializeValueField(t *testing.T) {
now := time.Now()
tags := map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{}
mS, err := s.Serialize(m)
assert.NoError(t, err)
expS := []string{
fmt.Sprintf("localhost.cpu0.us-west-2.cpu 91.5 %d", now.Unix()),
}
assert.Equal(t, expS, mS)
}
// test that a field named "value" gets ignored in middle of template.
func TestSerializeValueField2(t *testing.T) {
now := time.Now()
tags := map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
Template: "host.field.tags.measurement",
}
mS, err := s.Serialize(m)
assert.NoError(t, err)
expS := []string{
fmt.Sprintf("localhost.cpu0.us-west-2.cpu 91.5 %d", now.Unix()),
}
assert.Equal(t, expS, mS)
}
// test that a field named "value" gets ignored at beginning of template.
func TestSerializeValueField3(t *testing.T) {
now := time.Now()
tags := map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
fields := map[string]interface{}{
"value": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{
Template: "field.host.tags.measurement",
}
mS, err := s.Serialize(m)
assert.NoError(t, err)
expS := []string{
fmt.Sprintf("localhost.cpu0.us-west-2.cpu 91.5 %d", now.Unix()),
}
assert.Equal(t, expS, mS)
}
func TestSerializeMetricPrefix(t *testing.T) {
now := time.Now()
tags := map[string]string{
@@ -133,48 +226,128 @@ func TestSerializeBucketNameNoHost(t *testing.T) {
assert.NoError(t, err)
s := GraphiteSerializer{}
mS := s.SerializeBucketName(m, "usage_idle")
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := fmt.Sprintf("cpu0.us-west-2.cpu.usage_idle")
expS := "cpu0.us-west-2.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestSerializeBucketNameHost(t *testing.T) {
now := time.Now()
tags := map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{}
mS := s.SerializeBucketName(m, "usage_idle")
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := fmt.Sprintf("localhost.cpu0.us-west-2.cpu.usage_idle")
expS := "localhost.cpu0.us-west-2.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestSerializeBucketNamePrefix(t *testing.T) {
now := time.Now()
tags := map[string]string{
"host": "localhost",
"cpu": "cpu0",
"datacenter": "us-west-2",
}
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", tags, fields, now)
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Prefix: "prefix"}
mS := s.SerializeBucketName(m, "usage_idle")
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := fmt.Sprintf("prefix.localhost.cpu0.us-west-2.cpu.usage_idle")
expS := "prefix.localhost.cpu0.us-west-2.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestTemplate1(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template1}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "cpu0.us-west-2.localhost.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestTemplate2(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template2}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "localhost.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestTemplate3(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template3}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "localhost.cpu0.us-west-2.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestTemplate4(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template4}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "localhost.cpu0.us-west-2.cpu"
assert.Equal(t, expS, mS)
}
func TestTemplate5(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template5}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "localhost.us-west-2.cpu0.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}
func TestTemplate6(t *testing.T) {
now := time.Now()
fields := map[string]interface{}{
"usage_idle": float64(91.5),
}
m, err := telegraf.NewMetric("cpu", defaultTags, fields, now)
assert.NoError(t, err)
s := GraphiteSerializer{Template: template6}
mS := s.SerializeBucketName(m.Name(), m.Tags())
expS := "localhost.cpu0.us-west-2.cpu.FIELDNAME"
assert.Equal(t, expS, mS)
}

11
plugins/serializers/registry.go
View File

@@ -30,6 +30,10 @@ type Config struct {
// Prefix to add to all measurements, only supports Graphite
Prefix string
// Template for converting telegraf metrics into Graphite
// only supports Graphite
Template string
}
// NewSerializer a Serializer interface based on the given config.
@@ -40,7 +44,7 @@ func NewSerializer(config *Config) (Serializer, error) {
case "influx":
serializer, err = NewInfluxSerializer()
case "graphite":
serializer, err = NewGraphiteSerializer(config.Prefix)
serializer, err = NewGraphiteSerializer(config.Prefix, config.Template)
case "json":
serializer, err = NewJsonSerializer()
}
@@ -55,8 +59,9 @@ func NewInfluxSerializer() (Serializer, error) {
return &influx.InfluxSerializer{}, nil
}
func NewGraphiteSerializer(prefix string) (Serializer, error) {
func NewGraphiteSerializer(prefix, template string) (Serializer, error) {
return &graphite.GraphiteSerializer{
Prefix: prefix,
Prefix: prefix,
Template: template,
}, nil
}