GREEDY field templates for the graphite parser, and support for multiple specific field names

closes #789

CHANGELOG.md

@@ -9,6 +9,7 @@
 - [#849](https://github.com/influxdata/telegraf/issues/849): Adding ability to parse single values as an input data type.
 - [#844](https://github.com/influxdata/telegraf/pull/844): postgres_extensible plugin added. Thanks @menardorama!
 - [#866](https://github.com/influxdata/telegraf/pull/866): couchbase input plugin. Thanks @ljosa!
+- [#789](https://github.com/influxdata/telegraf/pull/789): Support multiple field specification and `field*` in graphite templates. Thanks @chrusty!
 
 ### Bugfixes
 - [#890](https://github.com/influxdata/telegraf/issues/890): Create TLS config even if only ssl_ca is provided.

docs/DATA_FORMATS_INPUT.md

@@ -220,17 +220,32 @@ So the following template:
 
 ```toml
 templates = [
-    "measurement.measurement.field.region"
+    "measurement.measurement.field.field.region"
 ]
 ```
 
 would result in the following Graphite -> Telegraf transformation.
 
 ```
-cpu.usage.idle.us-west 100
-=> cpu_usage,region=us-west idle=100
+cpu.usage.idle.percent.us-west 100
+=> cpu_usage,region=us-west idle_percent=100
 ```
 
+The field key can also be derived from the second "half" of the input metric-name by specifying ```field*```:
+```toml
+templates = [
+    "measurement.measurement.region.field*"
+]
+```
+
+would result in the following Graphite -> Telegraf transformation.
+
+```
+cpu.usage.us-west.idle.percentage 100
+=> cpu_usage,region=us-west idle_percentage=100
+```
+(This cannot be used in conjunction with "measurement*"!)
+
 #### Filter Templates:
 
 Users can also filter the template(s) to use based on the name of the bucket,

plugins/parsers/graphite/parser.go

@@ -231,6 +231,7 @@ func (p *GraphiteParser) ApplyTemplate(line string) (string, map[string]string,
 type template struct {
 	tags              []string
 	defaultTags       map[string]string
+	greedyField       bool
 	greedyMeasurement bool
 	separator         string
 }
@@ -248,6 +249,8 @@ func NewTemplate(pattern string, defaultTags map[string]string, separator string
 		}
 		if tag == "measurement*" {
 			template.greedyMeasurement = true
+		} else if tag == "field*" {
+			template.greedyField = true
 		}
 	}
 
@@ -265,7 +268,7 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
 	var (
 		measurement []string
 		tags        = make(map[string]string)
-		field       string
+		field       []string
 	)
 
 	// Set any default tags
@@ -273,6 +276,18 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
 		tags[k] = v
 	}
 
+	// See if an invalid combination has been specified in the template:
+	for _, tag := range t.tags {
+		if tag == "measurement*" {
+			t.greedyMeasurement = true
+		} else if tag == "field*" {
+			t.greedyField = true
+		}
+	}
+	if t.greedyField && t.greedyMeasurement {
+		return "", nil, "", fmt.Errorf("either 'field*' or 'measurement*' can be used in each template (but not both together): %q", strings.Join(t.tags, t.separator))
+	}
+
 	for i, tag := range t.tags {
 		if i >= len(fields) {
 			continue
@@ -281,10 +296,10 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
 		if tag == "measurement" {
 			measurement = append(measurement, fields[i])
 		} else if tag == "field" {
-			if len(field) != 0 {
-				return "", nil, "", fmt.Errorf("'field' can only be used once in each template: %q", line)
-			}
-			field = fields[i]
+			field = append(field, fields[i])
+		} else if tag == "field*" {
+			field = append(field, fields[i:]...)
+			break
 		} else if tag == "measurement*" {
 			measurement = append(measurement, fields[i:]...)
 			break
@@ -293,7 +308,7 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
 		}
 	}
 
-	return strings.Join(measurement, t.separator), tags, field, nil
+	return strings.Join(measurement, t.separator), tags, strings.Join(field, t.separator), nil
 }
 
 // matcher determines which template should be applied to a given metric

plugins/parsers/graphite/parser_test.go

@@ -94,6 +94,20 @@ func TestTemplateApply(t *testing.T) {
 			measurement: "cpu.load",
 			tags:        map[string]string{"zone": "us-west"},
 		},
+		{
+			test:        "conjoined fields",
+			input:       "prod.us-west.server01.cpu.util.idle.percent",
+			template:    "env.zone.host.measurement.measurement.field*",
+			measurement: "cpu.util",
+			tags:        map[string]string{"env": "prod", "zone": "us-west", "host": "server01"},
+		},
+		{
+			test:        "multiple fields",
+			input:       "prod.us-west.server01.cpu.util.idle.percent.free",
+			template:    "env.zone.host.measurement.measurement.field.field.reading",
+			measurement: "cpu.util",
+			tags:        map[string]string{"env": "prod", "zone": "us-west", "host": "server01", "reading": "free"},
+		},
 	}
 
 	for _, test := range tests {
@@ -187,6 +201,12 @@ func TestParse(t *testing.T) {
 			template: "measurement",
 			err:      `field "cpu" time: strconv.ParseFloat: parsing "14199724z57825": invalid syntax`,
 		},
+		{
+			test:     "measurement* and field* (invalid)",
+			input:    `prod.us-west.server01.cpu.util.idle.percent 99.99 1419972457825`,
+			template: "env.zone.host.measurement*.field*",
+			err:      `either 'field*' or 'measurement*' can be used in each template (but not both together): "env.zone.host.measurement*.field*"`,
+		},
 	}
 
 	for _, test := range tests {
@@ -574,15 +594,48 @@ func TestApplyTemplateField(t *testing.T) {
 	}
 }
 
-func TestApplyTemplateFieldError(t *testing.T) {
+func TestApplyTemplateMultipleFieldsTogether(t *testing.T) {
 	p, err := NewGraphiteParser("_",
-		[]string{"current.* measurement.field.field"}, nil)
+		[]string{"current.* measurement.measurement.field.field"}, nil)
 	assert.NoError(t, err)
 
-	_, _, _, err = p.ApplyTemplate("current.users.logged_in")
-	if err == nil {
-		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s", err,
-			"'field' can only be used once in each template: current.users.logged_in")
+	measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh")
+
+	assert.Equal(t, "current_users", measurement)
+
+	if field != "logged_in_ssh" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			field, "logged_in_ssh")
+	}
+}
+
+func TestApplyTemplateMultipleFieldsApart(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.measurement.field.method.field"}, nil)
+	assert.NoError(t, err)
+
+	measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh.total")
+
+	assert.Equal(t, "current_users", measurement)
+
+	if field != "logged_in_total" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			field, "logged_in_total")
+	}
+}
+
+func TestApplyTemplateGreedyField(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.measurement.field*"}, nil)
+	assert.NoError(t, err)
+
+	measurement, _, field, err := p.ApplyTemplate("current.users.logged_in")
+
+	assert.Equal(t, "current_users", measurement)
+
+	if field != "logged_in" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			field, "logged_in")
 	}
 }