Implementing generic parser plugins and documentation

This constitutes a large change in how we will parse different data
formats going forward (for the plugins that support it)

This is working off @henrypfhu's changes.

plugins/parsers/graphite/config.go

@@ -0,0 +1,135 @@
+package graphite
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	// DefaultSeparator is the default join character to use when joining multiple
+	// measurment parts in a template.
+	DefaultSeparator = "."
+)
+
+// Config represents the configuration for Graphite endpoints.
+type Config struct {
+	Separator string
+	Templates []string
+}
+
+// Validate validates the config's templates and tags.
+func (c *Config) Validate() error {
+	if err := c.validateTemplates(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Config) validateTemplates() error {
+	// map to keep track of filters we see
+	filters := map[string]struct{}{}
+
+	for i, t := range c.Templates {
+		parts := strings.Fields(t)
+		// Ensure template string is non-empty
+		if len(parts) == 0 {
+			return fmt.Errorf("missing template at position: %d", i)
+		}
+		if len(parts) == 1 && parts[0] == "" {
+			return fmt.Errorf("missing template at position: %d", i)
+		}
+
+		if len(parts) > 3 {
+			return fmt.Errorf("invalid template format: '%s'", t)
+		}
+
+		template := t
+		filter := ""
+		tags := ""
+		if len(parts) >= 2 {
+			// We could have <filter> <template>  or <template> <tags>.  Equals is only allowed in
+			// tags section.
+			if strings.Contains(parts[1], "=") {
+				template = parts[0]
+				tags = parts[1]
+			} else {
+				filter = parts[0]
+				template = parts[1]
+			}
+		}
+
+		if len(parts) == 3 {
+			tags = parts[2]
+		}
+
+		// Validate the template has one and only one measurement
+		if err := c.validateTemplate(template); err != nil {
+			return err
+		}
+
+		// Prevent duplicate filters in the config
+		if _, ok := filters[filter]; ok {
+			return fmt.Errorf("duplicate filter '%s' found at position: %d", filter, i)
+		}
+		filters[filter] = struct{}{}
+
+		if filter != "" {
+			// Validate filter expression is valid
+			if err := c.validateFilter(filter); err != nil {
+				return err
+			}
+		}
+
+		if tags != "" {
+			// Validate tags
+			for _, tagStr := range strings.Split(tags, ",") {
+				if err := c.validateTag(tagStr); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Config) validateTemplate(template string) error {
+	hasMeasurement := false
+	for _, p := range strings.Split(template, ".") {
+		if p == "measurement" || p == "measurement*" {
+			hasMeasurement = true
+		}
+	}
+
+	if !hasMeasurement {
+		return fmt.Errorf("no measurement in template `%s`", template)
+	}
+
+	return nil
+}
+
+func (c *Config) validateFilter(filter string) error {
+	for _, p := range strings.Split(filter, ".") {
+		if p == "" {
+			return fmt.Errorf("filter contains blank section: %s", filter)
+		}
+
+		if strings.Contains(p, "*") && p != "*" {
+			return fmt.Errorf("invalid filter wildcard section: %s", filter)
+		}
+	}
+	return nil
+}
+
+func (c *Config) validateTag(keyValue string) error {
+	parts := strings.Split(keyValue, "=")
+	if len(parts) != 2 {
+		return fmt.Errorf("invalid template tags: '%s'", keyValue)
+	}
+
+	if parts[0] == "" || parts[1] == "" {
+		return fmt.Errorf("invalid template tags: %s'", keyValue)
+	}
+
+	return nil
+}

plugins/parsers/graphite/errors.go

@@ -0,0 +1,14 @@
+package graphite
+
+import "fmt"
+
+// An UnsupposedValueError is returned when a parsed value is not
+// supposed.
+type UnsupposedValueError struct {
+	Field string
+	Value float64
+}
+
+func (err *UnsupposedValueError) Error() string {
+	return fmt.Sprintf(`field "%s" value: "%v" is unsupported`, err.Field, err.Value)
+}

plugins/parsers/graphite/parser.go

@@ -0,0 +1,425 @@
+package graphite
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+)
+
+// Minimum and maximum supported dates for timestamps.
+var (
+	MinDate = time.Date(1901, 12, 13, 0, 0, 0, 0, time.UTC)
+	MaxDate = time.Date(2038, 1, 19, 0, 0, 0, 0, time.UTC)
+)
+
+// Parser encapsulates a Graphite Parser.
+type GraphiteParser struct {
+	Separator   string
+	Templates   []string
+	DefaultTags map[string]string
+
+	matcher *matcher
+}
+
+func NewGraphiteParser(
+	separator string,
+	templates []string,
+	defaultTags map[string]string,
+) (*GraphiteParser, error) {
+	var err error
+
+	if separator == "" {
+		separator = DefaultSeparator
+	}
+	p := &GraphiteParser{
+		Separator: separator,
+		Templates: templates,
+	}
+
+	if defaultTags != nil {
+		p.DefaultTags = defaultTags
+	}
+
+	matcher := newMatcher()
+	p.matcher = matcher
+	defaultTemplate, _ := NewTemplate("measurement*", nil, p.Separator)
+	matcher.AddDefaultTemplate(defaultTemplate)
+
+	for _, pattern := range p.Templates {
+		template := pattern
+		filter := ""
+		// Format is [filter] <template> [tag1=value1,tag2=value2]
+		parts := strings.Fields(pattern)
+		if len(parts) < 1 {
+			continue
+		} else if len(parts) >= 2 {
+			if strings.Contains(parts[1], "=") {
+				template = parts[0]
+			} else {
+				filter = parts[0]
+				template = parts[1]
+			}
+		}
+
+		// Parse out the default tags specific to this template
+		tags := map[string]string{}
+		if strings.Contains(parts[len(parts)-1], "=") {
+			tagStrs := strings.Split(parts[len(parts)-1], ",")
+			for _, kv := range tagStrs {
+				parts := strings.Split(kv, "=")
+				tags[parts[0]] = parts[1]
+			}
+		}
+
+		tmpl, err1 := NewTemplate(template, tags, p.Separator)
+		if err1 != nil {
+			err = err1
+			break
+		}
+		matcher.Add(filter, tmpl)
+	}
+
+	if err != nil {
+		return p, fmt.Errorf("exec input parser config is error: %s ", err.Error())
+	} else {
+		return p, nil
+	}
+}
+
+func (p *GraphiteParser) Parse(buf []byte) ([]telegraf.Metric, error) {
+	// parse even if the buffer begins with a newline
+	buf = bytes.TrimPrefix(buf, []byte("\n"))
+	// add newline to end if not exists:
+	if len(buf) > 0 && !bytes.HasSuffix(buf, []byte("\n")) {
+		buf = append(buf, []byte("\n")...)
+	}
+
+	metrics := make([]telegraf.Metric, 0)
+
+	buffer := bytes.NewBuffer(buf)
+	reader := bufio.NewReader(buffer)
+	for {
+		// Read up to the next newline.
+		buf, err := reader.ReadBytes('\n')
+		if err == io.EOF {
+			return metrics, nil
+		}
+		if err != nil && err != io.EOF {
+			return metrics, err
+		}
+
+		// Trim the buffer, even though there should be no padding
+		line := strings.TrimSpace(string(buf))
+		if metric, err := p.ParseLine(line); err == nil {
+			metrics = append(metrics, metric)
+		}
+	}
+}
+
+// Parse performs Graphite parsing of a single line.
+func (p *GraphiteParser) ParseLine(line string) (telegraf.Metric, error) {
+	// Break into 3 fields (name, value, timestamp).
+	fields := strings.Fields(line)
+	if len(fields) != 2 && len(fields) != 3 {
+		return nil, fmt.Errorf("received %q which doesn't have required fields", line)
+	}
+
+	// decode the name and tags
+	template := p.matcher.Match(fields[0])
+	measurement, tags, field, err := template.Apply(fields[0])
+	if err != nil {
+		return nil, err
+	}
+
+	// Could not extract measurement, use the raw value
+	if measurement == "" {
+		measurement = fields[0]
+	}
+
+	// Parse value.
+	v, err := strconv.ParseFloat(fields[1], 64)
+	if err != nil {
+		return nil, fmt.Errorf(`field "%s" value: %s`, fields[0], err)
+	}
+
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return nil, &UnsupposedValueError{Field: fields[0], Value: v}
+	}
+
+	fieldValues := map[string]interface{}{}
+	if field != "" {
+		fieldValues[field] = v
+	} else {
+		fieldValues["value"] = v
+	}
+
+	// If no 3rd field, use now as timestamp
+	timestamp := time.Now().UTC()
+
+	if len(fields) == 3 {
+		// Parse timestamp.
+		unixTime, err := strconv.ParseFloat(fields[2], 64)
+		if err != nil {
+			return nil, fmt.Errorf(`field "%s" time: %s`, fields[0], err)
+		}
+
+		// -1 is a special value that gets converted to current UTC time
+		// See https://github.com/graphite-project/carbon/issues/54
+		if unixTime != float64(-1) {
+			// Check if we have fractional seconds
+			timestamp = time.Unix(int64(unixTime), int64((unixTime-math.Floor(unixTime))*float64(time.Second)))
+			if timestamp.Before(MinDate) || timestamp.After(MaxDate) {
+				return nil, fmt.Errorf("timestamp out of range")
+			}
+		}
+	}
+	// Set the default tags on the point if they are not already set
+	for k, v := range p.DefaultTags {
+		if _, ok := tags[k]; !ok {
+			tags[k] = v
+		}
+	}
+
+	return telegraf.NewMetric(measurement, tags, fieldValues, timestamp)
+}
+
+// ApplyTemplate extracts the template fields from the given line and
+// returns the measurement name and tags.
+func (p *GraphiteParser) ApplyTemplate(line string) (string, map[string]string, string, error) {
+	// Break line into fields (name, value, timestamp), only name is used
+	fields := strings.Fields(line)
+	if len(fields) == 0 {
+		return "", make(map[string]string), "", nil
+	}
+	// decode the name and tags
+	template := p.matcher.Match(fields[0])
+	name, tags, field, err := template.Apply(fields[0])
+
+	// Set the default tags on the point if they are not already set
+	for k, v := range p.DefaultTags {
+		if _, ok := tags[k]; !ok {
+			tags[k] = v
+		}
+	}
+
+	return name, tags, field, err
+}
+
+// template represents a pattern and tags to map a graphite metric string to a influxdb Point
+type template struct {
+	tags              []string
+	defaultTags       map[string]string
+	greedyMeasurement bool
+	separator         string
+}
+
+// NewTemplate returns a new template ensuring it has a measurement
+// specified.
+func NewTemplate(pattern string, defaultTags map[string]string, separator string) (*template, error) {
+	tags := strings.Split(pattern, ".")
+	hasMeasurement := false
+	template := &template{tags: tags, defaultTags: defaultTags, separator: separator}
+
+	for _, tag := range tags {
+		if strings.HasPrefix(tag, "measurement") {
+			hasMeasurement = true
+		}
+		if tag == "measurement*" {
+			template.greedyMeasurement = true
+		}
+	}
+
+	if !hasMeasurement {
+		return nil, fmt.Errorf("no measurement specified for template. %q", pattern)
+	}
+
+	return template, nil
+}
+
+// Apply extracts the template fields from the given line and returns the measurement
+// name and tags
+func (t *template) Apply(line string) (string, map[string]string, string, error) {
+	fields := strings.Split(line, ".")
+	var (
+		measurement []string
+		tags        = make(map[string]string)
+		field       string
+	)
+
+	// Set any default tags
+	for k, v := range t.defaultTags {
+		tags[k] = v
+	}
+
+	for i, tag := range t.tags {
+		if i >= len(fields) {
+			continue
+		}
+
+		if tag == "measurement" {
+			measurement = append(measurement, fields[i])
+		} else if tag == "field" {
+			if len(field) != 0 {
+				return "", nil, "", fmt.Errorf("'field' can only be used once in each template: %q", line)
+			}
+			field = fields[i]
+		} else if tag == "measurement*" {
+			measurement = append(measurement, fields[i:]...)
+			break
+		} else if tag != "" {
+			tags[tag] = fields[i]
+		}
+	}
+
+	return strings.Join(measurement, t.separator), tags, field, nil
+}
+
+// matcher determines which template should be applied to a given metric
+// based on a filter tree.
+type matcher struct {
+	root            *node
+	defaultTemplate *template
+}
+
+func newMatcher() *matcher {
+	return &matcher{
+		root: &node{},
+	}
+}
+
+// Add inserts the template in the filter tree based the given filter
+func (m *matcher) Add(filter string, template *template) {
+	if filter == "" {
+		m.AddDefaultTemplate(template)
+		return
+	}
+	m.root.Insert(filter, template)
+}
+
+func (m *matcher) AddDefaultTemplate(template *template) {
+	m.defaultTemplate = template
+}
+
+// Match returns the template that matches the given graphite line
+func (m *matcher) Match(line string) *template {
+	tmpl := m.root.Search(line)
+	if tmpl != nil {
+		return tmpl
+	}
+
+	return m.defaultTemplate
+}
+
+// node is an item in a sorted k-ary tree.  Each child is sorted by its value.
+// The special value of "*", is always last.
+type node struct {
+	value    string
+	children nodes
+	template *template
+}
+
+func (n *node) insert(values []string, template *template) {
+	// Add the end, set the template
+	if len(values) == 0 {
+		n.template = template
+		return
+	}
+
+	// See if the the current element already exists in the tree. If so, insert the
+	// into that sub-tree
+	for _, v := range n.children {
+		if v.value == values[0] {
+			v.insert(values[1:], template)
+			return
+		}
+	}
+
+	// New element, add it to the tree and sort the children
+	newNode := &node{value: values[0]}
+	n.children = append(n.children, newNode)
+	sort.Sort(&n.children)
+
+	// Now insert the rest of the tree into the new element
+	newNode.insert(values[1:], template)
+}
+
+// Insert inserts the given string template into the tree.  The filter string is separated
+// on "." and each part is used as the path in the tree.
+func (n *node) Insert(filter string, template *template) {
+	n.insert(strings.Split(filter, "."), template)
+}
+
+func (n *node) search(lineParts []string) *template {
+	// Nothing to search
+	if len(lineParts) == 0 || len(n.children) == 0 {
+		return n.template
+	}
+
+	// If last element is a wildcard, don't include in this search since it's sorted
+	// to the end but lexicographically it would not always be and sort.Search assumes
+	// the slice is sorted.
+	length := len(n.children)
+	if n.children[length-1].value == "*" {
+		length--
+	}
+
+	// Find the index of child with an exact match
+	i := sort.Search(length, func(i int) bool {
+		return n.children[i].value >= lineParts[0]
+	})
+
+	// Found an exact match, so search that child sub-tree
+	if i < len(n.children) && n.children[i].value == lineParts[0] {
+		return n.children[i].search(lineParts[1:])
+	}
+	// Not an exact match, see if we have a wildcard child to search
+	if n.children[len(n.children)-1].value == "*" {
+		return n.children[len(n.children)-1].search(lineParts[1:])
+	}
+	return n.template
+}
+
+func (n *node) Search(line string) *template {
+	return n.search(strings.Split(line, "."))
+}
+
+type nodes []*node
+
+// Less returns a boolean indicating whether the filter at position j
+// is less than the filter at position k.  Filters are order by string
+// comparison of each component parts.  A wildcard value "*" is never
+// less than a non-wildcard value.
+//
+// For example, the filters:
+//             "*.*"
+//             "servers.*"
+//             "servers.localhost"
+//             "*.localhost"
+//
+// Would be sorted as:
+//             "servers.localhost"
+//             "servers.*"
+//             "*.localhost"
+//             "*.*"
+func (n *nodes) Less(j, k int) bool {
+	if (*n)[j].value == "*" && (*n)[k].value != "*" {
+		return false
+	}
+
+	if (*n)[j].value != "*" && (*n)[k].value == "*" {
+		return true
+	}
+
+	return (*n)[j].value < (*n)[k].value
+}
+
+func (n *nodes) Swap(i, j int) { (*n)[i], (*n)[j] = (*n)[j], (*n)[i] }
+func (n *nodes) Len() int      { return len(*n) }

plugins/parsers/graphite/parser_test.go

@@ -0,0 +1,595 @@
+package graphite
+
+import (
+	"reflect"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/influxdata/telegraf"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func BenchmarkParse(b *testing.B) {
+	p, err := NewGraphiteParser("_", []string{
+		"*.* .wrong.measurement*",
+		"servers.* .host.measurement*",
+		"servers.localhost .host.measurement*",
+		"*.localhost .host.measurement*",
+		"*.*.cpu .host.measurement*",
+		"a.b.c .host.measurement*",
+		"influxd.*.foo .host.measurement*",
+		"prod.*.mem .host.measurement*",
+	}, nil)
+
+	if err != nil {
+		b.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	for i := 0; i < b.N; i++ {
+		p.Parse([]byte("servers.localhost.cpu.load 11 1435077219"))
+	}
+}
+
+func TestTemplateApply(t *testing.T) {
+	var tests = []struct {
+		test        string
+		input       string
+		template    string
+		measurement string
+		tags        map[string]string
+		err         string
+	}{
+		{
+			test:        "metric only",
+			input:       "cpu",
+			template:    "measurement",
+			measurement: "cpu",
+		},
+		{
+			test:        "metric with single series",
+			input:       "cpu.server01",
+			template:    "measurement.hostname",
+			measurement: "cpu",
+			tags:        map[string]string{"hostname": "server01"},
+		},
+		{
+			test:        "metric with multiple series",
+			input:       "cpu.us-west.server01",
+			template:    "measurement.region.hostname",
+			measurement: "cpu",
+			tags:        map[string]string{"hostname": "server01", "region": "us-west"},
+		},
+		{
+			test: "no metric",
+			tags: make(map[string]string),
+			err:  `no measurement specified for template. ""`,
+		},
+		{
+			test:        "ignore unnamed",
+			input:       "foo.cpu",
+			template:    "measurement",
+			measurement: "foo",
+			tags:        make(map[string]string),
+		},
+		{
+			test:        "name shorter than template",
+			input:       "foo",
+			template:    "measurement.A.B.C",
+			measurement: "foo",
+			tags:        make(map[string]string),
+		},
+		{
+			test:        "wildcard measurement at end",
+			input:       "prod.us-west.server01.cpu.load",
+			template:    "env.zone.host.measurement*",
+			measurement: "cpu.load",
+			tags:        map[string]string{"env": "prod", "zone": "us-west", "host": "server01"},
+		},
+		{
+			test:        "skip fields",
+			input:       "ignore.us-west.ignore-this-too.cpu.load",
+			template:    ".zone..measurement*",
+			measurement: "cpu.load",
+			tags:        map[string]string{"zone": "us-west"},
+		},
+	}
+
+	for _, test := range tests {
+		tmpl, err := NewTemplate(test.template, nil, DefaultSeparator)
+		if errstr(err) != test.err {
+			t.Fatalf("err does not match.  expected %v, got %v", test.err, err)
+		}
+		if err != nil {
+			// If we erred out,it was intended and the following tests won't work
+			continue
+		}
+
+		measurement, tags, _, _ := tmpl.Apply(test.input)
+		if measurement != test.measurement {
+			t.Fatalf("name parse failer.  expected %v, got %v", test.measurement, measurement)
+		}
+		if len(tags) != len(test.tags) {
+			t.Fatalf("unexpected number of tags.  expected %v, got %v", test.tags, tags)
+		}
+		for k, v := range test.tags {
+			if tags[k] != v {
+				t.Fatalf("unexpected tag value for tags[%s].  expected %q, got %q", k, v, tags[k])
+			}
+		}
+	}
+}
+
+func TestParseMissingMeasurement(t *testing.T) {
+	_, err := NewGraphiteParser("", []string{"a.b.c"}, nil)
+	if err == nil {
+		t.Fatalf("expected error creating parser, got nil")
+	}
+}
+
+func TestParse(t *testing.T) {
+	testTime := time.Now().Round(time.Second)
+	epochTime := testTime.Unix()
+	strTime := strconv.FormatInt(epochTime, 10)
+
+	var tests = []struct {
+		test        string
+		input       string
+		measurement string
+		tags        map[string]string
+		value       float64
+		time        time.Time
+		template    string
+		err         string
+	}{
+		{
+			test:        "normal case",
+			input:       `cpu.foo.bar 50 ` + strTime,
+			template:    "measurement.foo.bar",
+			measurement: "cpu",
+			tags: map[string]string{
+				"foo": "foo",
+				"bar": "bar",
+			},
+			value: 50,
+			time:  testTime,
+		},
+		{
+			test:        "metric only with float value",
+			input:       `cpu 50.554 ` + strTime,
+			measurement: "cpu",
+			template:    "measurement",
+			value:       50.554,
+			time:        testTime,
+		},
+		{
+			test:     "missing metric",
+			input:    `1419972457825`,
+			template: "measurement",
+			err:      `received "1419972457825" which doesn't have required fields`,
+		},
+		{
+			test:     "should error parsing invalid float",
+			input:    `cpu 50.554z 1419972457825`,
+			template: "measurement",
+			err:      `field "cpu" value: strconv.ParseFloat: parsing "50.554z": invalid syntax`,
+		},
+		{
+			test:     "should error parsing invalid int",
+			input:    `cpu 50z 1419972457825`,
+			template: "measurement",
+			err:      `field "cpu" value: strconv.ParseFloat: parsing "50z": invalid syntax`,
+		},
+		{
+			test:     "should error parsing invalid time",
+			input:    `cpu 50.554 14199724z57825`,
+			template: "measurement",
+			err:      `field "cpu" time: strconv.ParseFloat: parsing "14199724z57825": invalid syntax`,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := NewGraphiteParser("", []string{test.template}, nil)
+		if err != nil {
+			t.Fatalf("unexpected error creating graphite parser: %v", err)
+		}
+
+		metric, err := p.ParseLine(test.input)
+		if errstr(err) != test.err {
+			t.Fatalf("err does not match.  expected %v, got %v", test.err, err)
+		}
+		if err != nil {
+			// If we erred out,it was intended and the following tests won't work
+			continue
+		}
+		if metric.Name() != test.measurement {
+			t.Fatalf("name parse failer.  expected %v, got %v",
+				test.measurement, metric.Name())
+		}
+		if len(metric.Tags()) != len(test.tags) {
+			t.Fatalf("tags len mismatch.  expected %d, got %d",
+				len(test.tags), len(metric.Tags()))
+		}
+		f := metric.Fields()["value"].(float64)
+		if metric.Fields()["value"] != f {
+			t.Fatalf("floatValue value mismatch.  expected %v, got %v",
+				test.value, f)
+		}
+		if metric.Time().UnixNano()/1000000 != test.time.UnixNano()/1000000 {
+			t.Fatalf("time value mismatch.  expected %v, got %v",
+				test.time.UnixNano(), metric.Time().UnixNano())
+		}
+	}
+}
+
+func TestParseNaN(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"measurement*"}, nil)
+	assert.NoError(t, err)
+
+	_, err = p.ParseLine("servers.localhost.cpu_load NaN 1435077219")
+	assert.Error(t, err)
+
+	if _, ok := err.(*UnsupposedValueError); !ok {
+		t.Fatalf("expected *ErrUnsupportedValue, got %v", reflect.TypeOf(err))
+	}
+}
+
+func TestFilterMatchDefault(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement*"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("miss.servers.localhost.cpu_load",
+		map[string]string{},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("miss.servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchMultipleMeasurement(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement.measurement*"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu.cpu_load.10",
+		map[string]string{"host": "localhost"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu.cpu_load.10 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchMultipleMeasurementSeparator(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"servers.localhost .host.measurement.measurement*"},
+		nil,
+	)
+	assert.NoError(t, err)
+
+	exp, err := telegraf.NewMetric("cpu_cpu_load_10",
+		map[string]string{"host": "localhost"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu.cpu_load.10 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchSingle(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement*"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestParseNoMatch(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.*.cpu .host.measurement.cpu.measurement"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("servers.localhost.memory.VmallocChunk",
+		map[string]string{},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.memory.VmallocChunk 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchWildcard(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.* .host.measurement*"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchExactBeforeWildcard(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{
+		"servers.* .wrong.measurement*",
+		"servers.localhost .host.measurement*"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchMostLongestFilter(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{
+		"*.* .wrong.measurement*",
+		"servers.* .wrong.measurement*",
+		"servers.localhost .wrong.measurement*",
+		"servers.localhost.cpu .host.resource.measurement*", // should match this
+		"*.localhost .wrong.measurement*",
+	}, nil)
+
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost", "resource": "cpu"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestFilterMatchMultipleWildcards(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{
+		"*.* .wrong.measurement*",
+		"servers.* .host.measurement*", // should match this
+		"servers.localhost .wrong.measurement*",
+		"*.localhost .wrong.measurement*",
+	}, nil)
+
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "server01"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.server01.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestParseDefaultTags(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement*"}, map[string]string{
+		"region": "us-east",
+		"zone":   "1c",
+		"host":   "should not set",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestParseDefaultTemplateTags(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement* zone=1c"}, map[string]string{
+		"region": "us-east",
+		"host":   "should not set",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestParseDefaultTemplateTagsOverridGlobal(t *testing.T) {
+	p, err := NewGraphiteParser("", []string{"servers.localhost .host.measurement* zone=1c,region=us-east"}, map[string]string{
+		"region": "shot not be set",
+		"host":   "should not set",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+func TestParseTemplateWhitespace(t *testing.T) {
+	p, err := NewGraphiteParser("",
+		[]string{"servers.localhost        .host.measurement*           zone=1c"},
+		map[string]string{
+			"region": "us-east",
+			"host":   "should not set",
+		})
+	if err != nil {
+		t.Fatalf("unexpected error creating parser, got %v", err)
+	}
+
+	exp, err := telegraf.NewMetric("cpu_load",
+		map[string]string{"host": "localhost", "region": "us-east", "zone": "1c"},
+		map[string]interface{}{"value": float64(11)},
+		time.Unix(1435077219, 0))
+	assert.NoError(t, err)
+
+	m, err := p.ParseLine("servers.localhost.cpu_load 11 1435077219")
+	assert.NoError(t, err)
+
+	assert.Equal(t, exp.String(), m.String())
+}
+
+// Test basic functionality of ApplyTemplate
+func TestApplyTemplate(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.measurement"},
+		nil)
+	assert.NoError(t, err)
+
+	measurement, _, _, _ := p.ApplyTemplate("current.users")
+	assert.Equal(t, "current_users", measurement)
+}
+
+// Test basic functionality of ApplyTemplate
+func TestApplyTemplateNoMatch(t *testing.T) {
+	p, err := NewGraphiteParser(".",
+		[]string{"foo.bar measurement.measurement"},
+		nil)
+	assert.NoError(t, err)
+
+	measurement, _, _, _ := p.ApplyTemplate("current.users")
+	assert.Equal(t, "current.users", measurement)
+}
+
+// Test that most specific template is chosen
+func TestApplyTemplateSpecific(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{
+			"current.* measurement.measurement",
+			"current.*.* measurement.measurement.service",
+		}, nil)
+	assert.NoError(t, err)
+
+	measurement, tags, _, _ := p.ApplyTemplate("current.users.facebook")
+	assert.Equal(t, "current_users", measurement)
+
+	service, ok := tags["service"]
+	if !ok {
+		t.Error("Expected for template to apply a 'service' tag, but not found")
+	}
+	if service != "facebook" {
+		t.Errorf("Expected service='facebook' tag, got service='%s'", service)
+	}
+}
+
+func TestApplyTemplateTags(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.measurement region=us-west"}, nil)
+	assert.NoError(t, err)
+
+	measurement, tags, _, _ := p.ApplyTemplate("current.users")
+	assert.Equal(t, "current_users", measurement)
+
+	region, ok := tags["region"]
+	if !ok {
+		t.Error("Expected for template to apply a 'region' tag, but not found")
+	}
+	if region != "us-west" {
+		t.Errorf("Expected region='us-west' tag, got region='%s'", region)
+	}
+}
+
+func TestApplyTemplateField(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.measurement.field"}, nil)
+	assert.NoError(t, err)
+
+	measurement, _, field, err := p.ApplyTemplate("current.users.logged_in")
+
+	assert.Equal(t, "current_users", measurement)
+
+	if field != "logged_in" {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+			field, "logged_in")
+	}
+}
+
+func TestApplyTemplateFieldError(t *testing.T) {
+	p, err := NewGraphiteParser("_",
+		[]string{"current.* measurement.field.field"}, nil)
+	assert.NoError(t, err)
+
+	_, _, _, err = p.ApplyTemplate("current.users.logged_in")
+	if err == nil {
+		t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s", err,
+			"'field' can only be used once in each template: current.users.logged_in")
+	}
+}
+
+// Test Helpers
+func errstr(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}