Implement Glob matching for pass/drop filters

LICENSE_OF_DEPENDENCIES.md

@@ -29,4 +29,5 @@
 - gopkg.in/dancannon/gorethink.v1 [APACHE LICENSE](https://github.com/dancannon/gorethink/blob/v1.1.2/LICENSE)
 - gopkg.in/mgo.v2 [BSD LICENSE](https://github.com/go-mgo/mgo/blob/v2/LICENSE)
 - golang.org/x/crypto/* [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
+- internal Glob function [MIT LICENSE](https://github.com/ryanuber/go-glob/blob/master/LICENSE)
 

README.md

@@ -94,13 +94,12 @@ InfluxDB.
 There are 5 configuration options that are configurable per plugin:
 
 * **pass**: An array of strings that is used to filter metrics generated by the
-current plugin. Each string in the array is tested as a prefix against metric names
+current plugin. Each string in the array is tested as a glob match against metric names
 and if it matches, the metric is emitted.
 * **drop**: The inverse of pass, if a metric name matches, it is not emitted.
-* **tagpass**: (added in 0.1.5) tag names and arrays of strings that are used to filter metrics by
+* **tagpass**: tag names and arrays of strings that are used to filter metrics by the current plugin. Each string in the array is tested as a glob match against
-the current plugin. Each string in the array is tested as an exact match against
 the tag name, and if it matches the metric is emitted.
-* **tagdrop**: (added in 0.1.5) The inverse of tagpass. If a tag matches, the metric is not emitted.
+* **tagdrop**: The inverse of tagpass. If a tag matches, the metric is not emitted.
 This is tested on metrics that have passed the tagpass test.
 * **interval**: How often to gather this metric. Normal plugins use a single
 global interval, but if one particular plugin should be run less or more often,
@@ -132,10 +131,10 @@ measurements which begin with `cpu_time`.
 [[plugins.cpu]]
   percpu = true
   totalcpu = false
-  drop = ["cpu_time"]
+  drop = ["cpu_time*"]
 ```
 
-Below is how to configure `tagpass` and `tagdrop` parameters (added in 0.1.5)
+Below is how to configure `tagpass` and `tagdrop` parameters
 
 ```toml
 [plugins]
@@ -153,10 +152,11 @@ Below is how to configure `tagpass` and `tagdrop` parameters (added in 0.1.5)
     # If the (filesystem is ext4 or xfs) OR (the path is /opt or /home)
     # then the metric passes
     fstype = [ "ext4", "xfs" ]
-    path = [ "/opt", "/home" ]
+    # Globs can also be used on the tag values
+    path = [ "/opt", "/home*" ]
 ```
 
-Below is how to configure `pass` and `drop` parameters (added in 0.1.5)
+Below is how to configure `pass` and `drop` parameters
 
 ```toml
 # Drop all metrics for guest CPU usage
@@ -165,17 +165,22 @@ Below is how to configure `pass` and `drop` parameters (added in 0.1.5)
 
 # Only store inode related metrics for disks
 [[plugins.disk]]
-  pass = [ "disk_inodes" ]
+  pass = [ "disk_inodes*" ]
 ```
 
 
 Additional plugins (or outputs) of the same type can be specified,
-just define another instance in the config file:
+just define more instances in the config file:
 
 ```toml
 [[plugins.cpu]]
   percpu = false
   totalcpu = true
+
+[[plugins.cpu]]
+  percpu = true
+  totalcpu = false
+  drop = ["cpu_time*"]
 ```
 
 ## Supported Plugins
@@ -246,14 +251,14 @@ Outputs also support the same configurable options as plugins (pass, drop, tagpa
   database = "telegraf"
   precision = "s"
   # Drop all measurements that start with "aerospike"
-  drop = ["aerospike"]
+  drop = ["aerospike*"]
 
 [[outputs.influxdb]]
   urls = [ "http://localhost:8086" ]
   database = "telegraf-aerospike-data"
   precision = "s"
   # Only accept aerospike data:
-  pass = ["aerospike"]
+  pass = ["aerospike*"]
 
 [[outputs.influxdb]]
   urls = [ "http://localhost:8086" ]

internal/config/config.go

@@ -145,7 +145,9 @@ func (ro *RunningOutput) FilterPoints(points []*client.Point) []*client.Point {
 func (f Filter) ShouldPass(measurement string) bool {
 	if f.Pass != nil {
 		for _, pat := range f.Pass {
-			if strings.HasPrefix(measurement, pat) {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(measurement, pat) || internal.Glob(pat, measurement) {
 				return true
 			}
 		}
@@ -154,7 +156,9 @@ func (f Filter) ShouldPass(measurement string) bool {
 
 	if f.Drop != nil {
 		for _, pat := range f.Drop {
-			if strings.HasPrefix(measurement, pat) {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(measurement, pat) || internal.Glob(pat, measurement) {
 				return false
 			}
 		}
@@ -171,7 +175,7 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool {
 		for _, pat := range f.TagPass {
 			if tagval, ok := tags[pat.Name]; ok {
 				for _, filter := range pat.Filter {
-					if filter == tagval {
+					if internal.Glob(filter, tagval) {
 						return true
 					}
 				}
@@ -184,7 +188,7 @@ func (f Filter) ShouldTagsPass(tags map[string]string) bool {
 		for _, pat := range f.TagDrop {
 			if tagval, ok := tags[pat.Name]; ok {
 				for _, filter := range pat.Filter {
-					if filter == tagval {
+					if internal.Glob(filter, tagval) {
 						return false
 					}
 				}

internal/config/config_test.go

@@ -122,3 +122,175 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	assert.Equal(t, pConfig, c.Plugins[3].Config,
 		"Merged Testdata did not produce correct procstat metadata.")
 }
+
+func TestFilter_Empty(t *testing.T) {
+	f := Filter{}
+
+	measurements := []string{
+		"foo",
+		"bar",
+		"barfoo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"supercalifradjulisticexpialidocious",
+	}
+
+	for _, measurement := range measurements {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+}
+
+func TestFilter_Pass(t *testing.T) {
+	f := Filter{
+		Pass: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	passes := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	drops := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_Drop(t *testing.T) {
+	f := Filter{
+		Drop: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	drops := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	passes := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldPass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_TagPass(t *testing.T) {
+	filters := []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"cpu-*"},
+		},
+		TagFilter{
+			Name:   "mem",
+			Filter: []string{"mem_free"},
+		}}
+	f := Filter{
+		TagPass: filters,
+	}
+
+	passes := []map[string]string{
+		{"cpu": "cpu-total"},
+		{"cpu": "cpu-0"},
+		{"cpu": "cpu-1"},
+		{"cpu": "cpu-2"},
+		{"mem": "mem_free"},
+	}
+
+	drops := []map[string]string{
+		{"cpu": "cputotal"},
+		{"cpu": "cpu0"},
+		{"cpu": "cpu1"},
+		{"cpu": "cpu2"},
+		{"mem": "mem_used"},
+	}
+
+	for _, tags := range passes {
+		if !f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to pass", tags)
+		}
+	}
+
+	for _, tags := range drops {
+		if f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to drop", tags)
+		}
+	}
+}
+
+func TestFilter_TagDrop(t *testing.T) {
+	filters := []TagFilter{
+		TagFilter{
+			Name:   "cpu",
+			Filter: []string{"cpu-*"},
+		},
+		TagFilter{
+			Name:   "mem",
+			Filter: []string{"mem_free"},
+		}}
+	f := Filter{
+		TagDrop: filters,
+	}
+
+	drops := []map[string]string{
+		{"cpu": "cpu-total"},
+		{"cpu": "cpu-0"},
+		{"cpu": "cpu-1"},
+		{"cpu": "cpu-2"},
+		{"mem": "mem_free"},
+	}
+
+	passes := []map[string]string{
+		{"cpu": "cputotal"},
+		{"cpu": "cpu0"},
+		{"cpu": "cpu1"},
+		{"cpu": "cpu2"},
+		{"mem": "mem_used"},
+	}
+
+	for _, tags := range passes {
+		if !f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to pass", tags)
+		}
+	}
+
+	for _, tags := range drops {
+		if f.ShouldTagsPass(tags) {
+			t.Errorf("Expected tags %v to drop", tags)
+		}
+	}
+}

internal/internal.go

@@ -61,3 +61,59 @@ func ReadLinesOffsetN(filename string, offset uint, n int) ([]string, error) {
 
 	return ret, nil
 }
+
+// Glob will test a string pattern, potentially containing globs, against a
+// subject string. The result is a simple true/false, determining whether or
+// not the glob pattern matched the subject text.
+//
+// Adapted from https://github.com/ryanuber/go-glob/blob/master/glob.go
+// thanks Ryan Uber!
+func Glob(pattern, measurement string) bool {
+	// Empty pattern can only match empty subject
+	if pattern == "" {
+		return measurement == pattern
+	}
+
+	// If the pattern _is_ a glob, it matches everything
+	if pattern == "*" {
+		return true
+	}
+
+	parts := strings.Split(pattern, "*")
+
+	if len(parts) == 1 {
+		// No globs in pattern, so test for match
+		return pattern == measurement
+	}
+
+	leadingGlob := strings.HasPrefix(pattern, "*")
+	trailingGlob := strings.HasSuffix(pattern, "*")
+	end := len(parts) - 1
+
+	for i, part := range parts {
+		switch i {
+		case 0:
+			if leadingGlob {
+				continue
+			}
+			if !strings.HasPrefix(measurement, part) {
+				return false
+			}
+		case end:
+			if len(measurement) > 0 {
+				return trailingGlob || strings.HasSuffix(measurement, part)
+			}
+		default:
+			if !strings.Contains(measurement, part) {
+				return false
+			}
+		}
+
+		// Trim evaluated text from measurement as we loop over the pattern.
+		idx := strings.Index(measurement, part) + len(part)
+		measurement = measurement[idx:]
+	}
+
+	// All parts of the pattern matched
+	return true
+}

internal/internal_test.go

@@ -0,0 +1,44 @@
+package internal
+
+import "testing"
+
+func testGlobMatch(t *testing.T, pattern, subj string) {
+	if !Glob(pattern, subj) {
+		t.Errorf("%s should match %s", pattern, subj)
+	}
+}
+
+func testGlobNoMatch(t *testing.T, pattern, subj string) {
+	if Glob(pattern, subj) {
+		t.Errorf("%s should not match %s", pattern, subj)
+	}
+}
+
+func TestEmptyPattern(t *testing.T) {
+	testGlobMatch(t, "", "")
+	testGlobNoMatch(t, "", "test")
+}
+
+func TestPatternWithoutGlobs(t *testing.T) {
+	testGlobMatch(t, "test", "test")
+}
+
+func TestGlob(t *testing.T) {
+	for _, pattern := range []string{
+		"*test",           // Leading glob
+		"this*",           // Trailing glob
+		"*is*a*",          // Lots of globs
+		"**test**",        // Double glob characters
+		"**is**a***test*", // Varying number of globs
+	} {
+		testGlobMatch(t, pattern, "this_is_a_test")
+	}
+
+	for _, pattern := range []string{
+		"test*", // Implicit substring match should fail
+		"*is",   // Partial match should fail
+		"*no*",  // Globs without a match between them should fail
+	} {
+		testGlobNoMatch(t, pattern, "this_is_a_test")
+	}
+}

plugins/system/cpu.go

@@ -31,7 +31,7 @@ var sampleConfig = `
   # Whether to report total system cpu stats or not
   totalcpu = true
   # Comment this line if you want the raw CPU time metrics
-  drop = ["cpu_time"]
+  drop = ["cpu_time*"]
 `
 
 func (_ *CPUStats) SampleConfig() string {