Add metric pass/drop filter

agent/accumulator.go

@@ -43,6 +43,11 @@ func (ac *accumulator) Add(
 ) {
 	fields := make(map[string]interface{})
 	fields["value"] = value
+
+	if !ac.inputConfig.Filter.ShouldNamePass(measurement) {
+		return
+	}
+
 	ac.AddFields(measurement, fields, tags, t...)
 }
 
@@ -56,6 +61,10 @@ func (ac *accumulator) AddFields(
 		return
 	}
 
+	if !ac.inputConfig.Filter.ShouldNamePass(measurement) {
+		return
+	}
+
 	if !ac.inputConfig.Filter.ShouldTagsPass(tags) {
 		return
 	}
@@ -92,7 +101,7 @@ func (ac *accumulator) AddFields(
 	for k, v := range fields {
 		// Filter out any filtered fields
 		if ac.inputConfig != nil {
-			if !ac.inputConfig.Filter.ShouldPass(k) {
+			if !ac.inputConfig.Filter.ShouldFieldsPass(k) {
 				continue
 			}
 		}

docs/CONFIGURATION.md

@@ -58,10 +58,14 @@ you can configure that here.
 
 There are also filters that can be configured per input:
 
-* **pass**: An array of strings that is used to filter metrics generated by the
+* **namepass**: An array of strings that is used to filter metrics generated by the
+current input. Each string in the array is tested as a glob match against
+measurement names and if it matches, the field is emitted.
+* **namedrop**: The inverse of pass, if a measurement name matches, it is not emitted.
+* **fieldpass**: An array of strings that is used to filter metrics generated by the
 current input. Each string in the array is tested as a glob match against field names
 and if it matches, the field is emitted.
-* **drop**: The inverse of pass, if a field name matches, it is not emitted.
+* **fielddrop**: The inverse of pass, if a field name matches, it is not emitted.
 * **tagpass**: tag names and arrays of strings that are used to filter
 measurements by the current input. Each string in the array is tested as a glob
 match against the tag name, and if it matches the measurement is emitted.
@@ -117,18 +121,32 @@ fields which begin with `time_`.
     path = [ "/opt", "/home*" ]
 ```
 
-#### Input Config: pass and drop
+#### Input Config: fieldpass and fielddrop
 
 ```toml
 # Drop all metrics for guest & steal CPU usage
 [[inputs.cpu]]
   percpu = false
   totalcpu = true
-  drop = ["usage_guest", "usage_steal"]
+  fielddrop = ["usage_guest", "usage_steal"]
 
 # Only store inode related metrics for disks
 [[inputs.disk]]
-  pass = ["inodes*"]
+  fieldpass = ["inodes*"]
+```
+
+#### Input Config: namepass and namedrop
+
+```toml
+# Drop all metrics about containers for kubelet
+[[inputs.prometheus]]
+  urls = ["http://kube-node-1:4194/metrics"]
+  namedrop = ["container_"]
+
+# Only store rest client related metrics for kubelet
+[[inputs.prometheus]]
+  urls = ["http://kube-node-1:4194/metrics"]
+  namepass = ["rest_client_"]
 ```
 
 #### Input config: prefix, suffix, and override

internal/config/config.go

@@ -483,12 +483,12 @@ func (c *Config) addInput(name string, table *ast.Table) error {
 func buildFilter(tbl *ast.Table) internal_models.Filter {
 	f := internal_models.Filter{}
 
-	if node, ok := tbl.Fields["pass"]; ok {
+	if node, ok := tbl.Fields["namepass"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
 			if ary, ok := kv.Value.(*ast.Array); ok {
 				for _, elem := range ary.Value {
 					if str, ok := elem.(*ast.String); ok {
-						f.Pass = append(f.Pass, str.Value)
+						f.NamePass = append(f.NamePass, str.Value)
 						f.IsActive = true
 					}
 				}
@@ -496,12 +496,12 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 		}
 	}
 
-	if node, ok := tbl.Fields["drop"]; ok {
+	if node, ok := tbl.Fields["namedrop"]; ok {
 		if kv, ok := node.(*ast.KeyValue); ok {
 			if ary, ok := kv.Value.(*ast.Array); ok {
 				for _, elem := range ary.Value {
 					if str, ok := elem.(*ast.String); ok {
-						f.Drop = append(f.Drop, str.Value)
+						f.NameDrop = append(f.NameDrop, str.Value)
 						f.IsActive = true
 					}
 				}
@@ -509,6 +509,38 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 		}
 	}
 
+	fields := []string{"pass", "fieldpass"}
+	for _, field := range fields {
+		if node, ok := tbl.Fields[field]; ok {
+			if kv, ok := node.(*ast.KeyValue); ok {
+				if ary, ok := kv.Value.(*ast.Array); ok {
+					for _, elem := range ary.Value {
+						if str, ok := elem.(*ast.String); ok {
+							f.FieldPass = append(f.FieldPass, str.Value)
+							f.IsActive = true
+						}
+					}
+				}
+			}
+		}
+	}
+
+	fields = []string{"drop", "fielddrop"}
+	for _, field := range fields {
+		if node, ok := tbl.Fields[field]; ok {
+			if kv, ok := node.(*ast.KeyValue); ok {
+				if ary, ok := kv.Value.(*ast.Array); ok {
+					for _, elem := range ary.Value {
+						if str, ok := elem.(*ast.String); ok {
+							f.FieldDrop = append(f.FieldDrop, str.Value)
+							f.IsActive = true
+						}
+					}
+				}
+			}
+		}
+	}
+
 	if node, ok := tbl.Fields["tagpass"]; ok {
 		if subtbl, ok := node.(*ast.Table); ok {
 			for name, val := range subtbl.Fields {
@@ -547,6 +579,10 @@ func buildFilter(tbl *ast.Table) internal_models.Filter {
 		}
 	}
 
+	delete(tbl.Fields, "namedrop")
+	delete(tbl.Fields, "namepass")
+	delete(tbl.Fields, "fielddrop")
+	delete(tbl.Fields, "fieldpass")
 	delete(tbl.Fields, "drop")
 	delete(tbl.Fields, "pass")
 	delete(tbl.Fields, "tagdrop")

internal/config/config_test.go

@@ -23,8 +23,10 @@ func TestConfig_LoadSingleInput(t *testing.T) {
 	mConfig := &internal_models.InputConfig{
 		Name: "memcached",
 		Filter: internal_models.Filter{
-			Drop: []string{"other", "stuff"},
+			NameDrop:  []string{"metricname2"},
-			Pass: []string{"some", "strings"},
+			NamePass:  []string{"metricname1"},
+			FieldDrop: []string{"other", "stuff"},
+			FieldPass: []string{"some", "strings"},
 			TagDrop: []internal_models.TagFilter{
 				internal_models.TagFilter{
 					Name:   "badtag",
@@ -66,8 +68,10 @@ func TestConfig_LoadDirectory(t *testing.T) {
 	mConfig := &internal_models.InputConfig{
 		Name: "memcached",
 		Filter: internal_models.Filter{
-			Drop: []string{"other", "stuff"},
+			NameDrop:  []string{"metricname2"},
-			Pass: []string{"some", "strings"},
+			NamePass:  []string{"metricname1"},
+			FieldDrop: []string{"other", "stuff"},
+			FieldPass: []string{"some", "strings"},
 			TagDrop: []internal_models.TagFilter{
 				internal_models.TagFilter{
 					Name:   "badtag",

internal/config/testdata/single_plugin.toml

@@ -1,7 +1,9 @@
 [[inputs.memcached]]
   servers = ["localhost"]
-  pass = ["some", "strings"]
+  namepass = ["metricname1"]
-  drop = ["other", "stuff"]
+  namedrop = ["metricname2"]
+  fieldpass = ["some", "strings"]
+  fielddrop = ["other", "stuff"]
   interval = "5s"
   [inputs.memcached.tagpass]
     goodtag = ["mytag"]

internal/config/testdata/subconfig/memcached.conf

@@ -1,5 +1,7 @@
 [[inputs.memcached]]
   servers = ["192.168.1.1"]
+  namepass = ["metricname1"]
+  namedrop = ["metricname2"]
   pass = ["some", "strings"]
   drop = ["other", "stuff"]
   interval = "5s"

internal/models/filter.go

@@ -15,8 +15,11 @@ type TagFilter struct {
 
 // Filter containing drop/pass and tagdrop/tagpass rules
 type Filter struct {
-	Drop []string
+	NameDrop []string
-	Pass []string
+	NamePass []string
+
+	FieldDrop []string
+	FieldPass []string
 
 	TagDrop []TagFilter
 	TagPass []TagFilter
@@ -25,17 +28,17 @@ type Filter struct {
 }
 
 func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool {
-	if f.ShouldPass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
+	if f.ShouldFieldsPass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
 		return true
 	}
 	return false
 }
 
-// ShouldPass returns true if the metric should pass, false if should drop
+// ShouldFieldsPass returns true if the metric should pass, false if should drop
 // based on the drop/pass filter parameters
-func (f Filter) ShouldPass(key string) bool {
+func (f Filter) ShouldNamePass(key string) bool {
-	if f.Pass != nil {
+	if f.NamePass != nil {
-		for _, pat := range f.Pass {
+		for _, pat := range f.NamePass {
 			// TODO remove HasPrefix check, leaving it for now for legacy support.
 			// Cam, 2015-12-07
 			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
@@ -45,8 +48,36 @@ func (f Filter) ShouldPass(key string) bool {
 		return false
 	}
 
-	if f.Drop != nil {
+	if f.NameDrop != nil {
-		for _, pat := range f.Drop {
+		for _, pat := range f.NameDrop {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
+				return false
+			}
+		}
+
+		return true
+	}
+	return true
+}
+
+// ShouldFieldsPass returns true if the metric should pass, false if should drop
+// based on the drop/pass filter parameters
+func (f Filter) ShouldFieldsPass(key string) bool {
+	if f.FieldPass != nil {
+		for _, pat := range f.FieldPass {
+			// TODO remove HasPrefix check, leaving it for now for legacy support.
+			// Cam, 2015-12-07
+			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if f.FieldDrop != nil {
+		for _, pat := range f.FieldDrop {
 			// TODO remove HasPrefix check, leaving it for now for legacy support.
 			// Cam, 2015-12-07
 			if strings.HasPrefix(key, pat) || internal.Glob(pat, key) {

internal/models/filter_test.go

@@ -18,15 +18,15 @@ func TestFilter_Empty(t *testing.T) {
 	}
 
 	for _, measurement := range measurements {
-		if !f.ShouldPass(measurement) {
+		if !f.ShouldFieldsPass(measurement) {
 			t.Errorf("Expected measurement %s to pass", measurement)
 		}
 	}
 }
 
-func TestFilter_Pass(t *testing.T) {
+func TestFilter_NamePass(t *testing.T) {
 	f := Filter{
-		Pass: []string{"foo*", "cpu_usage_idle"},
+		NamePass: []string{"foo*", "cpu_usage_idle"},
 	}
 
 	passes := []string{
@@ -45,21 +45,21 @@ func TestFilter_Pass(t *testing.T) {
 	}
 
 	for _, measurement := range passes {
-		if !f.ShouldPass(measurement) {
+		if !f.ShouldNamePass(measurement) {
 			t.Errorf("Expected measurement %s to pass", measurement)
 		}
 	}
 
 	for _, measurement := range drops {
-		if f.ShouldPass(measurement) {
+		if f.ShouldNamePass(measurement) {
 			t.Errorf("Expected measurement %s to drop", measurement)
 		}
 	}
 }
 
-func TestFilter_Drop(t *testing.T) {
+func TestFilter_NameDrop(t *testing.T) {
 	f := Filter{
-		Drop: []string{"foo*", "cpu_usage_idle"},
+		NameDrop: []string{"foo*", "cpu_usage_idle"},
 	}
 
 	drops := []string{
@@ -78,13 +78,79 @@ func TestFilter_Drop(t *testing.T) {
 	}
 
 	for _, measurement := range passes {
-		if !f.ShouldPass(measurement) {
+		if !f.ShouldNamePass(measurement) {
 			t.Errorf("Expected measurement %s to pass", measurement)
 		}
 	}
 
 	for _, measurement := range drops {
-		if f.ShouldPass(measurement) {
+		if f.ShouldNamePass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_FieldPass(t *testing.T) {
+	f := Filter{
+		FieldPass: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	passes := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	drops := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldFieldsPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldFieldsPass(measurement) {
+			t.Errorf("Expected measurement %s to drop", measurement)
+		}
+	}
+}
+
+func TestFilter_FieldDrop(t *testing.T) {
+	f := Filter{
+		FieldDrop: []string{"foo*", "cpu_usage_idle"},
+	}
+
+	drops := []string{
+		"foo",
+		"foo_bar",
+		"foo.bar",
+		"foo-bar",
+		"cpu_usage_idle",
+	}
+
+	passes := []string{
+		"bar",
+		"barfoo",
+		"bar_foo",
+		"cpu_usage_busy",
+	}
+
+	for _, measurement := range passes {
+		if !f.ShouldFieldsPass(measurement) {
+			t.Errorf("Expected measurement %s to pass", measurement)
+		}
+	}
+
+	for _, measurement := range drops {
+		if f.ShouldFieldsPass(measurement) {
 			t.Errorf("Expected measurement %s to drop", measurement)
 		}
 	}