From 217946005fb20070876c68362acec8e9c05ef7da Mon Sep 17 00:00:00 2001 From: Thibault Cohen Date: Sat, 20 Feb 2016 00:35:12 -0500 Subject: [PATCH] Add metric pass/drop filter --- agent/accumulator.go | 11 ++- docs/CONFIGURATION.md | 28 +++++-- internal/config/config.go | 44 +++++++++- internal/config/config_test.go | 12 ++- internal/config/testdata/single_plugin.toml | 6 +- .../config/testdata/subconfig/memcached.conf | 2 + internal/models/filter.go | 49 +++++++++-- internal/models/filter_test.go | 84 +++++++++++++++++-- 8 files changed, 202 insertions(+), 34 deletions(-) diff --git a/agent/accumulator.go b/agent/accumulator.go index 9361ad82e..b04ff2b53 100644 --- a/agent/accumulator.go +++ b/agent/accumulator.go @@ -43,6 +43,11 @@ func (ac *accumulator) Add( ) { fields := make(map[string]interface{}) fields["value"] = value + + if !ac.inputConfig.Filter.ShouldNamePass(measurement) { + return + } + ac.AddFields(measurement, fields, tags, t...) } @@ -56,6 +61,10 @@ func (ac *accumulator) AddFields( return } + if !ac.inputConfig.Filter.ShouldNamePass(measurement) { + return + } + if !ac.inputConfig.Filter.ShouldTagsPass(tags) { return } @@ -92,7 +101,7 @@ func (ac *accumulator) AddFields( for k, v := range fields { // Filter out any filtered fields if ac.inputConfig != nil { - if !ac.inputConfig.Filter.ShouldPass(k) { + if !ac.inputConfig.Filter.ShouldFieldsPass(k) { continue } } diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f4214b5d4..a35c98efc 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -58,10 +58,14 @@ you can configure that here. There are also filters that can be configured per input: -* **pass**: An array of strings that is used to filter metrics generated by the +* **namepass**: An array of strings that is used to filter metrics generated by the +current input. Each string in the array is tested as a glob match against +measurement names and if it matches, the field is emitted. +* **namedrop**: The inverse of pass, if a measurement name matches, it is not emitted. +* **fieldpass**: An array of strings that is used to filter metrics generated by the current input. Each string in the array is tested as a glob match against field names and if it matches, the field is emitted. -* **drop**: The inverse of pass, if a field name matches, it is not emitted. +* **fielddrop**: The inverse of pass, if a field name matches, it is not emitted. * **tagpass**: tag names and arrays of strings that are used to filter measurements by the current input. Each string in the array is tested as a glob match against the tag name, and if it matches the measurement is emitted. @@ -117,18 +121,32 @@ fields which begin with `time_`. path = [ "/opt", "/home*" ] ``` -#### Input Config: pass and drop +#### Input Config: fieldpass and fielddrop ```toml # Drop all metrics for guest & steal CPU usage [[inputs.cpu]] percpu = false totalcpu = true - drop = ["usage_guest", "usage_steal"] + fielddrop = ["usage_guest", "usage_steal"] # Only store inode related metrics for disks [[inputs.disk]] - pass = ["inodes*"] + fieldpass = ["inodes*"] +``` + +#### Input Config: namepass and namedrop + +```toml +# Drop all metrics about containers for kubelet +[[inputs.prometheus]] + urls = ["http://kube-node-1:4194/metrics"] + namedrop = ["container_"] + +# Only store rest client related metrics for kubelet +[[inputs.prometheus]] + urls = ["http://kube-node-1:4194/metrics"] + namepass = ["rest_client_"] ``` #### Input config: prefix, suffix, and override diff --git a/internal/config/config.go b/internal/config/config.go index fc374d628..48322f238 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -484,12 +484,12 @@ func (c *Config) addInput(name string, table *ast.Table) error { func buildFilter(tbl *ast.Table) internal_models.Filter { f := internal_models.Filter{} - if node, ok := tbl.Fields["pass"]; ok { + if node, ok := tbl.Fields["namepass"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if ary, ok := kv.Value.(*ast.Array); ok { for _, elem := range ary.Value { if str, ok := elem.(*ast.String); ok { - f.Pass = append(f.Pass, str.Value) + f.NamePass = append(f.NamePass, str.Value) f.IsActive = true } } @@ -497,12 +497,12 @@ func buildFilter(tbl *ast.Table) internal_models.Filter { } } - if node, ok := tbl.Fields["drop"]; ok { + if node, ok := tbl.Fields["namedrop"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if ary, ok := kv.Value.(*ast.Array); ok { for _, elem := range ary.Value { if str, ok := elem.(*ast.String); ok { - f.Drop = append(f.Drop, str.Value) + f.NameDrop = append(f.NameDrop, str.Value) f.IsActive = true } } @@ -510,6 +510,38 @@ func buildFilter(tbl *ast.Table) internal_models.Filter { } } + fields := []string{"pass", "fieldpass"} + for _, field := range fields { + if node, ok := tbl.Fields[field]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + f.FieldPass = append(f.FieldPass, str.Value) + f.IsActive = true + } + } + } + } + } + } + + fields = []string{"drop", "fielddrop"} + for _, field := range fields { + if node, ok := tbl.Fields[field]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if ary, ok := kv.Value.(*ast.Array); ok { + for _, elem := range ary.Value { + if str, ok := elem.(*ast.String); ok { + f.FieldDrop = append(f.FieldDrop, str.Value) + f.IsActive = true + } + } + } + } + } + } + if node, ok := tbl.Fields["tagpass"]; ok { if subtbl, ok := node.(*ast.Table); ok { for name, val := range subtbl.Fields { @@ -548,6 +580,10 @@ func buildFilter(tbl *ast.Table) internal_models.Filter { } } + delete(tbl.Fields, "namedrop") + delete(tbl.Fields, "namepass") + delete(tbl.Fields, "fielddrop") + delete(tbl.Fields, "fieldpass") delete(tbl.Fields, "drop") delete(tbl.Fields, "pass") delete(tbl.Fields, "tagdrop") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0e9f2c967..f0add8b98 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -23,8 +23,10 @@ func TestConfig_LoadSingleInput(t *testing.T) { mConfig := &internal_models.InputConfig{ Name: "memcached", Filter: internal_models.Filter{ - Drop: []string{"other", "stuff"}, - Pass: []string{"some", "strings"}, + NameDrop: []string{"metricname2"}, + NamePass: []string{"metricname1"}, + FieldDrop: []string{"other", "stuff"}, + FieldPass: []string{"some", "strings"}, TagDrop: []internal_models.TagFilter{ internal_models.TagFilter{ Name: "badtag", @@ -66,8 +68,10 @@ func TestConfig_LoadDirectory(t *testing.T) { mConfig := &internal_models.InputConfig{ Name: "memcached", Filter: internal_models.Filter{ - Drop: []string{"other", "stuff"}, - Pass: []string{"some", "strings"}, + NameDrop: []string{"metricname2"}, + NamePass: []string{"metricname1"}, + FieldDrop: []string{"other", "stuff"}, + FieldPass: []string{"some", "strings"}, TagDrop: []internal_models.TagFilter{ internal_models.TagFilter{ Name: "badtag", diff --git a/internal/config/testdata/single_plugin.toml b/internal/config/testdata/single_plugin.toml index 6670f6b2f..664937b25 100644 --- a/internal/config/testdata/single_plugin.toml +++ b/internal/config/testdata/single_plugin.toml @@ -1,7 +1,9 @@ [[inputs.memcached]] servers = ["localhost"] - pass = ["some", "strings"] - drop = ["other", "stuff"] + namepass = ["metricname1"] + namedrop = ["metricname2"] + fieldpass = ["some", "strings"] + fielddrop = ["other", "stuff"] interval = "5s" [inputs.memcached.tagpass] goodtag = ["mytag"] diff --git a/internal/config/testdata/subconfig/memcached.conf b/internal/config/testdata/subconfig/memcached.conf index 4c43febc7..2cd07d15d 100644 --- a/internal/config/testdata/subconfig/memcached.conf +++ b/internal/config/testdata/subconfig/memcached.conf @@ -1,5 +1,7 @@ [[inputs.memcached]] servers = ["192.168.1.1"] + namepass = ["metricname1"] + namedrop = ["metricname2"] pass = ["some", "strings"] drop = ["other", "stuff"] interval = "5s" diff --git a/internal/models/filter.go b/internal/models/filter.go index 9b4f2ba90..48143d3ac 100644 --- a/internal/models/filter.go +++ b/internal/models/filter.go @@ -15,8 +15,11 @@ type TagFilter struct { // Filter containing drop/pass and tagdrop/tagpass rules type Filter struct { - Drop []string - Pass []string + NameDrop []string + NamePass []string + + FieldDrop []string + FieldPass []string TagDrop []TagFilter TagPass []TagFilter @@ -25,17 +28,17 @@ type Filter struct { } func (f Filter) ShouldMetricPass(metric telegraf.Metric) bool { - if f.ShouldPass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) { + if f.ShouldFieldsPass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) { return true } return false } -// ShouldPass returns true if the metric should pass, false if should drop +// ShouldFieldsPass returns true if the metric should pass, false if should drop // based on the drop/pass filter parameters -func (f Filter) ShouldPass(key string) bool { - if f.Pass != nil { - for _, pat := range f.Pass { +func (f Filter) ShouldNamePass(key string) bool { + if f.NamePass != nil { + for _, pat := range f.NamePass { // TODO remove HasPrefix check, leaving it for now for legacy support. // Cam, 2015-12-07 if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { @@ -45,8 +48,36 @@ func (f Filter) ShouldPass(key string) bool { return false } - if f.Drop != nil { - for _, pat := range f.Drop { + if f.NameDrop != nil { + for _, pat := range f.NameDrop { + // TODO remove HasPrefix check, leaving it for now for legacy support. + // Cam, 2015-12-07 + if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { + return false + } + } + + return true + } + return true +} + +// ShouldFieldsPass returns true if the metric should pass, false if should drop +// based on the drop/pass filter parameters +func (f Filter) ShouldFieldsPass(key string) bool { + if f.FieldPass != nil { + for _, pat := range f.FieldPass { + // TODO remove HasPrefix check, leaving it for now for legacy support. + // Cam, 2015-12-07 + if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { + return true + } + } + return false + } + + if f.FieldDrop != nil { + for _, pat := range f.FieldDrop { // TODO remove HasPrefix check, leaving it for now for legacy support. // Cam, 2015-12-07 if strings.HasPrefix(key, pat) || internal.Glob(pat, key) { diff --git a/internal/models/filter_test.go b/internal/models/filter_test.go index 320c38407..c69398494 100644 --- a/internal/models/filter_test.go +++ b/internal/models/filter_test.go @@ -18,15 +18,15 @@ func TestFilter_Empty(t *testing.T) { } for _, measurement := range measurements { - if !f.ShouldPass(measurement) { + if !f.ShouldFieldsPass(measurement) { t.Errorf("Expected measurement %s to pass", measurement) } } } -func TestFilter_Pass(t *testing.T) { +func TestFilter_NamePass(t *testing.T) { f := Filter{ - Pass: []string{"foo*", "cpu_usage_idle"}, + NamePass: []string{"foo*", "cpu_usage_idle"}, } passes := []string{ @@ -45,21 +45,21 @@ func TestFilter_Pass(t *testing.T) { } for _, measurement := range passes { - if !f.ShouldPass(measurement) { + if !f.ShouldNamePass(measurement) { t.Errorf("Expected measurement %s to pass", measurement) } } for _, measurement := range drops { - if f.ShouldPass(measurement) { + if f.ShouldNamePass(measurement) { t.Errorf("Expected measurement %s to drop", measurement) } } } -func TestFilter_Drop(t *testing.T) { +func TestFilter_NameDrop(t *testing.T) { f := Filter{ - Drop: []string{"foo*", "cpu_usage_idle"}, + NameDrop: []string{"foo*", "cpu_usage_idle"}, } drops := []string{ @@ -78,13 +78,79 @@ func TestFilter_Drop(t *testing.T) { } for _, measurement := range passes { - if !f.ShouldPass(measurement) { + if !f.ShouldNamePass(measurement) { t.Errorf("Expected measurement %s to pass", measurement) } } for _, measurement := range drops { - if f.ShouldPass(measurement) { + if f.ShouldNamePass(measurement) { + t.Errorf("Expected measurement %s to drop", measurement) + } + } +} + +func TestFilter_FieldPass(t *testing.T) { + f := Filter{ + FieldPass: []string{"foo*", "cpu_usage_idle"}, + } + + passes := []string{ + "foo", + "foo_bar", + "foo.bar", + "foo-bar", + "cpu_usage_idle", + } + + drops := []string{ + "bar", + "barfoo", + "bar_foo", + "cpu_usage_busy", + } + + for _, measurement := range passes { + if !f.ShouldFieldsPass(measurement) { + t.Errorf("Expected measurement %s to pass", measurement) + } + } + + for _, measurement := range drops { + if f.ShouldFieldsPass(measurement) { + t.Errorf("Expected measurement %s to drop", measurement) + } + } +} + +func TestFilter_FieldDrop(t *testing.T) { + f := Filter{ + FieldDrop: []string{"foo*", "cpu_usage_idle"}, + } + + drops := []string{ + "foo", + "foo_bar", + "foo.bar", + "foo-bar", + "cpu_usage_idle", + } + + passes := []string{ + "bar", + "barfoo", + "bar_foo", + "cpu_usage_busy", + } + + for _, measurement := range passes { + if !f.ShouldFieldsPass(measurement) { + t.Errorf("Expected measurement %s to pass", measurement) + } + } + + for _, measurement := range drops { + if f.ShouldFieldsPass(measurement) { t.Errorf("Expected measurement %s to drop", measurement) } }