diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d4d74ff2..e773a6e82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ time before a new metric is included by the plugin. - [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin. - [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser. - [#1265](https://github.com/influxdata/telegraf/pull/1265): Make dns lookups for chrony configurable. Thanks @zbindenren! +- [#1275](https://github.com/influxdata/telegraf/pull/1275): Allow wildcard filtering of varnish stats. ### Bugfixes diff --git a/Godeps b/Godeps index 42c499fc7..ed28e1470 100644 --- a/Godeps +++ b/Godeps @@ -16,7 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367 github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee -github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa +github.com/gobwas/glob 49571a1557cd20e6a2410adc6421f85b66c730b5 github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2 diff --git a/internal/internal.go b/internal/internal.go index a0c583471..33ee40a26 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -16,6 +16,8 @@ import ( "strings" "time" "unicode" + + "github.com/gobwas/glob" ) const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" @@ -205,3 +207,24 @@ func WaitTimeout(c *exec.Cmd, timeout time.Duration) error { return TimeoutErr } } + +// CompileFilter takes a list of glob "filters", ie: +// ["MAIN.*", "CPU.*", "NET"] +// and compiles them into a glob object. This glob object can +// then be used to match keys to the filter. +func CompileFilter(filters []string) (glob.Glob, error) { + var out glob.Glob + + // return if there is nothing to compile + if len(filters) == 0 { + return out, nil + } + + var err error + if len(filters) == 1 { + out, err = glob.Compile(filters[0]) + } else { + out, err = glob.Compile("{" + strings.Join(filters, ",") + "}") + } + return out, err +} diff --git a/internal/internal_test.go b/internal/internal_test.go index 90e1badc1..341fdd370 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -106,3 +106,34 @@ func TestRunError(t *testing.T) { assert.Error(t, err) } + +func TestCompileFilter(t *testing.T) { + f, err := CompileFilter([]string{}) + assert.NoError(t, err) + assert.Nil(t, f) + + f, err = CompileFilter([]string{"cpu"}) + assert.NoError(t, err) + assert.True(t, f.Match("cpu")) + assert.False(t, f.Match("cpu0")) + assert.False(t, f.Match("mem")) + + f, err = CompileFilter([]string{"cpu*"}) + assert.NoError(t, err) + assert.True(t, f.Match("cpu")) + assert.True(t, f.Match("cpu0")) + assert.False(t, f.Match("mem")) + + f, err = CompileFilter([]string{"cpu", "mem"}) + assert.NoError(t, err) + assert.True(t, f.Match("cpu")) + assert.False(t, f.Match("cpu0")) + assert.True(t, f.Match("mem")) + + f, err = CompileFilter([]string{"cpu", "mem", "net*"}) + assert.NoError(t, err) + assert.True(t, f.Match("cpu")) + assert.False(t, f.Match("cpu0")) + assert.True(t, f.Match("mem")) + assert.True(t, f.Match("network")) +} diff --git a/internal/models/filter.go b/internal/models/filter.go index d78492a5d..71d71c23e 100644 --- a/internal/models/filter.go +++ b/internal/models/filter.go @@ -2,11 +2,11 @@ package internal_models import ( "fmt" - "strings" "github.com/gobwas/glob" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" ) // TagFilter is the name of a tag, and the values on which to filter @@ -42,41 +42,41 @@ type Filter struct { // Compile all Filter lists into glob.Glob objects. func (f *Filter) CompileFilter() error { var err error - f.nameDrop, err = compileFilter(f.NameDrop) + f.nameDrop, err = internal.CompileFilter(f.NameDrop) if err != nil { return fmt.Errorf("Error compiling 'namedrop', %s", err) } - f.namePass, err = compileFilter(f.NamePass) + f.namePass, err = internal.CompileFilter(f.NamePass) if err != nil { return fmt.Errorf("Error compiling 'namepass', %s", err) } - f.fieldDrop, err = compileFilter(f.FieldDrop) + f.fieldDrop, err = internal.CompileFilter(f.FieldDrop) if err != nil { return fmt.Errorf("Error compiling 'fielddrop', %s", err) } - f.fieldPass, err = compileFilter(f.FieldPass) + f.fieldPass, err = internal.CompileFilter(f.FieldPass) if err != nil { return fmt.Errorf("Error compiling 'fieldpass', %s", err) } - f.tagExclude, err = compileFilter(f.TagExclude) + f.tagExclude, err = internal.CompileFilter(f.TagExclude) if err != nil { return fmt.Errorf("Error compiling 'tagexclude', %s", err) } - f.tagInclude, err = compileFilter(f.TagInclude) + f.tagInclude, err = internal.CompileFilter(f.TagInclude) if err != nil { return fmt.Errorf("Error compiling 'taginclude', %s", err) } for i, _ := range f.TagDrop { - f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter) + f.TagDrop[i].filter, err = internal.CompileFilter(f.TagDrop[i].Filter) if err != nil { return fmt.Errorf("Error compiling 'tagdrop', %s", err) } } for i, _ := range f.TagPass { - f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter) + f.TagPass[i].filter, err = internal.CompileFilter(f.TagPass[i].Filter) if err != nil { return fmt.Errorf("Error compiling 'tagpass', %s", err) } @@ -84,20 +84,6 @@ func (f *Filter) CompileFilter() error { return nil } -func compileFilter(filter []string) (glob.Glob, error) { - if len(filter) == 0 { - return nil, nil - } - var g glob.Glob - var err error - if len(filter) == 1 { - g, err = glob.Compile(filter[0]) - } else { - g, err = glob.Compile("{" + strings.Join(filter, ",") + "}") - } - return g, err -} - func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool { if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) { return true diff --git a/plugins/inputs/varnish/varnish.go b/plugins/inputs/varnish/varnish.go index 0834a6e9d..1a3e4c558 100644 --- a/plugins/inputs/varnish/varnish.go +++ b/plugins/inputs/varnish/varnish.go @@ -10,33 +10,37 @@ import ( "os/exec" "strconv" "strings" + "time" + + "github.com/gobwas/glob" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" - "time" ) -const ( - kwAll = "all" -) +type runner func(cmdName string) (*bytes.Buffer, error) // Varnish is used to store configuration values type Varnish struct { Stats []string Binary string + + filter glob.Glob + run runner } var defaultStats = []string{"MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"} var defaultBinary = "/usr/bin/varnishstat" -var varnishSampleConfig = ` +var sampleConfig = ` ## The default location of the varnishstat binary can be overridden with: binary = "/usr/bin/varnishstat" ## By default, telegraf gather stats for 3 metric points. ## Setting stats will override the defaults shown below. - ## stats may also be set to ["all"], which will collect all stats + ## Glob matching can be used, ie, stats = ["MAIN.*"] + ## stats may also be set to ["*"], which will collect all stats stats = ["MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"] ` @@ -46,44 +50,11 @@ func (s *Varnish) Description() string { // SampleConfig displays configuration instructions func (s *Varnish) SampleConfig() string { - return fmt.Sprintf(varnishSampleConfig, strings.Join(defaultStats, "\",\"")) -} - -func (s *Varnish) setDefaults() { - if len(s.Stats) == 0 { - s.Stats = defaultStats - } - - if s.Binary == "" { - s.Binary = defaultBinary - } -} - -// Builds a filter function that will indicate whether a given stat should -// be reported -func (s *Varnish) statsFilter() func(string) bool { - s.setDefaults() - - // Build a set for constant-time lookup of whether stats should be included - filter := make(map[string]struct{}) - for _, s := range s.Stats { - filter[s] = struct{}{} - } - - // Create a function that respects the kwAll by always returning true - // if it is set - return func(stat string) bool { - if s.Stats[0] == kwAll { - return true - } - - _, found := filter[stat] - return found - } + return sampleConfig } // Shell out to varnish_stat and return the output -var varnishStat = func(cmdName string) (*bytes.Buffer, error) { +func varnishRunner(cmdName string) (*bytes.Buffer, error) { cmdArgs := []string{"-1"} cmd := exec.Command(cmdName, cmdArgs...) @@ -104,13 +75,27 @@ var varnishStat = func(cmdName string) (*bytes.Buffer, error) { // 'section' tag and all stats that share that prefix will be reported as fields // with that tag func (s *Varnish) Gather(acc telegraf.Accumulator) error { - s.setDefaults() - out, err := varnishStat(s.Binary) + if s.filter == nil { + var err error + if len(s.Stats) == 0 { + s.filter, err = internal.CompileFilter(defaultStats) + } else { + // legacy support, change "all" -> "*": + if s.Stats[0] == "all" { + s.Stats[0] = "*" + } + s.filter, err = internal.CompileFilter(s.Stats) + } + if err != nil { + return err + } + } + + out, err := s.run(s.Binary) if err != nil { return fmt.Errorf("error gathering metrics: %s", err) } - statsFilter := s.statsFilter() sectionMap := make(map[string]map[string]interface{}) scanner := bufio.NewScanner(out) for scanner.Scan() { @@ -125,7 +110,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error { stat := cols[0] value := cols[1] - if !statsFilter(stat) { + if s.filter != nil && !s.filter.Match(stat) { continue } @@ -138,7 +123,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error { sectionMap[section] = make(map[string]interface{}) } - sectionMap[section][field], err = strconv.Atoi(value) + sectionMap[section][field], err = strconv.ParseUint(value, 10, 64) if err != nil { fmt.Fprintf(os.Stderr, "Expected a numeric value for %s = %v\n", stat, value) @@ -160,5 +145,9 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error { } func init() { - inputs.Add("varnish", func() telegraf.Input { return &Varnish{} }) + inputs.Add("varnish", func() telegraf.Input { + return &Varnish{ + run: varnishRunner, + } + }) } diff --git a/plugins/inputs/varnish/varnish_test.go b/plugins/inputs/varnish/varnish_test.go index fd5d3d448..e3f848093 100644 --- a/plugins/inputs/varnish/varnish_test.go +++ b/plugins/inputs/varnish/varnish_test.go @@ -17,38 +17,12 @@ func fakeVarnishStat(output string) func(string) (*bytes.Buffer, error) { } } -func TestConfigsUsed(t *testing.T) { - saved := varnishStat - defer func() { - varnishStat = saved - }() - - expecations := map[string]string{ - "": defaultBinary, - "/foo/bar/baz": "/foo/bar/baz", - } - - for in, expected := range expecations { - varnishStat = func(actual string) (*bytes.Buffer, error) { - assert.Equal(t, expected, actual) - return &bytes.Buffer{}, nil - } - - acc := &testutil.Accumulator{} - v := &Varnish{Binary: in} - v.Gather(acc) - } -} - func TestGather(t *testing.T) { - saved := varnishStat - defer func() { - varnishStat = saved - }() - varnishStat = fakeVarnishStat(smOutput) - acc := &testutil.Accumulator{} - v := &Varnish{Stats: []string{"all"}} + v := &Varnish{ + run: fakeVarnishStat(smOutput), + Stats: []string{"*"}, + } v.Gather(acc) acc.HasMeasurement("varnish") @@ -60,14 +34,11 @@ func TestGather(t *testing.T) { } func TestParseFullOutput(t *testing.T) { - saved := varnishStat - defer func() { - varnishStat = saved - }() - varnishStat = fakeVarnishStat(fullOutput) - acc := &testutil.Accumulator{} - v := &Varnish{Stats: []string{"all"}} + v := &Varnish{ + run: fakeVarnishStat(fullOutput), + Stats: []string{"*"}, + } err := v.Gather(acc) assert.NoError(t, err) @@ -77,15 +48,24 @@ func TestParseFullOutput(t *testing.T) { assert.Equal(t, 293, len(flat)) } -func TestFieldConfig(t *testing.T) { - saved := varnishStat - defer func() { - varnishStat = saved - }() - varnishStat = fakeVarnishStat(fullOutput) +func TestFilterSomeStats(t *testing.T) { + acc := &testutil.Accumulator{} + v := &Varnish{ + run: fakeVarnishStat(fullOutput), + Stats: []string{"MGT.*", "VBE.*"}, + } + err := v.Gather(acc) + assert.NoError(t, err) + acc.HasMeasurement("varnish") + flat := flatten(acc.Metrics) + assert.Len(t, acc.Metrics, 2) + assert.Equal(t, 16, len(flat)) +} + +func TestFieldConfig(t *testing.T) { expect := map[string]int{ - "all": 293, + "*": 293, "": 0, // default "MAIN.uptime": 1, "MEMPOOL.req0.sz_needed,MAIN.fetch_bad": 2, @@ -93,7 +73,10 @@ func TestFieldConfig(t *testing.T) { for fieldCfg, expected := range expect { acc := &testutil.Accumulator{} - v := &Varnish{Stats: strings.Split(fieldCfg, ",")} + v := &Varnish{ + run: fakeVarnishStat(fullOutput), + Stats: strings.Split(fieldCfg, ","), + } err := v.Gather(acc) assert.NoError(t, err) @@ -130,18 +113,18 @@ MEMPOOL.vbc.sz_wanted 88 . Size requested var parsedSmOutput = map[string]map[string]interface{}{ "MAIN": map[string]interface{}{ - "uptime": 895, - "cache_hit": 95, - "cache_miss": 5, + "uptime": uint64(895), + "cache_hit": uint64(95), + "cache_miss": uint64(5), }, "MGT": map[string]interface{}{ - "uptime": 896, - "child_start": 1, + "uptime": uint64(896), + "child_start": uint64(1), }, "MEMPOOL": map[string]interface{}{ - "vbc.live": 0, - "vbc.pool": 10, - "vbc.sz_wanted": 88, + "vbc.live": uint64(0), + "vbc.pool": uint64(10), + "vbc.sz_wanted": uint64(88), }, }