Allow wildcard filtering of varnish stats

closes #1275

CHANGELOG.md

@@ -24,6 +24,7 @@ time before a new metric is included by the plugin.
 - [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin.
 - [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser.
 - [#1265](https://github.com/influxdata/telegraf/pull/1265): Make dns lookups for chrony configurable. Thanks @zbindenren!
+- [#1275](https://github.com/influxdata/telegraf/pull/1275): Allow wildcard filtering of varnish stats.
 
 ### Bugfixes
 

Godeps

@@ -16,7 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
 github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
 github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
 github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
-github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa
+github.com/gobwas/glob 49571a1557cd20e6a2410adc6421f85b66c730b5
 github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
 github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
 github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2

internal/internal.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"
 	"unicode"
+
+	"github.com/gobwas/glob"
 )
 
 const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@@ -205,3 +207,24 @@ func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
 		return TimeoutErr
 	}
 }
+
+// CompileFilter takes a list of glob "filters", ie:
+//   ["MAIN.*", "CPU.*", "NET"]
+// and compiles them into a glob object. This glob object can
+// then be used to match keys to the filter.
+func CompileFilter(filters []string) (glob.Glob, error) {
+	var out glob.Glob
+
+	// return if there is nothing to compile
+	if len(filters) == 0 {
+		return out, nil
+	}
+
+	var err error
+	if len(filters) == 1 {
+		out, err = glob.Compile(filters[0])
+	} else {
+		out, err = glob.Compile("{" + strings.Join(filters, ",") + "}")
+	}
+	return out, err
+}

internal/internal_test.go

@@ -106,3 +106,34 @@ func TestRunError(t *testing.T) {
 
 	assert.Error(t, err)
 }
+
+func TestCompileFilter(t *testing.T) {
+	f, err := CompileFilter([]string{})
+	assert.NoError(t, err)
+	assert.Nil(t, f)
+
+	f, err = CompileFilter([]string{"cpu"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.False(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu*"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.True(t, f.Match("cpu0"))
+	assert.False(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu", "mem"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.True(t, f.Match("mem"))
+
+	f, err = CompileFilter([]string{"cpu", "mem", "net*"})
+	assert.NoError(t, err)
+	assert.True(t, f.Match("cpu"))
+	assert.False(t, f.Match("cpu0"))
+	assert.True(t, f.Match("mem"))
+	assert.True(t, f.Match("network"))
+}

internal/models/filter.go

@@ -2,11 +2,11 @@ package internal_models
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/gobwas/glob"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 )
 
 // TagFilter is the name of a tag, and the values on which to filter
@@ -42,41 +42,41 @@ type Filter struct {
 // Compile all Filter lists into glob.Glob objects.
 func (f *Filter) CompileFilter() error {
 	var err error
-	f.nameDrop, err = compileFilter(f.NameDrop)
+	f.nameDrop, err = internal.CompileFilter(f.NameDrop)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'namedrop', %s", err)
 	}
-	f.namePass, err = compileFilter(f.NamePass)
+	f.namePass, err = internal.CompileFilter(f.NamePass)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'namepass', %s", err)
 	}
 
-	f.fieldDrop, err = compileFilter(f.FieldDrop)
+	f.fieldDrop, err = internal.CompileFilter(f.FieldDrop)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'fielddrop', %s", err)
 	}
-	f.fieldPass, err = compileFilter(f.FieldPass)
+	f.fieldPass, err = internal.CompileFilter(f.FieldPass)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'fieldpass', %s", err)
 	}
 
-	f.tagExclude, err = compileFilter(f.TagExclude)
+	f.tagExclude, err = internal.CompileFilter(f.TagExclude)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'tagexclude', %s", err)
 	}
-	f.tagInclude, err = compileFilter(f.TagInclude)
+	f.tagInclude, err = internal.CompileFilter(f.TagInclude)
 	if err != nil {
 		return fmt.Errorf("Error compiling 'taginclude', %s", err)
 	}
 
 	for i, _ := range f.TagDrop {
-		f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter)
+		f.TagDrop[i].filter, err = internal.CompileFilter(f.TagDrop[i].Filter)
 		if err != nil {
 			return fmt.Errorf("Error compiling 'tagdrop', %s", err)
 		}
 	}
 	for i, _ := range f.TagPass {
-		f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter)
+		f.TagPass[i].filter, err = internal.CompileFilter(f.TagPass[i].Filter)
 		if err != nil {
 			return fmt.Errorf("Error compiling 'tagpass', %s", err)
 		}
@@ -84,20 +84,6 @@ func (f *Filter) CompileFilter() error {
 	return nil
 }
 
-func compileFilter(filter []string) (glob.Glob, error) {
-	if len(filter) == 0 {
-		return nil, nil
-	}
-	var g glob.Glob
-	var err error
-	if len(filter) == 1 {
-		g, err = glob.Compile(filter[0])
-	} else {
-		g, err = glob.Compile("{" + strings.Join(filter, ",") + "}")
-	}
-	return g, err
-}
-
 func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool {
 	if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
 		return true

plugins/inputs/varnish/varnish.go

@@ -10,33 +10,37 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/gobwas/glob"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
-	"time"
 )
 
-const (
+type runner func(cmdName string) (*bytes.Buffer, error)
-	kwAll = "all"
-)
 
 // Varnish is used to store configuration values
 type Varnish struct {
 	Stats  []string
 	Binary string
+
+	filter glob.Glob
+	run    runner
 }
 
 var defaultStats = []string{"MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"}
 var defaultBinary = "/usr/bin/varnishstat"
 
-var varnishSampleConfig = `
+var sampleConfig = `
   ## The default location of the varnishstat binary can be overridden with:
   binary = "/usr/bin/varnishstat"
 
   ## By default, telegraf gather stats for 3 metric points.
   ## Setting stats will override the defaults shown below.
-  ## stats may also be set to ["all"], which will collect all stats
+  ## Glob matching can be used, ie, stats = ["MAIN.*"]
+  ## stats may also be set to ["*"], which will collect all stats
   stats = ["MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"]
 `
 
@@ -46,44 +50,11 @@ func (s *Varnish) Description() string {
 
 // SampleConfig displays configuration instructions
 func (s *Varnish) SampleConfig() string {
-	return fmt.Sprintf(varnishSampleConfig, strings.Join(defaultStats, "\",\""))
+	return sampleConfig
-}
-
-func (s *Varnish) setDefaults() {
-	if len(s.Stats) == 0 {
-		s.Stats = defaultStats
-	}
-
-	if s.Binary == "" {
-		s.Binary = defaultBinary
-	}
-}
-
-// Builds a filter function that will indicate whether a given stat should
-// be reported
-func (s *Varnish) statsFilter() func(string) bool {
-	s.setDefaults()
-
-	// Build a set for constant-time lookup of whether stats should be included
-	filter := make(map[string]struct{})
-	for _, s := range s.Stats {
-		filter[s] = struct{}{}
-	}
-
-	// Create a function that respects the kwAll by always returning true
-	// if it is set
-	return func(stat string) bool {
-		if s.Stats[0] == kwAll {
-			return true
-		}
-
-		_, found := filter[stat]
-		return found
-	}
 }
 
 // Shell out to varnish_stat and return the output
-var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
+func varnishRunner(cmdName string) (*bytes.Buffer, error) {
 	cmdArgs := []string{"-1"}
 
 	cmd := exec.Command(cmdName, cmdArgs...)
@@ -104,13 +75,27 @@ var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
 // 'section' tag and all stats that share that prefix will be reported as fields
 // with that tag
 func (s *Varnish) Gather(acc telegraf.Accumulator) error {
-	s.setDefaults()
+	if s.filter == nil {
-	out, err := varnishStat(s.Binary)
+		var err error
+		if len(s.Stats) == 0 {
+			s.filter, err = internal.CompileFilter(defaultStats)
+		} else {
+			// legacy support, change "all" -> "*":
+			if s.Stats[0] == "all" {
+				s.Stats[0] = "*"
+			}
+			s.filter, err = internal.CompileFilter(s.Stats)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	out, err := s.run(s.Binary)
 	if err != nil {
 		return fmt.Errorf("error gathering metrics: %s", err)
 	}
 
-	statsFilter := s.statsFilter()
 	sectionMap := make(map[string]map[string]interface{})
 	scanner := bufio.NewScanner(out)
 	for scanner.Scan() {
@@ -125,7 +110,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 		stat := cols[0]
 		value := cols[1]
 
-		if !statsFilter(stat) {
+		if s.filter != nil && !s.filter.Match(stat) {
 			continue
 		}
 
@@ -138,7 +123,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 			sectionMap[section] = make(map[string]interface{})
 		}
 
-		sectionMap[section][field], err = strconv.Atoi(value)
+		sectionMap[section][field], err = strconv.ParseUint(value, 10, 64)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Expected a numeric value for %s = %v\n",
 				stat, value)
@@ -160,5 +145,9 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
 }
 
 func init() {
-	inputs.Add("varnish", func() telegraf.Input { return &Varnish{} })
+	inputs.Add("varnish", func() telegraf.Input {
+		return &Varnish{
+			run: varnishRunner,
+		}
+	})
 }

plugins/inputs/varnish/varnish_test.go

@@ -17,38 +17,12 @@ func fakeVarnishStat(output string) func(string) (*bytes.Buffer, error) {
 	}
 }
 
-func TestConfigsUsed(t *testing.T) {
-	saved := varnishStat
-	defer func() {
-		varnishStat = saved
-	}()
-
-	expecations := map[string]string{
-		"":             defaultBinary,
-		"/foo/bar/baz": "/foo/bar/baz",
-	}
-
-	for in, expected := range expecations {
-		varnishStat = func(actual string) (*bytes.Buffer, error) {
-			assert.Equal(t, expected, actual)
-			return &bytes.Buffer{}, nil
-		}
-
-		acc := &testutil.Accumulator{}
-		v := &Varnish{Binary: in}
-		v.Gather(acc)
-	}
-}
-
 func TestGather(t *testing.T) {
-	saved := varnishStat
-	defer func() {
-		varnishStat = saved
-	}()
-	varnishStat = fakeVarnishStat(smOutput)
-
 	acc := &testutil.Accumulator{}
-	v := &Varnish{Stats: []string{"all"}}
+	v := &Varnish{
+		run:   fakeVarnishStat(smOutput),
+		Stats: []string{"*"},
+	}
 	v.Gather(acc)
 
 	acc.HasMeasurement("varnish")
@@ -60,14 +34,11 @@ func TestGather(t *testing.T) {
 }
 
 func TestParseFullOutput(t *testing.T) {
-	saved := varnishStat
-	defer func() {
-		varnishStat = saved
-	}()
-	varnishStat = fakeVarnishStat(fullOutput)
-
 	acc := &testutil.Accumulator{}
-	v := &Varnish{Stats: []string{"all"}}
+	v := &Varnish{
+		run:   fakeVarnishStat(fullOutput),
+		Stats: []string{"*"},
+	}
 	err := v.Gather(acc)
 
 	assert.NoError(t, err)
@@ -77,15 +48,24 @@ func TestParseFullOutput(t *testing.T) {
 	assert.Equal(t, 293, len(flat))
 }
 
-func TestFieldConfig(t *testing.T) {
+func TestFilterSomeStats(t *testing.T) {
-	saved := varnishStat
+	acc := &testutil.Accumulator{}
-	defer func() {
+	v := &Varnish{
-		varnishStat = saved
+		run:   fakeVarnishStat(fullOutput),
-	}()
+		Stats: []string{"MGT.*", "VBE.*"},
-	varnishStat = fakeVarnishStat(fullOutput)
+	}
+	err := v.Gather(acc)
 
+	assert.NoError(t, err)
+	acc.HasMeasurement("varnish")
+	flat := flatten(acc.Metrics)
+	assert.Len(t, acc.Metrics, 2)
+	assert.Equal(t, 16, len(flat))
+}
+
+func TestFieldConfig(t *testing.T) {
 	expect := map[string]int{
-		"all":                                   293,
+		"*":                                     293,
 		"":                                      0, // default
 		"MAIN.uptime":                           1,
 		"MEMPOOL.req0.sz_needed,MAIN.fetch_bad": 2,
@@ -93,7 +73,10 @@ func TestFieldConfig(t *testing.T) {
 
 	for fieldCfg, expected := range expect {
 		acc := &testutil.Accumulator{}
-		v := &Varnish{Stats: strings.Split(fieldCfg, ",")}
+		v := &Varnish{
+			run:   fakeVarnishStat(fullOutput),
+			Stats: strings.Split(fieldCfg, ","),
+		}
 		err := v.Gather(acc)
 
 		assert.NoError(t, err)
@@ -130,18 +113,18 @@ MEMPOOL.vbc.sz_wanted              88          .   Size requested
 
 var parsedSmOutput = map[string]map[string]interface{}{
 	"MAIN": map[string]interface{}{
-		"uptime":     895,
+		"uptime":     uint64(895),
-		"cache_hit":  95,
+		"cache_hit":  uint64(95),
-		"cache_miss": 5,
+		"cache_miss": uint64(5),
 	},
 	"MGT": map[string]interface{}{
-		"uptime":      896,
+		"uptime":      uint64(896),
-		"child_start": 1,
+		"child_start": uint64(1),
 	},
 	"MEMPOOL": map[string]interface{}{
-		"vbc.live":      0,
+		"vbc.live":      uint64(0),
-		"vbc.pool":      10,
+		"vbc.pool":      uint64(10),
-		"vbc.sz_wanted": 88,
+		"vbc.sz_wanted": uint64(88),
 	},
 }