dom
/
telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Allow wildcard filtering of varnish stats 

closes #1275
This commit is contained in:
Cameron Sparr 2016-05-26 10:40:03 +01:00
parent a8334c3261
commit 9bbdb2d562
7 changed files with 137 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -24,6 +24,7 @@ time before a new metric is included by the plugin.
- [#1264](https://github.com/influxdata/telegraf/pull/1264): Add SSL config options to http_response plugin.
- [#1272](https://github.com/influxdata/telegraf/pull/1272): graphite parser: add ability to specify multiple tag keys, for consistency with influxdb parser.
- [#1265](https://github.com/influxdata/telegraf/pull/1265): Make dns lookups for chrony configurable. Thanks @zbindenren!
- [#1275](https://github.com/influxdata/telegraf/pull/1275): Allow wildcard filtering of varnish stats.
### Bugfixes

2
Godeps
View File

@ -16,7 +16,7 @@ github.com/eapache/go-resiliency b86b1ec0dd4209a588dc1285cdd471e73525c0b3
github.com/eapache/queue ded5959c0d4e360646dc9e9908cff48666781367
github.com/eclipse/paho.mqtt.golang 0f7a459f04f13a41b7ed752d47944528d4bf9a86
github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
github.com/gobwas/glob d877f6352135181470c40c73ebb81aefa22115fa
github.com/gobwas/glob 49571a1557cd20e6a2410adc6421f85b66c730b5
github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
github.com/golang/snappy 427fb6fc07997f43afa32f35e850833760e489a7
github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2

23
internal/internal.go
View File

@ -16,6 +16,8 @@ import (
"strings"
"time"
"unicode"
"github.com/gobwas/glob"
)
const alphanum string = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@ -205,3 +207,24 @@ func WaitTimeout(c *exec.Cmd, timeout time.Duration) error {
return TimeoutErr
}
}
// CompileFilter takes a list of glob "filters", ie:
// ["MAIN.*", "CPU.*", "NET"]
// and compiles them into a glob object. This glob object can
// then be used to match keys to the filter.
func CompileFilter(filters []string) (glob.Glob, error) {
var out glob.Glob
// return if there is nothing to compile
if len(filters) == 0 {
return out, nil
}
var err error
if len(filters) == 1 {
out, err = glob.Compile(filters[0])
} else {
out, err = glob.Compile("{" + strings.Join(filters, ",") + "}")
}
return out, err
}

31
internal/internal_test.go
View File

@ -106,3 +106,34 @@ func TestRunError(t *testing.T) {
assert.Error(t, err)
}
func TestCompileFilter(t *testing.T) {
f, err := CompileFilter([]string{})
assert.NoError(t, err)
assert.Nil(t, f)
f, err = CompileFilter([]string{"cpu"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))
f, err = CompileFilter([]string{"cpu*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.True(t, f.Match("cpu0"))
assert.False(t, f.Match("mem"))
f, err = CompileFilter([]string{"cpu", "mem"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))
f, err = CompileFilter([]string{"cpu", "mem", "net*"})
assert.NoError(t, err)
assert.True(t, f.Match("cpu"))
assert.False(t, f.Match("cpu0"))
assert.True(t, f.Match("mem"))
assert.True(t, f.Match("network"))
}

32
internal/models/filter.go
View File

@ -2,11 +2,11 @@ package internal_models
import (
"fmt"
"strings"
"github.com/gobwas/glob"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
)
// TagFilter is the name of a tag, and the values on which to filter
@ -42,41 +42,41 @@ type Filter struct {
// Compile all Filter lists into glob.Glob objects.
func (f *Filter) CompileFilter() error {
var err error
f.nameDrop, err = compileFilter(f.NameDrop)
f.nameDrop, err = internal.CompileFilter(f.NameDrop)
if err != nil {
return fmt.Errorf("Error compiling 'namedrop', %s", err)
}
f.namePass, err = compileFilter(f.NamePass)
f.namePass, err = internal.CompileFilter(f.NamePass)
if err != nil {
return fmt.Errorf("Error compiling 'namepass', %s", err)
}
f.fieldDrop, err = compileFilter(f.FieldDrop)
f.fieldDrop, err = internal.CompileFilter(f.FieldDrop)
if err != nil {
return fmt.Errorf("Error compiling 'fielddrop', %s", err)
}
f.fieldPass, err = compileFilter(f.FieldPass)
f.fieldPass, err = internal.CompileFilter(f.FieldPass)
if err != nil {
return fmt.Errorf("Error compiling 'fieldpass', %s", err)
}
f.tagExclude, err = compileFilter(f.TagExclude)
f.tagExclude, err = internal.CompileFilter(f.TagExclude)
if err != nil {
return fmt.Errorf("Error compiling 'tagexclude', %s", err)
}
f.tagInclude, err = compileFilter(f.TagInclude)
f.tagInclude, err = internal.CompileFilter(f.TagInclude)
if err != nil {
return fmt.Errorf("Error compiling 'taginclude', %s", err)
}
for i, _ := range f.TagDrop {
f.TagDrop[i].filter, err = compileFilter(f.TagDrop[i].Filter)
f.TagDrop[i].filter, err = internal.CompileFilter(f.TagDrop[i].Filter)
if err != nil {
return fmt.Errorf("Error compiling 'tagdrop', %s", err)
}
}
for i, _ := range f.TagPass {
f.TagPass[i].filter, err = compileFilter(f.TagPass[i].Filter)
f.TagPass[i].filter, err = internal.CompileFilter(f.TagPass[i].Filter)
if err != nil {
return fmt.Errorf("Error compiling 'tagpass', %s", err)
}
@ -84,20 +84,6 @@ func (f *Filter) CompileFilter() error {
return nil
}
func compileFilter(filter []string) (glob.Glob, error) {
if len(filter) == 0 {
return nil, nil
}
var g glob.Glob
var err error
if len(filter) == 1 {
g, err = glob.Compile(filter[0])
} else {
g, err = glob.Compile("{" + strings.Join(filter, ",") + "}")
}
return g, err
}
func (f *Filter) ShouldMetricPass(metric telegraf.Metric) bool {
if f.ShouldNamePass(metric.Name()) && f.ShouldTagsPass(metric.Tags()) {
return true

83
plugins/inputs/varnish/varnish.go
View File

@ -10,33 +10,37 @@ import (
"os/exec"
"strconv"
"strings"
"time"
"github.com/gobwas/glob"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/inputs"
"time"
)
const (
kwAll = "all"
)
type runner func(cmdName string) (*bytes.Buffer, error)
// Varnish is used to store configuration values
type Varnish struct {
Stats []string
Binary string
filter glob.Glob
run runner
}
var defaultStats = []string{"MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"}
var defaultBinary = "/usr/bin/varnishstat"
var varnishSampleConfig = `
var sampleConfig = `
## The default location of the varnishstat binary can be overridden with:
binary = "/usr/bin/varnishstat"
## By default, telegraf gather stats for 3 metric points.
## Setting stats will override the defaults shown below.
## stats may also be set to ["all"], which will collect all stats
## Glob matching can be used, ie, stats = ["MAIN.*"]
## stats may also be set to ["*"], which will collect all stats
stats = ["MAIN.cache_hit", "MAIN.cache_miss", "MAIN.uptime"]
`
@ -46,44 +50,11 @@ func (s *Varnish) Description() string {
// SampleConfig displays configuration instructions
func (s *Varnish) SampleConfig() string {
return fmt.Sprintf(varnishSampleConfig, strings.Join(defaultStats, "\",\""))
}
func (s *Varnish) setDefaults() {
if len(s.Stats) == 0 {
s.Stats = defaultStats
}
if s.Binary == "" {
s.Binary = defaultBinary
}
}
// Builds a filter function that will indicate whether a given stat should
// be reported
func (s *Varnish) statsFilter() func(string) bool {
s.setDefaults()
// Build a set for constant-time lookup of whether stats should be included
filter := make(map[string]struct{})
for _, s := range s.Stats {
filter[s] = struct{}{}
}
// Create a function that respects the kwAll by always returning true
// if it is set
return func(stat string) bool {
if s.Stats[0] == kwAll {
return true
}
_, found := filter[stat]
return found
}
return sampleConfig
}
// Shell out to varnish_stat and return the output
var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
func varnishRunner(cmdName string) (*bytes.Buffer, error) {
cmdArgs := []string{"-1"}
cmd := exec.Command(cmdName, cmdArgs...)
@ -104,13 +75,27 @@ var varnishStat = func(cmdName string) (*bytes.Buffer, error) {
// 'section' tag and all stats that share that prefix will be reported as fields
// with that tag
func (s *Varnish) Gather(acc telegraf.Accumulator) error {
s.setDefaults()
out, err := varnishStat(s.Binary)
if s.filter == nil {
var err error
if len(s.Stats) == 0 {
s.filter, err = internal.CompileFilter(defaultStats)
} else {
// legacy support, change "all" -> "*":
if s.Stats[0] == "all" {
s.Stats[0] = "*"
}
s.filter, err = internal.CompileFilter(s.Stats)
}
if err != nil {
return err
}
}
out, err := s.run(s.Binary)
if err != nil {
return fmt.Errorf("error gathering metrics: %s", err)
}
statsFilter := s.statsFilter()
sectionMap := make(map[string]map[string]interface{})
scanner := bufio.NewScanner(out)
for scanner.Scan() {
@ -125,7 +110,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
stat := cols[0]
value := cols[1]
if !statsFilter(stat) {
if s.filter != nil && !s.filter.Match(stat) {
continue
}
@ -138,7 +123,7 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
sectionMap[section] = make(map[string]interface{})
}
sectionMap[section][field], err = strconv.Atoi(value)
sectionMap[section][field], err = strconv.ParseUint(value, 10, 64)
if err != nil {
fmt.Fprintf(os.Stderr, "Expected a numeric value for %s = %v\n",
stat, value)
@ -160,5 +145,9 @@ func (s *Varnish) Gather(acc telegraf.Accumulator) error {
}
func init() {
inputs.Add("varnish", func() telegraf.Input { return &Varnish{} })
inputs.Add("varnish", func() telegraf.Input {
return &Varnish{
run: varnishRunner,
}
})
}

89
plugins/inputs/varnish/varnish_test.go
View File

@ -17,38 +17,12 @@ func fakeVarnishStat(output string) func(string) (*bytes.Buffer, error) {
}
}
func TestConfigsUsed(t *testing.T) {
saved := varnishStat
defer func() {
varnishStat = saved
}()
expecations := map[string]string{
"": defaultBinary,
"/foo/bar/baz": "/foo/bar/baz",
}
for in, expected := range expecations {
varnishStat = func(actual string) (*bytes.Buffer, error) {
assert.Equal(t, expected, actual)
return &bytes.Buffer{}, nil
}
acc := &testutil.Accumulator{}
v := &Varnish{Binary: in}
v.Gather(acc)
}
}
func TestGather(t *testing.T) {
saved := varnishStat
defer func() {
varnishStat = saved
}()
varnishStat = fakeVarnishStat(smOutput)
acc := &testutil.Accumulator{}
v := &Varnish{Stats: []string{"all"}}
v := &Varnish{
run: fakeVarnishStat(smOutput),
Stats: []string{"*"},
}
v.Gather(acc)
acc.HasMeasurement("varnish")
@ -60,14 +34,11 @@ func TestGather(t *testing.T) {
}
func TestParseFullOutput(t *testing.T) {
saved := varnishStat
defer func() {
varnishStat = saved
}()
varnishStat = fakeVarnishStat(fullOutput)
acc := &testutil.Accumulator{}
v := &Varnish{Stats: []string{"all"}}
v := &Varnish{
run: fakeVarnishStat(fullOutput),
Stats: []string{"*"},
}
err := v.Gather(acc)
assert.NoError(t, err)
@ -77,15 +48,24 @@ func TestParseFullOutput(t *testing.T) {
assert.Equal(t, 293, len(flat))
}
func TestFieldConfig(t *testing.T) {
saved := varnishStat
defer func() {
varnishStat = saved
}()
varnishStat = fakeVarnishStat(fullOutput)
func TestFilterSomeStats(t *testing.T) {
acc := &testutil.Accumulator{}
v := &Varnish{
run: fakeVarnishStat(fullOutput),
Stats: []string{"MGT.*", "VBE.*"},
}
err := v.Gather(acc)
assert.NoError(t, err)
acc.HasMeasurement("varnish")
flat := flatten(acc.Metrics)
assert.Len(t, acc.Metrics, 2)
assert.Equal(t, 16, len(flat))
}
func TestFieldConfig(t *testing.T) {
expect := map[string]int{
"all": 293,
"*": 293,
"": 0, // default
"MAIN.uptime": 1,
"MEMPOOL.req0.sz_needed,MAIN.fetch_bad": 2,
@ -93,7 +73,10 @@ func TestFieldConfig(t *testing.T) {
for fieldCfg, expected := range expect {
acc := &testutil.Accumulator{}
v := &Varnish{Stats: strings.Split(fieldCfg, ",")}
v := &Varnish{
run: fakeVarnishStat(fullOutput),
Stats: strings.Split(fieldCfg, ","),
}
err := v.Gather(acc)
assert.NoError(t, err)
@ -130,18 +113,18 @@ MEMPOOL.vbc.sz_wanted 88 . Size requested
var parsedSmOutput = map[string]map[string]interface{}{
"MAIN": map[string]interface{}{
"uptime": 895,
"cache_hit": 95,
"cache_miss": 5,
"uptime": uint64(895),
"cache_hit": uint64(95),
"cache_miss": uint64(5),
},
"MGT": map[string]interface{}{
"uptime": 896,
"child_start": 1,
"uptime": uint64(896),
"child_start": uint64(1),
},
"MEMPOOL": map[string]interface{}{
"vbc.live": 0,
"vbc.pool": 10,
"vbc.sz_wanted": 88,
"vbc.live": uint64(0),
"vbc.pool": uint64(10),
"vbc.sz_wanted": uint64(88),
},
}