From 499b5befd661b7f0ce005978ae77b3ac9034115e Mon Sep 17 00:00:00 2001 From: Roman Statsevich Date: Mon, 19 Oct 2015 20:38:16 +0300 Subject: [PATCH] add bcache plugin Closes #286 --- CHANGELOG.md | 1 + README.md | 1 + plugins/all/all.go | 1 + plugins/bcache/README.md | 89 ++++++++++++++++++++ plugins/bcache/bcache.go | 144 ++++++++++++++++++++++++++++++++ plugins/bcache/bcache_test.go | 153 ++++++++++++++++++++++++++++++++++ 6 files changed, 389 insertions(+) create mode 100644 plugins/bcache/README.md create mode 100644 plugins/bcache/bcache.go create mode 100644 plugins/bcache/bcache_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 983a46b92..1a6d7d190 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ of metrics collected and from how many plugins. - [#273](https://github.com/influxdb/telegraf/pull/273): puppet agent plugin, thats @jrxFive! - [#280](https://github.com/influxdb/telegraf/issues/280): Use InfluxDB client v2. - [#281](https://github.com/influxdb/telegraf/issues/281): Eliminate need to deep copy Batch Points. +- [#286](https://github.com/influxdb/telegraf/issues/286): bcache plugin, thanks @cornerot! ### Bugfixes - [#228](https://github.com/influxdb/telegraf/pull/228): New version of package will replace old one. Thanks @ekini! diff --git a/README.md b/README.md index 4fcaefc0e..71fca6c49 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ Below is how to configure `tagpass` and `tagdrop` parameters (added in 0.1.5) Telegraf currently has support for collecting metrics from * apache +* bcache * disque * elasticsearch * exec (generic JSON-emitting executable plugin) diff --git a/plugins/all/all.go b/plugins/all/all.go index 119c46678..3f4163d17 100644 --- a/plugins/all/all.go +++ b/plugins/all/all.go @@ -2,6 +2,7 @@ package all import ( _ "github.com/influxdb/telegraf/plugins/apache" + _ "github.com/influxdb/telegraf/plugins/bcache" _ "github.com/influxdb/telegraf/plugins/disque" _ "github.com/influxdb/telegraf/plugins/elasticsearch" _ "github.com/influxdb/telegraf/plugins/exec" diff --git a/plugins/bcache/README.md b/plugins/bcache/README.md new file mode 100644 index 000000000..27062b915 --- /dev/null +++ b/plugins/bcache/README.md @@ -0,0 +1,89 @@ +# Telegraf plugin: bcache + +Get bcache stat from stats_total directory and dirty_data file. + +# Measurements + +Meta: + +- tags: `backing_dev=dev bcache_dev=dev` + +Measurement names: + +- dirty_data +- bypassed +- cache_bypass_hits +- cache_bypass_misses +- cache_hit_ratio +- cache_hits +- cache_miss_collisions +- cache_misses +- cache_readaheads + +### Description + +``` +dirty_data + Amount of dirty data for this backing device in the cache. Continuously + updated unlike the cache set's version, but may be slightly off. + +bypassed + Amount of IO (both reads and writes) that has bypassed the cache + + +cache_bypass_hits +cache_bypass_misses + Hits and misses for IO that is intended to skip the cache are still counted, + but broken out here. + +cache_hits +cache_misses +cache_hit_ratio + Hits and misses are counted per individual IO as bcache sees them; a + partial hit is counted as a miss. + +cache_miss_collisions + Counts instances where data was going to be inserted into the cache from a + cache miss, but raced with a write and data was already present (usually 0 + since the synchronization for cache misses was rewritten) + +cache_readaheads + Count of times readahead occurred. +``` + +# Example output + +Using this configuration: + +``` +[bcache] + # Bcache sets path + # If not specified, then default is: + # bcachePath = "/sys/fs/bcache" + # + # By default, telegraf gather stats for all bcache devices + # Setting devices will restrict the stats to the specified + # bcache devices. + # bcacheDevs = ["bcache0", ...] +``` + +When run with: + +``` +./telegraf -config telegraf.conf -filter bcache -test +``` + +It produces: + +``` +* Plugin: bcache, Collection 1 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_dirty_data value=11639194 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_bypassed value=5167704440832 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_bypass_hits value=146270986 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_bypass_misses value=0 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_hit_ratio value=90 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_hits value=511941651 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_miss_collisions value=157678 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_misses value=50647396 +> [backing_dev="md10" bcache_dev="bcache0"] bcache_cache_readaheads value=0 +``` diff --git a/plugins/bcache/bcache.go b/plugins/bcache/bcache.go new file mode 100644 index 000000000..ee63f3c48 --- /dev/null +++ b/plugins/bcache/bcache.go @@ -0,0 +1,144 @@ +package bcache + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/influxdb/telegraf/plugins" +) + +type Bcache struct { + BcachePath string + BcacheDevs []string +} + +var sampleConfig = ` + # Bcache sets path + # If not specified, then default is: + # bcachePath = "/sys/fs/bcache" + # + # By default, telegraf gather stats for all bcache devices + # Setting devices will restrict the stats to the specified + # bcache devices. + # bcacheDevs = ["bcache0", ...] +` + +func (b *Bcache) SampleConfig() string { + return sampleConfig +} + +func (b *Bcache) Description() string { + return "Read metrics of bcache from stats_total and dirty_data" +} + +func getBackingDevs(bcachePath string) []string { + bdevs, err := filepath.Glob(bcachePath + "/*/bdev*") + if len(bdevs) < 1 { + panic("Can't found any bcache device") + } + if err != nil { + panic(err) + } + return bdevs +} + +func getTags(bdev string) map[string]string { + backingDevFile, _ := os.Readlink(bdev) + backingDevPath := strings.Split(backingDevFile, "/") + backingDev := backingDevPath[len(backingDevPath)-2] + + bcacheDevFile, _ := os.Readlink(bdev + "/dev") + bcacheDevPath := strings.Split(bcacheDevFile, "/") + bcacheDev := bcacheDevPath[len(bcacheDevPath)-1] + + return map[string]string{"backing_dev": backingDev, "bcache_dev": bcacheDev} +} + +func prettyToBytes(v string) uint64 { + var factors = map[string]uint64{ + "k": 1 << 10, + "M": 1 << 20, + "G": 1 << 30, + "T": 1 << 40, + "P": 1 << 50, + "E": 1 << 60, + } + var factor uint64 + factor = 1 + prefix := v[len(v)-1 : len(v)] + if factors[prefix] != 0 { + v = v[:len(v)-1] + factor = factors[prefix] + } + result, _ := strconv.ParseFloat(v, 32) + result = result * float64(factor) + + return uint64(result) +} + +func (b *Bcache) gatherBcache(bdev string, acc plugins.Accumulator) error { + tags := getTags(bdev) + metrics, err := filepath.Glob(bdev + "/stats_total/*") + if len(metrics) < 0 { + panic("Can't read any stats file") + } + file, err := ioutil.ReadFile(bdev + "/dirty_data") + if err != nil { + panic(err) + } + rawValue := strings.TrimSpace(string(file)) + value := prettyToBytes(rawValue) + acc.Add("dirty_data", value, tags) + + for _, path := range metrics { + key := filepath.Base(path) + file, err := ioutil.ReadFile(path) + rawValue := strings.TrimSpace(string(file)) + if err != nil { + panic(err) + } + if key == "bypassed" { + value := prettyToBytes(rawValue) + acc.Add(key, value, tags) + } else { + value, _ := strconv.ParseUint(rawValue, 10, 64) + acc.Add(key, value, tags) + } + } + return nil +} + +func (b *Bcache) Gather(acc plugins.Accumulator) error { + bcacheDevsChecked := make(map[string]bool) + var restrictDevs bool + if len(b.BcacheDevs) != 0 { + restrictDevs = true + for _, bcacheDev := range b.BcacheDevs { + bcacheDevsChecked[bcacheDev] = true + } + } + + bcachePath := b.BcachePath + if len(bcachePath) == 0 { + bcachePath = "/sys/fs/bcache" + } + for _, bdev := range getBackingDevs(bcachePath) { + if restrictDevs { + bcacheDev := getTags(bdev)["bcache_dev"] + if !bcacheDevsChecked[bcacheDev] { + continue + } + } + b.gatherBcache(bdev, acc) + } + return nil +} + +func init() { + plugins.Add("bcache", func() plugins.Plugin { + return &Bcache{} + }) +} diff --git a/plugins/bcache/bcache_test.go b/plugins/bcache/bcache_test.go new file mode 100644 index 000000000..b2b83bfec --- /dev/null +++ b/plugins/bcache/bcache_test.go @@ -0,0 +1,153 @@ +package bcache + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/influxdb/telegraf/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + dirty_data = "1.5G" + bypassed = "4.7T" + cache_bypass_hits = "146155333" + cache_bypass_misses = "0" + cache_hit_ratio = "90" + cache_hits = "511469583" + cache_miss_collisions = "157567" + cache_misses = "50616331" + cache_readaheads = "2" +) + +var ( + testBcachePath = os.TempDir() + "/telegraf/sys/fs/bcache" + testBcacheUuidPath = testBcachePath + "/663955a3-765a-4737-a9fd-8250a7a78411" + testBcacheDevPath = os.TempDir() + "/telegraf/sys/devices/virtual/block/bcache0" + testBcacheBackingDevPath = os.TempDir() + "/telegraf/sys/devices/virtual/block/md10" +) + +type metrics struct { + name string + value uint64 +} + +func TestBcacheGeneratesMetrics(t *testing.T) { + err := os.MkdirAll(testBcacheUuidPath, 0755) + require.NoError(t, err) + + err = os.MkdirAll(testBcacheDevPath, 0755) + require.NoError(t, err) + + err = os.MkdirAll(testBcacheBackingDevPath+"/bcache", 0755) + require.NoError(t, err) + + err = os.Symlink(testBcacheBackingDevPath+"/bcache", testBcacheUuidPath+"/bdev0") + require.NoError(t, err) + + err = os.Symlink(testBcacheDevPath, testBcacheUuidPath+"/bdev0/dev") + require.NoError(t, err) + + err = os.MkdirAll(testBcacheUuidPath+"/bdev0/stats_total", 0755) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/dirty_data", []byte(dirty_data), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/bypassed", []byte(bypassed), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_bypass_hits", []byte(cache_bypass_hits), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_bypass_misses", []byte(cache_bypass_misses), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_hit_ratio", []byte(cache_hit_ratio), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_hits", []byte(cache_hits), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_miss_collisions", []byte(cache_miss_collisions), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_misses", []byte(cache_misses), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testBcacheUuidPath+"/bdev0/stats_total/cache_readaheads", []byte(cache_readaheads), 0644) + require.NoError(t, err) + + intMetrics := []*metrics{ + { + name: "dirty_data", + value: 1610612736, + }, + { + name: "bypassed", + value: 5167704440832, + }, + { + name: "cache_bypass_hits", + value: 146155333, + }, + { + name: "cache_bypass_misses", + value: 0, + }, + { + name: "cache_hit_ratio", + value: 90, + }, + { + name: "cache_hits", + value: 511469583, + }, + { + name: "cache_miss_collisions", + value: 157567, + }, + { + name: "cache_misses", + value: 50616331, + }, + { + name: "cache_readaheads", + value: 2, + }, + } + + tags := map[string]string{ + "backing_dev": "md10", + "bcache_dev": "bcache0", + } + + var acc testutil.Accumulator + + //all devs + b := &Bcache{BcachePath: testBcachePath} + + err = b.Gather(&acc) + require.NoError(t, err) + + for _, metric := range intMetrics { + assert.True(t, acc.HasUIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + //one exist dev + b = &Bcache{BcachePath: testBcachePath, BcacheDevs: []string{"bcache0"}} + + err = b.Gather(&acc) + require.NoError(t, err) + + for _, metric := range intMetrics { + assert.True(t, acc.HasUIntValue(metric.name), metric.name) + assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) + } + + err = os.RemoveAll(os.TempDir() + "/telegraf") + require.NoError(t, err) +}