From 9ada89d51a5353c850c7a665ed1dee1a0fbebcce Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Wed, 6 Jan 2016 16:55:28 -0700 Subject: [PATCH] 0.3.0 unit tests: jolokia, kafka_consumer, leofs, lustre2 --- accumulator.go | 4 ++ plugins/jolokia/jolokia.go | 1 - plugins/jolokia/jolokia_test.go | 63 +++++------------- plugins/kafka_consumer/kafka_consumer_test.go | 3 +- plugins/leofs/leofs_test.go | 4 +- plugins/lustre2/lustre2.go | 29 +++++++-- plugins/lustre2/lustre2_test.go | 64 ++++++++----------- testutil/accumulator.go | 4 ++ 8 files changed, 75 insertions(+), 97 deletions(-) diff --git a/accumulator.go b/accumulator.go index 2defc8c7b..f14df63f7 100644 --- a/accumulator.go +++ b/accumulator.go @@ -69,6 +69,10 @@ func (ac *accumulator) AddFields( tags map[string]string, t ...time.Time, ) { + if len(fields) == 0 || len(measurement) == 0 { + return + } + if !ac.pluginConfig.Filter.ShouldTagsPass(tags) { return } diff --git a/plugins/jolokia/jolokia.go b/plugins/jolokia/jolokia.go index 8bebbc3c5..610f08cd5 100644 --- a/plugins/jolokia/jolokia.go +++ b/plugins/jolokia/jolokia.go @@ -75,7 +75,6 @@ func (j *Jolokia) getAttr(requestUrl *url.URL) (map[string]interface{}, error) { if err != nil { return nil, err } - defer req.Body.Close() resp, err := j.jClient.MakeRequest(req) if err != nil { diff --git a/plugins/jolokia/jolokia_test.go b/plugins/jolokia/jolokia_test.go index 95df76e7b..d29b8a810 100644 --- a/plugins/jolokia/jolokia_test.go +++ b/plugins/jolokia/jolokia_test.go @@ -48,7 +48,7 @@ const empty = "" var Servers = []Server{Server{Name: "as1", Host: "127.0.0.1", Port: "8080"}} var HeapMetric = Metric{Name: "heap_memory_usage", Jmx: "/java.lang:type=Memory/HeapMemoryUsage"} -var UsedHeapMetric = Metric{Name: "heap_memory_usage", Jmx: "/java.lang:type=Memory/HeapMemoryUsage", Pass: []string{"used"}} +var UsedHeapMetric = Metric{Name: "heap_memory_usage", Jmx: "/java.lang:type=Memory/HeapMemoryUsage"} type jolokiaClientStub struct { responseBody string @@ -79,7 +79,6 @@ func genJolokiaClientStub(response string, statusCode int, servers []Server, met // Test that the proper values are ignored or collected func TestHttpJsonMultiValue(t *testing.T) { - jolokia := genJolokiaClientStub(validMultiValueJSON, 200, Servers, []Metric{HeapMetric}) var acc testutil.Accumulator @@ -88,58 +87,28 @@ func TestHttpJsonMultiValue(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 1, len(acc.Points)) - assert.True(t, acc.CheckFieldsValue("heap_memory_usage", map[string]interface{}{"init": 67108864.0, - "committed": 456130560.0, - "max": 477626368.0, - "used": 203288528.0})) -} - -// Test that the proper values are ignored or collected -func TestHttpJsonMultiValueWithPass(t *testing.T) { - - jolokia := genJolokiaClientStub(validMultiValueJSON, 200, Servers, []Metric{UsedHeapMetric}) - - var acc testutil.Accumulator - err := jolokia.Gather(&acc) - - assert.Nil(t, err) - assert.Equal(t, 1, len(acc.Points)) - - assert.True(t, acc.CheckFieldsValue("heap_memory_usage", map[string]interface{}{"used": 203288528.0})) -} - -// Test that the proper values are ignored or collected -func TestHttpJsonMultiValueTags(t *testing.T) { - - jolokia := genJolokiaClientStub(validMultiValueJSON, 200, Servers, []Metric{UsedHeapMetric}) - - var acc testutil.Accumulator - err := jolokia.Gather(&acc) - - assert.Nil(t, err) - assert.Equal(t, 1, len(acc.Points)) - assert.NoError(t, acc.ValidateTaggedFieldsValue("heap_memory_usage", map[string]interface{}{"used": 203288528.0}, map[string]string{"host": "127.0.0.1", "port": "8080", "server": "as1"})) -} - -// Test that the proper values are ignored or collected -func TestHttpJsonSingleValueTags(t *testing.T) { - - jolokia := genJolokiaClientStub(validSingleValueJSON, 200, Servers, []Metric{UsedHeapMetric}) - - var acc testutil.Accumulator - err := jolokia.Gather(&acc) - - assert.Nil(t, err) - assert.Equal(t, 1, len(acc.Points)) - assert.NoError(t, acc.ValidateTaggedFieldsValue("heap_memory_usage", map[string]interface{}{"value": 209274376.0}, map[string]string{"host": "127.0.0.1", "port": "8080", "server": "as1"})) + fields := map[string]interface{}{ + "heap_memory_usage_init": 67108864.0, + "heap_memory_usage_committed": 456130560.0, + "heap_memory_usage_max": 477626368.0, + "heap_memory_usage_used": 203288528.0, + } + tags := map[string]string{ + "host": "127.0.0.1", + "port": "8080", + "server": "as1", + } + acc.AssertContainsTaggedFields(t, "jolokia", fields, tags) } // Test that the proper values are ignored or collected func TestHttpJsonOn404(t *testing.T) { - jolokia := genJolokiaClientStub(validMultiValueJSON, 404, Servers, []Metric{UsedHeapMetric}) + jolokia := genJolokiaClientStub(validMultiValueJSON, 404, Servers, + []Metric{UsedHeapMetric}) var acc testutil.Accumulator + acc.SetDebug(true) err := jolokia.Gather(&acc) assert.Nil(t, err) diff --git a/plugins/kafka_consumer/kafka_consumer_test.go b/plugins/kafka_consumer/kafka_consumer_test.go index eb0473361..dcd38f6c4 100644 --- a/plugins/kafka_consumer/kafka_consumer_test.go +++ b/plugins/kafka_consumer/kafka_consumer_test.go @@ -85,7 +85,8 @@ func TestRunParserAndGather(t *testing.T) { k.Gather(&acc) assert.Equal(t, len(acc.Points), 1) - assert.True(t, acc.CheckValue("cpu_load_short", 23422.0)) + acc.AssertContainsFields(t, "cpu_load_short", + map[string]interface{}{"value": float64(23422)}) } func saramaMsg(val string) *sarama.ConsumerMessage { diff --git a/plugins/leofs/leofs_test.go b/plugins/leofs/leofs_test.go index 62a9f3fa3..48a82a18a 100644 --- a/plugins/leofs/leofs_test.go +++ b/plugins/leofs/leofs_test.go @@ -129,7 +129,6 @@ func buildFakeSNMPCmd(src string) { } func testMain(t *testing.T, code string, endpoint string, serverType ServerType) { - // Build the fake snmpwalk for test src := makeFakeSNMPSrc(code) defer os.Remove(src) @@ -145,6 +144,7 @@ func testMain(t *testing.T, code string, endpoint string, serverType ServerType) } var acc testutil.Accumulator + acc.SetDebug(true) err := l.Gather(&acc) require.NoError(t, err) @@ -152,7 +152,7 @@ func testMain(t *testing.T, code string, endpoint string, serverType ServerType) floatMetrics := KeyMapping[serverType] for _, metric := range floatMetrics { - assert.True(t, acc.HasFloatValue(metric), metric) + assert.True(t, acc.HasFloatField("leofs", metric), metric) } } diff --git a/plugins/lustre2/lustre2.go b/plugins/lustre2/lustre2.go index 29cd06acf..65f936966 100644 --- a/plugins/lustre2/lustre2.go +++ b/plugins/lustre2/lustre2.go @@ -22,6 +22,9 @@ import ( type Lustre2 struct { Ost_procfiles []string Mds_procfiles []string + + // allFields maps and OST name to the metric fields associated with that OST + allFields map[string]map[string]interface{} } var sampleConfig = ` @@ -140,8 +143,11 @@ func (l *Lustre2) GetLustreProcStats(fileglob string, wanted_fields []*mapping, */ path := strings.Split(file, "/") name := path[len(path)-2] - tags := map[string]string{ - "name": name, + var fields map[string]interface{} + fields, ok := l.allFields[name] + if !ok { + fields = make(map[string]interface{}) + l.allFields[name] = fields } lines, err := internal.ReadLines(file) @@ -149,7 +155,6 @@ func (l *Lustre2) GetLustreProcStats(fileglob string, wanted_fields []*mapping, return err } - fields := make(map[string]interface{}) for _, line := range lines { parts := strings.Fields(line) for _, wanted := range wanted_fields { @@ -173,7 +178,6 @@ func (l *Lustre2) GetLustreProcStats(fileglob string, wanted_fields []*mapping, } } } - acc.AddFields("lustre2", fields, tags) } return nil } @@ -190,15 +194,18 @@ func (l *Lustre2) Description() string { // Gather reads stats from all lustre targets func (l *Lustre2) Gather(acc plugins.Accumulator) error { + l.allFields = make(map[string]map[string]interface{}) if len(l.Ost_procfiles) == 0 { // read/write bytes are in obdfilter//stats - err := l.GetLustreProcStats("/proc/fs/lustre/obdfilter/*/stats", wanted_ost_fields, acc) + err := l.GetLustreProcStats("/proc/fs/lustre/obdfilter/*/stats", + wanted_ost_fields, acc) if err != nil { return err } // cache counters are in osd-ldiskfs//stats - err = l.GetLustreProcStats("/proc/fs/lustre/osd-ldiskfs/*/stats", wanted_ost_fields, acc) + err = l.GetLustreProcStats("/proc/fs/lustre/osd-ldiskfs/*/stats", + wanted_ost_fields, acc) if err != nil { return err } @@ -206,7 +213,8 @@ func (l *Lustre2) Gather(acc plugins.Accumulator) error { if len(l.Mds_procfiles) == 0 { // Metadata server stats - err := l.GetLustreProcStats("/proc/fs/lustre/mdt/*/md_stats", wanted_mds_fields, acc) + err := l.GetLustreProcStats("/proc/fs/lustre/mdt/*/md_stats", + wanted_mds_fields, acc) if err != nil { return err } @@ -225,6 +233,13 @@ func (l *Lustre2) Gather(acc plugins.Accumulator) error { } } + for name, fields := range l.allFields { + tags := map[string]string{ + "name": name, + } + acc.AddFields("lustre2", fields, tags) + } + return nil } diff --git a/plugins/lustre2/lustre2_test.go b/plugins/lustre2/lustre2_test.go index 850a4ff32..cea98fa1e 100644 --- a/plugins/lustre2/lustre2_test.go +++ b/plugins/lustre2/lustre2_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/influxdb/telegraf/testutil" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -58,11 +57,6 @@ samedir_rename 259625 samples [reqs] crossdir_rename 369571 samples [reqs] ` -type metrics struct { - name string - value uint64 -} - func TestLustre2GeneratesMetrics(t *testing.T) { tempdir := os.TempDir() + "/telegraf/proc/fs/lustre/" @@ -103,41 +97,33 @@ func TestLustre2GeneratesMetrics(t *testing.T) { "name": ost_name, } - intMetrics := []*metrics{ - { - name: "write_bytes", - value: 15201500833981, - }, - { - name: "read_bytes", - value: 78026117632000, - }, - { - name: "write_calls", - value: 71893382, - }, - { - name: "read_calls", - value: 203238095, - }, - { - name: "cache_hit", - value: 7393729777, - }, - { - name: "cache_access", - value: 19047063027, - }, - { - name: "cache_miss", - value: 11653333250, - }, + fields := map[string]interface{}{ + "cache_access": uint64(19047063027), + "cache_hit": uint64(7393729777), + "cache_miss": uint64(11653333250), + "close": uint64(873243496), + "crossdir_rename": uint64(369571), + "getattr": uint64(1503663097), + "getxattr": uint64(6145349681), + "link": uint64(445), + "mkdir": uint64(705499), + "mknod": uint64(349042), + "open": uint64(1024577037), + "read_bytes": uint64(78026117632000), + "read_calls": uint64(203238095), + "rename": uint64(629196), + "rmdir": uint64(227434), + "samedir_rename": uint64(259625), + "setattr": uint64(1898364), + "setxattr": uint64(83969), + "statfs": uint64(2916320), + "sync": uint64(434081), + "unlink": uint64(3549417), + "write_bytes": uint64(15201500833981), + "write_calls": uint64(71893382), } - for _, metric := range intMetrics { - assert.True(t, acc.HasUIntValue(metric.name), metric.name) - assert.True(t, acc.CheckTaggedValue(metric.name, metric.value, tags)) - } + acc.AssertContainsTaggedFields(t, "lustre2", fields, tags) err = os.RemoveAll(os.TempDir() + "/telegraf") require.NoError(t, err) diff --git a/testutil/accumulator.go b/testutil/accumulator.go index 3800fef4a..7cdfb4155 100644 --- a/testutil/accumulator.go +++ b/testutil/accumulator.go @@ -55,6 +55,10 @@ func (a *Accumulator) AddFields( tags = map[string]string{} } + if len(fields) == 0 { + return + } + var t time.Time if len(timestamp) > 0 { t = timestamp[0]