From bc3429ed48a59de5c6cb031603ca0b0c2f2fee70 Mon Sep 17 00:00:00 2001 From: vikkyomkar Date: Fri, 24 Jan 2020 01:38:16 +0530 Subject: [PATCH] Add compatibility for Kibana 6.4 and later (#6923) --- plugins/inputs/kibana/README.md | 11 +- plugins/inputs/kibana/kibana.go | 44 +++- plugins/inputs/kibana/kibana_test.go | 40 ++- .../{testdata_test.go => testdata_test6_3.go} | 5 +- plugins/inputs/kibana/testdata_test6_5.go | 227 ++++++++++++++++++ 5 files changed, 306 insertions(+), 21 deletions(-) rename plugins/inputs/kibana/{testdata_test.go => testdata_test6_3.go} (96%) create mode 100644 plugins/inputs/kibana/testdata_test6_5.go diff --git a/plugins/inputs/kibana/README.md b/plugins/inputs/kibana/README.md index 7d885aed1..f24e3d33a 100644 --- a/plugins/inputs/kibana/README.md +++ b/plugins/inputs/kibana/README.md @@ -42,6 +42,7 @@ with following rules: - kibana - status_code: integer (1, 2, 3, 0) + - heap_total_bytes: integer - heap_max_bytes: integer - heap_used_bytes: integer - uptime_ms: integer @@ -52,12 +53,12 @@ with following rules: ### Tags -- status (Kibana health: green, yellow, red) - name (Kibana reported name) -- uuid (Kibana reported UUID) -- version (Kibana version) - source (Kibana server hostname or IP) +- status (Kibana health: green, yellow, red) +- version (Kibana version) ### Example Output - -kibana,host=myhost,name=my-kibana,source=localhost:5601,version=6.3.2 concurrent_connections=0i,heap_max_bytes=136478720i,heap_used_bytes=119231088i,response_time_avg_ms=0i,response_time_max_ms=0i,status="green",status_code=1i,uptime_ms=2187428019i 1534864502000000000 +``` +kibana,host=myhost,name=my-kibana,source=localhost:5601,status=green,version=6.5.4 concurrent_connections=8i,heap_max_bytes=447778816i,heap_total_bytes=447778816i,heap_used_bytes=380603352i,requests_per_sec=1,response_time_avg_ms=57.6,response_time_max_ms=220i,status_code=1i,uptime_ms=6717489805i 1534864502000000000 +``` diff --git a/plugins/inputs/kibana/kibana.go b/plugins/inputs/kibana/kibana.go index 0e21ad800..64353013c 100644 --- a/plugins/inputs/kibana/kibana.go +++ b/plugins/inputs/kibana/kibana.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "strings" "sync" "time" @@ -54,7 +55,9 @@ type responseTimes struct { } type process struct { - Mem mem `json:"mem"` + Mem mem `json:"mem"` + Memory memory `json:"memory"` + UptimeInMillis int64 `json:"uptime_in_millis"` } type requests struct { @@ -66,6 +69,15 @@ type mem struct { HeapUsedInBytes int64 `json:"heap_used_in_bytes"` } +type memory struct { + Heap heap `json:"heap"` +} + +type heap struct { + TotalInBytes int64 `json:"total_in_bytes"` + UsedInBytes int64 `json:"used_in_bytes"` +} + const sampleConfig = ` ## specify a list of one or more Kibana servers servers = ["http://localhost:5601"] @@ -187,15 +199,37 @@ func (k *Kibana) gatherKibanaStatus(baseUrl string, acc telegraf.Accumulator) er tags["status"] = kibanaStatus.Status.Overall.State fields["status_code"] = mapHealthStatusToCode(kibanaStatus.Status.Overall.State) - - fields["uptime_ms"] = kibanaStatus.Metrics.UptimeInMillis fields["concurrent_connections"] = kibanaStatus.Metrics.ConcurrentConnections - fields["heap_max_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapMaxInBytes - fields["heap_used_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapUsedInBytes fields["response_time_avg_ms"] = kibanaStatus.Metrics.ResponseTimes.AvgInMillis fields["response_time_max_ms"] = kibanaStatus.Metrics.ResponseTimes.MaxInMillis fields["requests_per_sec"] = float64(kibanaStatus.Metrics.Requests.Total) / float64(kibanaStatus.Metrics.CollectionIntervalInMilles) * 1000 + versionArray := strings.Split(kibanaStatus.Version.Number, ".") + arrayElement := 1 + + if len(versionArray) > 1 { + arrayElement = 2 + } + versionNumber, err := strconv.ParseFloat(strings.Join(versionArray[:arrayElement], "."), 64) + if err != nil { + return err + } + + // Same value will be assigned to both the metrics [heap_max_bytes and heap_total_bytes ] + // Which keeps the code backward compatible + if versionNumber >= 6.4 { + fields["uptime_ms"] = kibanaStatus.Metrics.Process.UptimeInMillis + fields["heap_max_bytes"] = kibanaStatus.Metrics.Process.Memory.Heap.TotalInBytes + fields["heap_total_bytes"] = kibanaStatus.Metrics.Process.Memory.Heap.TotalInBytes + fields["heap_used_bytes"] = kibanaStatus.Metrics.Process.Memory.Heap.UsedInBytes + } else { + fields["uptime_ms"] = kibanaStatus.Metrics.UptimeInMillis + fields["heap_max_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapMaxInBytes + fields["heap_total_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapMaxInBytes + fields["heap_used_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapUsedInBytes + + } + acc.AddFields("kibana", fields, tags) return nil diff --git a/plugins/inputs/kibana/kibana_test.go b/plugins/inputs/kibana/kibana_test.go index ad5e32d29..537f6b560 100644 --- a/plugins/inputs/kibana/kibana_test.go +++ b/plugins/inputs/kibana/kibana_test.go @@ -9,7 +9,7 @@ import ( "github.com/influxdata/telegraf/testutil" ) -func defaultTags() map[string]string { +func defaultTags6_3() map[string]string { return map[string]string{ "name": "my-kibana", "source": "example.com:5601", @@ -18,6 +18,15 @@ func defaultTags() map[string]string { } } +func defaultTags6_5() map[string]string { + return map[string]string{ + "name": "my-kibana", + "source": "example.com:5601", + "version": "6.5.4", + "status": "green", + } +} + type transportMock struct { statusCode int body string @@ -41,22 +50,35 @@ func (t *transportMock) RoundTrip(r *http.Request) (*http.Response, error) { return res, nil } -func checkKibanaStatusResult(t *testing.T, acc *testutil.Accumulator) { - tags := defaultTags() - acc.AssertContainsTaggedFields(t, "kibana", kibanaStatusExpected, tags) +func checkKibanaStatusResult(version string, t *testing.T, acc *testutil.Accumulator) { + if version == "6.3.2" { + tags := defaultTags6_3() + acc.AssertContainsTaggedFields(t, "kibana", kibanaStatusExpected6_3, tags) + } else { + tags := defaultTags6_5() + acc.AssertContainsTaggedFields(t, "kibana", kibanaStatusExpected6_5, tags) + } } func TestGather(t *testing.T) { ks := newKibanahWithClient() ks.Servers = []string{"http://example.com:5601"} - ks.client.Transport = newTransportMock(http.StatusOK, kibanaStatusResponse) - - var acc testutil.Accumulator - if err := acc.GatherError(ks.Gather); err != nil { + // Unit test for Kibana version < 6.5 + ks.client.Transport = newTransportMock(http.StatusOK, kibanaStatusResponse6_3) + var acc1 testutil.Accumulator + if err := acc1.GatherError(ks.Gather); err != nil { t.Fatal(err) } + checkKibanaStatusResult(defaultTags6_3()["version"], t, &acc1) + + //Unit test for Kibana version >= 6.5 + ks.client.Transport = newTransportMock(http.StatusOK, kibanaStatusResponse6_5) + var acc2 testutil.Accumulator + if err := acc2.GatherError(ks.Gather); err != nil { + t.Fatal(err) + } + checkKibanaStatusResult(defaultTags6_5()["version"], t, &acc2) - checkKibanaStatusResult(t, &acc) } func newKibanahWithClient() *Kibana { diff --git a/plugins/inputs/kibana/testdata_test.go b/plugins/inputs/kibana/testdata_test6_3.go similarity index 96% rename from plugins/inputs/kibana/testdata_test.go rename to plugins/inputs/kibana/testdata_test6_3.go index ec393bb19..bda529273 100644 --- a/plugins/inputs/kibana/testdata_test.go +++ b/plugins/inputs/kibana/testdata_test6_3.go @@ -1,6 +1,6 @@ package kibana -const kibanaStatusResponse = ` +const kibanaStatusResponse6_3 = ` { "name": "my-kibana", "uuid": "00000000-0000-0000-0000-000000000000", @@ -187,8 +187,9 @@ const kibanaStatusResponse = ` } ` -var kibanaStatusExpected = map[string]interface{}{ +var kibanaStatusExpected6_3 = map[string]interface{}{ "status_code": 1, + "heap_total_bytes": int64(149954560), "heap_max_bytes": int64(149954560), "heap_used_bytes": int64(126274392), "uptime_ms": int64(2173595336), diff --git a/plugins/inputs/kibana/testdata_test6_5.go b/plugins/inputs/kibana/testdata_test6_5.go new file mode 100644 index 000000000..f47878b11 --- /dev/null +++ b/plugins/inputs/kibana/testdata_test6_5.go @@ -0,0 +1,227 @@ +package kibana + +const kibanaStatusResponse6_5 = ` +{ + "name": "my-kibana", + "uuid": "00000000-0000-0000-0000-000000000000", + "version": { + "number": "6.5.4", + "build_hash": "53d0c6758ac3fb38a3a1df198c1d4c87765e63f7", + "build_number": 17307, + "build_snapshot": false + }, + "status": { + "overall": { + "state": "green", + "title": "Green", + "nickname": "Looking good", + "icon": "success", + "since": "2018-07-27T07:37:42.567Z" + }, + "statuses": [{ + "id": "plugin:kibana@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.567Z" + }, + { + "id": "plugin:elasticsearch@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:04.920Z" + }, + { + "id": "plugin:xpack_main@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.393Z" + }, + { + "id": "plugin:searchprofiler@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.395Z" + }, + { + "id": "plugin:tilemap@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.396Z" + }, + { + "id": "plugin:watcher@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.397Z" + }, + { + "id": "plugin:license_management@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.668Z" + }, + { + "id": "plugin:index_management@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.399Z" + }, + { + "id": "plugin:timelion@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.912Z" + }, + { + "id": "plugin:logtrail@0.1.29", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.919Z" + }, + { + "id": "plugin:monitoring@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.922Z" + }, + { + "id": "plugin:grokdebugger@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.400Z" + }, + { + "id": "plugin:dashboard_mode@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.928Z" + }, + { + "id": "plugin:logstash@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.401Z" + }, + { + "id": "plugin:apm@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.950Z" + }, + { + "id": "plugin:console@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.958Z" + }, + { + "id": "plugin:console_extensions@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.961Z" + }, + { + "id": "plugin:metrics@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.965Z" + }, + { + "id": "plugin:reporting@6.5.4", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.402Z" + }] + }, + "metrics": { + "last_updated": "2020-01-15T09:40:17.733Z", + "collection_interval_in_millis": 5000, + "process": { + "memory": { + "heap": { + "total_in_bytes": 149954560, + "used_in_bytes": 126274392, + "size_limit": 1501560832 + }, + "resident_set_size_in_bytes": 286650368 + }, + "event_loop_delay": 0.5314235687255859, + "pid": 6, + "uptime_in_millis": 2173595336 + }, + "os": { + "load": { + "1m": 2.66015625, + "5m": 2.8173828125, + "15m": 2.51025390625 + }, + "memory": { + "total_in_bytes": 404355756032, + "free_in_bytes": 294494244864, + "used_in_bytes": 109861511168 + }, + "uptime_in_millis": 8220745000, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 1086527218898 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": -1, + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + } + } + } + }, + "response_times": { + "avg_in_millis": 12.5, + "max_in_millis": 123 + }, + "requests": { + "total": 2, + "disconnects": 0, + "status_codes": { + "200": 1, + "304": 1 + } + }, + "concurrent_connections": 10 + } +} +` + +var kibanaStatusExpected6_5 = map[string]interface{}{ + "status_code": 1, + "heap_total_bytes": int64(149954560), + "heap_max_bytes": int64(149954560), + "heap_used_bytes": int64(126274392), + "uptime_ms": int64(2173595336), + "response_time_avg_ms": float64(12.5), + "response_time_max_ms": int64(123), + "concurrent_connections": int64(10), + "requests_per_sec": float64(0.4), +}