change jolokia input to use bulk requests (#2253)

This commit is contained in:
Patrick Hemmer 2017-04-18 16:00:41 -04:00 committed by Daniel Nelson
parent eb7ef5392e
commit 2542ef6d62
3 changed files with 215 additions and 139 deletions

View File

@ -72,6 +72,7 @@ be deprecated eventually.
- [#1820](https://github.com/influxdata/telegraf/issues/1820): easier plugin testing without outputs - [#1820](https://github.com/influxdata/telegraf/issues/1820): easier plugin testing without outputs
- [#2493](https://github.com/influxdata/telegraf/pull/2493): Check signature in the GitHub webhook plugin - [#2493](https://github.com/influxdata/telegraf/pull/2493): Check signature in the GitHub webhook plugin
- [#2038](https://github.com/influxdata/telegraf/issues/2038): Add papertrail support to webhooks - [#2038](https://github.com/influxdata/telegraf/issues/2038): Add papertrail support to webhooks
- [#2253](https://github.com/influxdata/telegraf/pull/2253): Change jolokia plugin to use bulk requests.
### Bugfixes ### Bugfixes

View File

@ -3,7 +3,6 @@ package jolokia
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -130,7 +129,7 @@ func (j *Jolokia) Description() string {
return "Read JMX metrics through Jolokia" return "Read JMX metrics through Jolokia"
} }
func (j *Jolokia) doRequest(req *http.Request) (map[string]interface{}, error) { func (j *Jolokia) doRequest(req *http.Request) ([]map[string]interface{}, error) {
resp, err := j.jClient.MakeRequest(req) resp, err := j.jClient.MakeRequest(req)
if err != nil { if err != nil {
return nil, err return nil, err
@ -155,85 +154,81 @@ func (j *Jolokia) doRequest(req *http.Request) (map[string]interface{}, error) {
} }
// Unmarshal json // Unmarshal json
var jsonOut map[string]interface{} var jsonOut []map[string]interface{}
if err = json.Unmarshal([]byte(body), &jsonOut); err != nil { if err = json.Unmarshal([]byte(body), &jsonOut); err != nil {
return nil, errors.New("Error decoding JSON response") return nil, fmt.Errorf("Error decoding JSON response: %s: %s", err, body)
}
if status, ok := jsonOut["status"]; ok {
if status != float64(200) {
return nil, fmt.Errorf("Not expected status value in response body: %3.f",
status)
}
} else {
return nil, fmt.Errorf("Missing status in response body")
} }
return jsonOut, nil return jsonOut, nil
} }
func (j *Jolokia) prepareRequest(server Server, metric Metric) (*http.Request, error) { func (j *Jolokia) prepareRequest(server Server, metrics []Metric) (*http.Request, error) {
var jolokiaUrl *url.URL var jolokiaUrl *url.URL
context := j.Context // Usually "/jolokia/" context := j.Context // Usually "/jolokia/"
// Create bodyContent var bulkBodyContent []map[string]interface{}
bodyContent := map[string]interface{}{ for _, metric := range metrics {
"type": "read", // Create bodyContent
"mbean": metric.Mbean, bodyContent := map[string]interface{}{
"type": "read",
"mbean": metric.Mbean,
}
if metric.Attribute != "" {
bodyContent["attribute"] = metric.Attribute
if metric.Path != "" {
bodyContent["path"] = metric.Path
}
}
// Add target, only in proxy mode
if j.Mode == "proxy" {
serviceUrl := fmt.Sprintf("service:jmx:rmi:///jndi/rmi://%s:%s/jmxrmi",
server.Host, server.Port)
target := map[string]string{
"url": serviceUrl,
}
if server.Username != "" {
target["user"] = server.Username
}
if server.Password != "" {
target["password"] = server.Password
}
bodyContent["target"] = target
proxy := j.Proxy
// Prepare ProxyURL
proxyUrl, err := url.Parse("http://" + proxy.Host + ":" + proxy.Port + context)
if err != nil {
return nil, err
}
if proxy.Username != "" || proxy.Password != "" {
proxyUrl.User = url.UserPassword(proxy.Username, proxy.Password)
}
jolokiaUrl = proxyUrl
} else {
serverUrl, err := url.Parse("http://" + server.Host + ":" + server.Port + context)
if err != nil {
return nil, err
}
if server.Username != "" || server.Password != "" {
serverUrl.User = url.UserPassword(server.Username, server.Password)
}
jolokiaUrl = serverUrl
}
bulkBodyContent = append(bulkBodyContent, bodyContent)
} }
if metric.Attribute != "" { requestBody, err := json.Marshal(bulkBodyContent)
bodyContent["attribute"] = metric.Attribute
if metric.Path != "" {
bodyContent["path"] = metric.Path
}
}
// Add target, only in proxy mode
if j.Mode == "proxy" {
serviceUrl := fmt.Sprintf("service:jmx:rmi:///jndi/rmi://%s:%s/jmxrmi",
server.Host, server.Port)
target := map[string]string{
"url": serviceUrl,
}
if server.Username != "" {
target["user"] = server.Username
}
if server.Password != "" {
target["password"] = server.Password
}
bodyContent["target"] = target
proxy := j.Proxy
// Prepare ProxyURL
proxyUrl, err := url.Parse("http://" + proxy.Host + ":" + proxy.Port + context)
if err != nil {
return nil, err
}
if proxy.Username != "" || proxy.Password != "" {
proxyUrl.User = url.UserPassword(proxy.Username, proxy.Password)
}
jolokiaUrl = proxyUrl
} else {
serverUrl, err := url.Parse("http://" + server.Host + ":" + server.Port + context)
if err != nil {
return nil, err
}
if server.Username != "" || server.Password != "" {
serverUrl.User = url.UserPassword(server.Username, server.Password)
}
jolokiaUrl = serverUrl
}
requestBody, err := json.Marshal(bodyContent)
req, err := http.NewRequest("POST", jolokiaUrl.String(), bytes.NewBuffer(requestBody)) req, err := http.NewRequest("POST", jolokiaUrl.String(), bytes.NewBuffer(requestBody))
@ -276,25 +271,35 @@ func (j *Jolokia) Gather(acc telegraf.Accumulator) error {
tags["jolokia_host"] = server.Host tags["jolokia_host"] = server.Host
fields := make(map[string]interface{}) fields := make(map[string]interface{})
for _, metric := range metrics { req, err := j.prepareRequest(server, metrics)
measurement := metric.Name if err != nil {
acc.AddError(fmt.Errorf("unable to create request: %s", err))
continue
}
out, err := j.doRequest(req)
if err != nil {
acc.AddError(fmt.Errorf("error performing request: %s", err))
continue
}
req, err := j.prepareRequest(server, metric) if len(out) != len(metrics) {
if err != nil { acc.AddError(fmt.Errorf("did not receive the correct number of metrics in response. expected %d, received %d", len(metrics), len(out)))
return err continue
}
for i, resp := range out {
if status, ok := resp["status"]; ok && status != float64(200) {
acc.AddError(fmt.Errorf("Not expected status value in response body (%s:%s mbean=\"%s\" attribute=\"%s\"): %3.f",
server.Host, server.Port, metrics[i].Mbean, metrics[i].Attribute, status))
continue
} else if !ok {
acc.AddError(fmt.Errorf("Missing status in response body"))
continue
} }
out, err := j.doRequest(req) if values, ok := resp["value"]; ok {
j.extractValues(metrics[i].Name, values, fields)
if err != nil {
fmt.Printf("Error handling response: %s\n", err)
} else { } else {
if values, ok := out["value"]; ok { acc.AddError(fmt.Errorf("Missing key 'value' in output response\n"))
j.extractValues(measurement, values, fields)
} else {
fmt.Printf("Missing key 'value' in output response\n")
}
} }
} }

View File

@ -13,65 +13,105 @@ import (
) )
const validThreeLevelMultiValueJSON = ` const validThreeLevelMultiValueJSON = `
{ [
"request":{ {
"mbean":"java.lang:type=*", "request":{
"type":"read" "mbean":"java.lang:type=*",
"type":"read"
},
"value":{
"java.lang:type=Memory":{
"ObjectPendingFinalizationCount":0,
"Verbose":false,
"HeapMemoryUsage":{
"init":134217728,
"committed":173015040,
"max":1908932608,
"used":16840016
},
"NonHeapMemoryUsage":{
"init":2555904,
"committed":51380224,
"max":-1,
"used":49944048
},
"ObjectName":{
"objectName":"java.lang:type=Memory"
}
}
},
"timestamp":1446129191,
"status":200
}
]`
const validBulkResponseJSON = `
[
{
"request":{
"mbean":"java.lang:type=Memory",
"attribute":"HeapMemoryUsage",
"type":"read"
},
"value":{
"init":67108864,
"committed":456130560,
"max":477626368,
"used":203288528
},
"timestamp":1446129191,
"status":200
}, },
"value":{ {
"java.lang:type=Memory":{ "request":{
"ObjectPendingFinalizationCount":0, "mbean":"java.lang:type=Memory",
"Verbose":false, "attribute":"NonHeapMemoryUsage",
"HeapMemoryUsage":{ "type":"read"
"init":134217728, },
"committed":173015040, "value":{
"max":1908932608, "init":2555904,
"used":16840016 "committed":51380224,
}, "max":-1,
"NonHeapMemoryUsage":{ "used":49944048
"init":2555904, },
"committed":51380224, "timestamp":1446129191,
"max":-1, "status":200
"used":49944048 }
}, ]`
"ObjectName":{
"objectName":"java.lang:type=Memory"
}
}
},
"timestamp":1446129191,
"status":200
}`
const validMultiValueJSON = ` const validMultiValueJSON = `
{ [
"request":{ {
"mbean":"java.lang:type=Memory", "request":{
"attribute":"HeapMemoryUsage", "mbean":"java.lang:type=Memory",
"type":"read" "attribute":"HeapMemoryUsage",
}, "type":"read"
"value":{ },
"init":67108864, "value":{
"committed":456130560, "init":67108864,
"max":477626368, "committed":456130560,
"used":203288528 "max":477626368,
}, "used":203288528
"timestamp":1446129191, },
"status":200 "timestamp":1446129191,
}` "status":200
}
]`
const validSingleValueJSON = ` const validSingleValueJSON = `
{ [
"request":{ {
"path":"used", "request":{
"mbean":"java.lang:type=Memory", "path":"used",
"attribute":"HeapMemoryUsage", "mbean":"java.lang:type=Memory",
"type":"read" "attribute":"HeapMemoryUsage",
}, "type":"read"
"value":209274376, },
"timestamp":1446129256, "value":209274376,
"status":200 "timestamp":1446129256,
}` "status":200
}
]`
const invalidJSON = "I don't think this is JSON" const invalidJSON = "I don't think this is JSON"
@ -82,6 +122,8 @@ var HeapMetric = Metric{Name: "heap_memory_usage",
Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"} Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"}
var UsedHeapMetric = Metric{Name: "heap_memory_usage", var UsedHeapMetric = Metric{Name: "heap_memory_usage",
Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"} Mbean: "java.lang:type=Memory", Attribute: "HeapMemoryUsage"}
var NonHeapMetric = Metric{Name: "non_heap_memory_usage",
Mbean: "java.lang:type=Memory", Attribute: "NonHeapMemoryUsage"}
type jolokiaClientStub struct { type jolokiaClientStub struct {
responseBody string responseBody string
@ -135,6 +177,34 @@ func TestHttpJsonMultiValue(t *testing.T) {
acc.AssertContainsTaggedFields(t, "jolokia", fields, tags) acc.AssertContainsTaggedFields(t, "jolokia", fields, tags)
} }
// Test that bulk responses are handled
func TestHttpJsonBulkResponse(t *testing.T) {
jolokia := genJolokiaClientStub(validBulkResponseJSON, 200, Servers, []Metric{HeapMetric, NonHeapMetric})
var acc testutil.Accumulator
err := jolokia.Gather(&acc)
assert.Nil(t, err)
assert.Equal(t, 1, len(acc.Metrics))
fields := map[string]interface{}{
"heap_memory_usage_init": 67108864.0,
"heap_memory_usage_committed": 456130560.0,
"heap_memory_usage_max": 477626368.0,
"heap_memory_usage_used": 203288528.0,
"non_heap_memory_usage_init": 2555904.0,
"non_heap_memory_usage_committed": 51380224.0,
"non_heap_memory_usage_max": -1.0,
"non_heap_memory_usage_used": 49944048.0,
}
tags := map[string]string{
"jolokia_host": "127.0.0.1",
"jolokia_port": "8080",
"jolokia_name": "as1",
}
acc.AssertContainsTaggedFields(t, "jolokia", fields, tags)
}
// Test that the proper values are ignored or collected // Test that the proper values are ignored or collected
func TestHttpJsonThreeLevelMultiValue(t *testing.T) { func TestHttpJsonThreeLevelMultiValue(t *testing.T) {
jolokia := genJolokiaClientStub(validThreeLevelMultiValueJSON, 200, Servers, []Metric{HeapMetric}) jolokia := genJolokiaClientStub(validThreeLevelMultiValueJSON, 200, Servers, []Metric{HeapMetric})