renaming plugins -> inputs
This commit is contained in:
Cameron Sparr
148
plugins/inputs/httpjson/README.md
Normal file
148
plugins/inputs/httpjson/README.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# HTTP JSON Plugin
|
||||
|
||||
The httpjson plugin can collect data from remote URLs which respond with JSON. Then it flattens JSON and finds all numeric values, treating them as floats.
|
||||
|
||||
For example, if you have a service called _mycollector_, which has HTTP endpoint for gathering stats at http://my.service.com/_stats, you would configure the HTTP JSON
|
||||
plugin like this:
|
||||
|
||||
```
|
||||
[[httpjson.services]]
|
||||
name = "mycollector"
|
||||
|
||||
servers = [
|
||||
"http://my.service.com/_stats"
|
||||
]
|
||||
|
||||
# HTTP method to use (case-sensitive)
|
||||
method = "GET"
|
||||
```
|
||||
|
||||
`name` is used as a prefix for the measurements.
|
||||
|
||||
`method` specifies HTTP method to use for requests.
|
||||
|
||||
You can also specify which keys from server response should be considered tags:
|
||||
|
||||
```
|
||||
[[httpjson.services]]
|
||||
...
|
||||
|
||||
tag_keys = [
|
||||
"role",
|
||||
"version"
|
||||
]
|
||||
```
|
||||
|
||||
You can also specify additional request parameters for the service:
|
||||
|
||||
```
|
||||
[[httpjson.services]]
|
||||
...
|
||||
|
||||
[httpjson.services.parameters]
|
||||
event_type = "cpu_spike"
|
||||
threshold = "0.75"
|
||||
|
||||
```
|
||||
|
||||
|
||||
# Example:
|
||||
|
||||
Let's say that we have a service named "mycollector" configured like this:
|
||||
|
||||
```
|
||||
[httpjson]
|
||||
[[httpjson.services]]
|
||||
name = "mycollector"
|
||||
|
||||
servers = [
|
||||
"http://my.service.com/_stats"
|
||||
]
|
||||
|
||||
# HTTP method to use (case-sensitive)
|
||||
method = "GET"
|
||||
|
||||
tag_keys = ["service"]
|
||||
```
|
||||
|
||||
which responds with the following JSON:
|
||||
|
||||
```json
|
||||
{
|
||||
"service": "service01",
|
||||
"a": 0.5,
|
||||
"b": {
|
||||
"c": "some text",
|
||||
"d": 0.1,
|
||||
"e": 5
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The collected metrics will be:
|
||||
```
|
||||
httpjson_mycollector_a,service='service01',server='http://my.service.com/_stats' value=0.5
|
||||
httpjson_mycollector_b_d,service='service01',server='http://my.service.com/_stats' value=0.1
|
||||
httpjson_mycollector_b_e,service='service01',server='http://my.service.com/_stats' value=5
|
||||
```
|
||||
|
||||
# Example 2, Multiple Services:
|
||||
|
||||
There is also the option to collect JSON from multiple services, here is an
|
||||
example doing that.
|
||||
|
||||
```
|
||||
[httpjson]
|
||||
[[httpjson.services]]
|
||||
name = "mycollector1"
|
||||
|
||||
servers = [
|
||||
"http://my.service1.com/_stats"
|
||||
]
|
||||
|
||||
# HTTP method to use (case-sensitive)
|
||||
method = "GET"
|
||||
|
||||
[[httpjson.services]]
|
||||
name = "mycollector2"
|
||||
|
||||
servers = [
|
||||
"http://service.net/json/stats"
|
||||
]
|
||||
|
||||
# HTTP method to use (case-sensitive)
|
||||
method = "POST"
|
||||
```
|
||||
|
||||
The services respond with the following JSON:
|
||||
|
||||
mycollector1:
|
||||
```json
|
||||
{
|
||||
"a": 0.5,
|
||||
"b": {
|
||||
"c": "some text",
|
||||
"d": 0.1,
|
||||
"e": 5
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
mycollector2:
|
||||
```json
|
||||
{
|
||||
"load": 100,
|
||||
"users": 1335
|
||||
}
|
||||
```
|
||||
|
||||
The collected metrics will be:
|
||||
|
||||
```
|
||||
httpjson_mycollector1_a,server='http://my.service.com/_stats' value=0.5
|
||||
httpjson_mycollector1_b_d,server='http://my.service.com/_stats' value=0.1
|
||||
httpjson_mycollector1_b_e,server='http://my.service.com/_stats' value=5
|
||||
|
||||
httpjson_mycollector2_load,server='http://service.net/json/stats' value=100
|
||||
httpjson_mycollector2_users,server='http://service.net/json/stats' value=1335
|
||||
```
|
||||
216
plugins/inputs/httpjson/httpjson.go
Normal file
216
plugins/inputs/httpjson/httpjson.go
Normal file
@@ -0,0 +1,216 @@
|
||||
package httpjson
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/influxdb/telegraf/internal"
|
||||
"github.com/influxdb/telegraf/plugins/inputs"
|
||||
)
|
||||
|
||||
type HttpJson struct {
|
||||
Name string
|
||||
Servers []string
|
||||
Method string
|
||||
TagKeys []string
|
||||
Parameters map[string]string
|
||||
client HTTPClient
|
||||
}
|
||||
|
||||
type HTTPClient interface {
|
||||
// Returns the result of an http request
|
||||
//
|
||||
// Parameters:
|
||||
// req: HTTP request object
|
||||
//
|
||||
// Returns:
|
||||
// http.Response: HTTP respons object
|
||||
// error : Any error that may have occurred
|
||||
MakeRequest(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
type RealHTTPClient struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
|
||||
return c.client.Do(req)
|
||||
}
|
||||
|
||||
var sampleConfig = `
|
||||
# a name for the service being polled
|
||||
name = "webserver_stats"
|
||||
|
||||
# URL of each server in the service's cluster
|
||||
servers = [
|
||||
"http://localhost:9999/stats/",
|
||||
"http://localhost:9998/stats/",
|
||||
]
|
||||
|
||||
# HTTP method to use (case-sensitive)
|
||||
method = "GET"
|
||||
|
||||
# List of tag names to extract from top-level of JSON server response
|
||||
# tag_keys = [
|
||||
# "my_tag_1",
|
||||
# "my_tag_2"
|
||||
# ]
|
||||
|
||||
# HTTP parameters (all values must be strings)
|
||||
[inputs.httpjson.parameters]
|
||||
event_type = "cpu_spike"
|
||||
threshold = "0.75"
|
||||
`
|
||||
|
||||
func (h *HttpJson) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
|
||||
func (h *HttpJson) Description() string {
|
||||
return "Read flattened metrics from one or more JSON HTTP endpoints"
|
||||
}
|
||||
|
||||
// Gathers data for all servers.
|
||||
func (h *HttpJson) Gather(acc inputs.Accumulator) error {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
errorChannel := make(chan error, len(h.Servers))
|
||||
|
||||
for _, server := range h.Servers {
|
||||
wg.Add(1)
|
||||
go func(server string) {
|
||||
defer wg.Done()
|
||||
if err := h.gatherServer(acc, server); err != nil {
|
||||
errorChannel <- err
|
||||
}
|
||||
}(server)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errorChannel)
|
||||
|
||||
// Get all errors and return them as one giant error
|
||||
errorStrings := []string{}
|
||||
for err := range errorChannel {
|
||||
errorStrings = append(errorStrings, err.Error())
|
||||
}
|
||||
|
||||
if len(errorStrings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return errors.New(strings.Join(errorStrings, "\n"))
|
||||
}
|
||||
|
||||
// Gathers data from a particular server
|
||||
// Parameters:
|
||||
// acc : The telegraf Accumulator to use
|
||||
// serverURL: endpoint to send request to
|
||||
// service : the service being queried
|
||||
//
|
||||
// Returns:
|
||||
// error: Any error that may have occurred
|
||||
func (h *HttpJson) gatherServer(
|
||||
acc inputs.Accumulator,
|
||||
serverURL string,
|
||||
) error {
|
||||
resp, err := h.sendRequest(serverURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var jsonOut map[string]interface{}
|
||||
if err = json.Unmarshal([]byte(resp), &jsonOut); err != nil {
|
||||
return errors.New("Error decoding JSON response")
|
||||
}
|
||||
|
||||
tags := map[string]string{
|
||||
"server": serverURL,
|
||||
}
|
||||
|
||||
for _, tag := range h.TagKeys {
|
||||
switch v := jsonOut[tag].(type) {
|
||||
case string:
|
||||
tags[tag] = v
|
||||
}
|
||||
delete(jsonOut, tag)
|
||||
}
|
||||
|
||||
f := internal.JSONFlattener{}
|
||||
err = f.FlattenJSON("", jsonOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var msrmnt_name string
|
||||
if h.Name == "" {
|
||||
msrmnt_name = "httpjson"
|
||||
} else {
|
||||
msrmnt_name = "httpjson_" + h.Name
|
||||
}
|
||||
acc.AddFields(msrmnt_name, f.Fields, tags)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sends an HTTP request to the server using the HttpJson object's HTTPClient
|
||||
// Parameters:
|
||||
// serverURL: endpoint to send request to
|
||||
//
|
||||
// Returns:
|
||||
// string: body of the response
|
||||
// error : Any error that may have occurred
|
||||
func (h *HttpJson) sendRequest(serverURL string) (string, error) {
|
||||
// Prepare URL
|
||||
requestURL, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Invalid server URL \"%s\"", serverURL)
|
||||
}
|
||||
|
||||
params := url.Values{}
|
||||
for k, v := range h.Parameters {
|
||||
params.Add(k, v)
|
||||
}
|
||||
requestURL.RawQuery = params.Encode()
|
||||
|
||||
// Create + send request
|
||||
req, err := http.NewRequest(h.Method, requestURL.String(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := h.client.MakeRequest(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return string(body), err
|
||||
}
|
||||
|
||||
// Process response
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
|
||||
requestURL.String(),
|
||||
resp.StatusCode,
|
||||
http.StatusText(resp.StatusCode),
|
||||
http.StatusOK,
|
||||
http.StatusText(http.StatusOK))
|
||||
return string(body), err
|
||||
}
|
||||
|
||||
return string(body), err
|
||||
}
|
||||
|
||||
func init() {
|
||||
inputs.Add("httpjson", func() inputs.Input {
|
||||
return &HttpJson{client: RealHTTPClient{client: &http.Client{}}}
|
||||
})
|
||||
}
|
||||
198
plugins/inputs/httpjson/httpjson_test.go
Normal file
198
plugins/inputs/httpjson/httpjson_test.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package httpjson
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/influxdb/telegraf/testutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const validJSON = `
|
||||
{
|
||||
"parent": {
|
||||
"child": 3,
|
||||
"ignored_child": "hi"
|
||||
},
|
||||
"ignored_null": null,
|
||||
"integer": 4,
|
||||
"ignored_list": [3, 4],
|
||||
"ignored_parent": {
|
||||
"another_ignored_list": [4],
|
||||
"another_ignored_null": null,
|
||||
"ignored_string": "hello, world!"
|
||||
}
|
||||
}`
|
||||
|
||||
const validJSONTags = `
|
||||
{
|
||||
"value": 15,
|
||||
"role": "master",
|
||||
"build": "123"
|
||||
}`
|
||||
|
||||
var expectedFields = map[string]interface{}{
|
||||
"parent_child": float64(3),
|
||||
"integer": float64(4),
|
||||
}
|
||||
|
||||
const invalidJSON = "I don't think this is JSON"
|
||||
|
||||
const empty = ""
|
||||
|
||||
type mockHTTPClient struct {
|
||||
responseBody string
|
||||
statusCode int
|
||||
}
|
||||
|
||||
// Mock implementation of MakeRequest. Usually returns an http.Response with
|
||||
// hard-coded responseBody and statusCode. However, if the request uses a
|
||||
// nonstandard method, it uses status code 405 (method not allowed)
|
||||
func (c mockHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
|
||||
resp := http.Response{}
|
||||
resp.StatusCode = c.statusCode
|
||||
|
||||
// basic error checking on request method
|
||||
allowedMethods := []string{"GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "CONNECT"}
|
||||
methodValid := false
|
||||
for _, method := range allowedMethods {
|
||||
if req.Method == method {
|
||||
methodValid = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !methodValid {
|
||||
resp.StatusCode = 405 // Method not allowed
|
||||
}
|
||||
|
||||
resp.Body = ioutil.NopCloser(strings.NewReader(c.responseBody))
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
// Generates a pointer to an HttpJson object that uses a mock HTTP client.
|
||||
// Parameters:
|
||||
// response : Body of the response that the mock HTTP client should return
|
||||
// statusCode: HTTP status code the mock HTTP client should return
|
||||
//
|
||||
// Returns:
|
||||
// *HttpJson: Pointer to an HttpJson object that uses the generated mock HTTP client
|
||||
func genMockHttpJson(response string, statusCode int) []*HttpJson {
|
||||
return []*HttpJson{
|
||||
&HttpJson{
|
||||
client: mockHTTPClient{responseBody: response, statusCode: statusCode},
|
||||
Servers: []string{
|
||||
"http://server1.example.com/metrics/",
|
||||
"http://server2.example.com/metrics/",
|
||||
},
|
||||
Name: "my_webapp",
|
||||
Method: "GET",
|
||||
Parameters: map[string]string{
|
||||
"httpParam1": "12",
|
||||
"httpParam2": "the second parameter",
|
||||
},
|
||||
},
|
||||
&HttpJson{
|
||||
client: mockHTTPClient{responseBody: response, statusCode: statusCode},
|
||||
Servers: []string{
|
||||
"http://server3.example.com/metrics/",
|
||||
"http://server4.example.com/metrics/",
|
||||
},
|
||||
Name: "other_webapp",
|
||||
Method: "POST",
|
||||
Parameters: map[string]string{
|
||||
"httpParam1": "12",
|
||||
"httpParam2": "the second parameter",
|
||||
},
|
||||
TagKeys: []string{
|
||||
"role",
|
||||
"build",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Test that the proper values are ignored or collected
|
||||
func TestHttpJson200(t *testing.T) {
|
||||
httpjson := genMockHttpJson(validJSON, 200)
|
||||
|
||||
for _, service := range httpjson {
|
||||
var acc testutil.Accumulator
|
||||
err := service.Gather(&acc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 4, acc.NFields())
|
||||
for _, srv := range service.Servers {
|
||||
tags := map[string]string{"server": srv}
|
||||
mname := "httpjson_" + service.Name
|
||||
acc.AssertContainsTaggedFields(t, mname, expectedFields, tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test response to HTTP 500
|
||||
func TestHttpJson500(t *testing.T) {
|
||||
httpjson := genMockHttpJson(validJSON, 500)
|
||||
|
||||
var acc testutil.Accumulator
|
||||
err := httpjson[0].Gather(&acc)
|
||||
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, 0, acc.NFields())
|
||||
}
|
||||
|
||||
// Test response to HTTP 405
|
||||
func TestHttpJsonBadMethod(t *testing.T) {
|
||||
httpjson := genMockHttpJson(validJSON, 200)
|
||||
httpjson[0].Method = "NOT_A_REAL_METHOD"
|
||||
|
||||
var acc testutil.Accumulator
|
||||
err := httpjson[0].Gather(&acc)
|
||||
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, 0, acc.NFields())
|
||||
}
|
||||
|
||||
// Test response to malformed JSON
|
||||
func TestHttpJsonBadJson(t *testing.T) {
|
||||
httpjson := genMockHttpJson(invalidJSON, 200)
|
||||
|
||||
var acc testutil.Accumulator
|
||||
err := httpjson[0].Gather(&acc)
|
||||
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, 0, acc.NFields())
|
||||
}
|
||||
|
||||
// Test response to empty string as response objectgT
|
||||
func TestHttpJsonEmptyResponse(t *testing.T) {
|
||||
httpjson := genMockHttpJson(empty, 200)
|
||||
|
||||
var acc testutil.Accumulator
|
||||
err := httpjson[0].Gather(&acc)
|
||||
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, 0, acc.NFields())
|
||||
}
|
||||
|
||||
// Test that the proper values are ignored or collected
|
||||
func TestHttpJson200Tags(t *testing.T) {
|
||||
httpjson := genMockHttpJson(validJSONTags, 200)
|
||||
|
||||
for _, service := range httpjson {
|
||||
if service.Name == "other_webapp" {
|
||||
var acc testutil.Accumulator
|
||||
err := service.Gather(&acc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, acc.NFields())
|
||||
for _, srv := range service.Servers {
|
||||
tags := map[string]string{"server": srv, "role": "master", "build": "123"}
|
||||
fields := map[string]interface{}{"value": float64(15)}
|
||||
mname := "httpjson_" + service.Name
|
||||
acc.AssertContainsTaggedFields(t, mname, fields, tags)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user