From 763ff0be81c32acef7cc95b808bda0d85c7d6835 Mon Sep 17 00:00:00 2001 From: palkan Date: Fri, 16 Oct 2015 17:10:08 +0300 Subject: [PATCH 1/3] [Fix #190] Add httpjson tags support --- plugins/httpjson/httpjson.go | 17 ++++++++++++- plugins/httpjson/httpjson_test.go | 41 +++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/plugins/httpjson/httpjson.go b/plugins/httpjson/httpjson.go index c9717d558..23eda0d5f 100644 --- a/plugins/httpjson/httpjson.go +++ b/plugins/httpjson/httpjson.go @@ -22,6 +22,7 @@ type Service struct { Name string Servers []string Method string + Tags []string Parameters map[string]string } @@ -61,6 +62,12 @@ var sampleConfig = ` # HTTP method to use (case-sensitive) method = "GET" + # List of tag names to extract from server response + tags = [ + "my_tag_1", + "my_tag_2" + ] + # HTTP parameters (all values must be strings) [httpjson.services.parameters] event_type = "cpu_spike" @@ -126,7 +133,7 @@ func (h *HttpJson) gatherServer(acc plugins.Accumulator, service Service, server return err } - var jsonOut interface{} + var jsonOut map[string]interface{} if err = json.Unmarshal([]byte(resp), &jsonOut); err != nil { return errors.New("Error decoding JSON response") } @@ -135,6 +142,14 @@ func (h *HttpJson) gatherServer(acc plugins.Accumulator, service Service, server "server": serverURL, } + for _, tag := range service.Tags { + switch v := jsonOut[tag].(type) { + case string: + tags[tag] = v + } + delete(jsonOut, tag) + } + processResponse(acc, service.Name, tags, jsonOut) return nil } diff --git a/plugins/httpjson/httpjson_test.go b/plugins/httpjson/httpjson_test.go index 9e9b7d2b7..c0f62613e 100644 --- a/plugins/httpjson/httpjson_test.go +++ b/plugins/httpjson/httpjson_test.go @@ -28,6 +28,14 @@ const validJSON = ` } }` +const validJSONTags = ` + { + "value": 15, + "role": "master", + "build": "123" + }` + + const invalidJSON = "I don't think this is JSON" const empty = "" @@ -87,8 +95,8 @@ func genMockHttpJson(response string, statusCode int) *HttpJson { }, Service{ Servers: []string{ - "http://server1.example.com/metrics/", - "http://server2.example.com/metrics/", + "http://server3.example.com/metrics/", + "http://server4.example.com/metrics/", }, Name: "other_webapp", Method: "POST", @@ -96,6 +104,10 @@ func genMockHttpJson(response string, statusCode int) *HttpJson { "httpParam1": "12", "httpParam2": "the second parameter", }, + Tags: []string{ + "role", + "build", + }, }, }, } @@ -185,3 +197,28 @@ func TestHttpJsonEmptyResponse(t *testing.T) { assert.Equal(t, len(strings.Split(err.Error(), "\n")), 4) assert.Equal(t, 0, len(acc.Points)) } + +// Test that the proper values are ignored or collected +func TestHttpJson200Tags(t *testing.T) { + httpjson := genMockHttpJson(validJSONTags, 200) + + var acc testutil.Accumulator + err := httpjson.Gather(&acc) + require.NoError(t, err) + + assert.Equal(t, 4, len(acc.Points)) + + for _, service := range httpjson.Services { + if service.Name == "other_webapp" { + for _, srv := range service.Servers { + require.NoError(t, + acc.ValidateTaggedValue( + fmt.Sprintf("%s_value", service.Name), + 15.0, + map[string]string{"server": srv, "role": "master", "build": "123"}, + ), + ) + } + } + } +} From 1e822952b86ce7b45acc57c1e03fd9ddf45d47bb Mon Sep 17 00:00:00 2001 From: palkan Date: Sat, 17 Oct 2015 13:59:48 +0300 Subject: [PATCH 2/3] Rename Tags to TagKeys --- plugins/httpjson/httpjson.go | 12 ++++++------ plugins/httpjson/httpjson_test.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/httpjson/httpjson.go b/plugins/httpjson/httpjson.go index 23eda0d5f..5c9bee02a 100644 --- a/plugins/httpjson/httpjson.go +++ b/plugins/httpjson/httpjson.go @@ -22,7 +22,7 @@ type Service struct { Name string Servers []string Method string - Tags []string + TagKeys []string Parameters map[string]string } @@ -63,10 +63,10 @@ var sampleConfig = ` method = "GET" # List of tag names to extract from server response - tags = [ - "my_tag_1", - "my_tag_2" - ] + # tagKeys = [ + # "my_tag_1", + # "my_tag_2" + # ] # HTTP parameters (all values must be strings) [httpjson.services.parameters] @@ -142,7 +142,7 @@ func (h *HttpJson) gatherServer(acc plugins.Accumulator, service Service, server "server": serverURL, } - for _, tag := range service.Tags { + for _, tag := range service.TagKeys { switch v := jsonOut[tag].(type) { case string: tags[tag] = v diff --git a/plugins/httpjson/httpjson_test.go b/plugins/httpjson/httpjson_test.go index c0f62613e..335568724 100644 --- a/plugins/httpjson/httpjson_test.go +++ b/plugins/httpjson/httpjson_test.go @@ -104,7 +104,7 @@ func genMockHttpJson(response string, statusCode int) *HttpJson { "httpParam1": "12", "httpParam2": "the second parameter", }, - Tags: []string{ + TagKeys: []string{ "role", "build", }, From 529a894c55b8ec4ccb324043109ed8ec9039ae81 Mon Sep 17 00:00:00 2001 From: palkan Date: Sat, 17 Oct 2015 14:38:50 +0300 Subject: [PATCH 3/3] Add httpjson readme --- plugins/httpjson/README.md | 69 +++++++++++++++++++++++++++++++ plugins/httpjson/httpjson.go | 6 +-- plugins/httpjson/httpjson_test.go | 1 - 3 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 plugins/httpjson/README.md diff --git a/plugins/httpjson/README.md b/plugins/httpjson/README.md new file mode 100644 index 000000000..53ee7d8fc --- /dev/null +++ b/plugins/httpjson/README.md @@ -0,0 +1,69 @@ +# HTTP JSON Plugin + +The httpjson plugin can collect data from remote URLs which respond with JSON. Then it flattens JSON and finds all numeric values, treating them as floats. + +For example, if you have a service called _mycollector_, which has HTTP endpoint for gathering stats http://my.service.com/_stats: + +``` +[[httpjson.services]] + name = "mycollector" + + servers = [ + "http://my.service.com/_stats" + ] + + # HTTP method to use (case-sensitive) + method = "GET" +``` + +The name is used as a prefix for the measurements. + +The `method` specifies HTTP method to use for requests. + +You can specify which keys from server response should be considered as tags: + +``` +[[httpjson.services]] + ... + + tagKeys = [ + "role", + "version" + ] +``` + +**NOTE**: tag values should be strings. + +You can also specify additional request parameters for the service: + +``` +[[httpjson.services]] + ... + + [httpjson.services.parameters] + event_type = "cpu_spike" + threshold = "0.75" + +``` + + +# Sample + +Let's say that we have a service named "mycollector", which responds with: +```json +{ + "a": 0.5, + "b": { + "c": "some text", + "d": 0.1, + "e": 5 + } +} +``` + +The collected metrics will be: +``` +httpjson_mycollector_a value=0.5 +httpjson_mycollector_b_d value=0.1 +httpjson_mycollector_b_e value=5 +``` diff --git a/plugins/httpjson/httpjson.go b/plugins/httpjson/httpjson.go index 5c9bee02a..88a7cbe63 100644 --- a/plugins/httpjson/httpjson.go +++ b/plugins/httpjson/httpjson.go @@ -22,7 +22,7 @@ type Service struct { Name string Servers []string Method string - TagKeys []string + TagKeys []string Parameters map[string]string } @@ -144,8 +144,8 @@ func (h *HttpJson) gatherServer(acc plugins.Accumulator, service Service, server for _, tag := range service.TagKeys { switch v := jsonOut[tag].(type) { - case string: - tags[tag] = v + case string: + tags[tag] = v } delete(jsonOut, tag) } diff --git a/plugins/httpjson/httpjson_test.go b/plugins/httpjson/httpjson_test.go index 335568724..8f9bfe3ac 100644 --- a/plugins/httpjson/httpjson_test.go +++ b/plugins/httpjson/httpjson_test.go @@ -35,7 +35,6 @@ const validJSONTags = ` "build": "123" }` - const invalidJSON = "I don't think this is JSON" const empty = ""