From a627c348a4b44088f2b6394de32396d5f7fdd094 Mon Sep 17 00:00:00 2001 From: Daniel Hahn Date: Fri, 15 Apr 2016 19:09:00 +0200 Subject: [PATCH] Newrelic Plugin Adds a plugin to send new relic metric data. This is generally functional, although not all kinks are worked out. We currently use a "demo" uid for testing purposes. --- .gitignore | 1 + README.md | 1 + plugins/outputs/all/all.go | 1 + plugins/outputs/newrelic/newrelic.go | 128 ++++++++++++++++++ .../outputs/newrelic/newrelic_component.go | 91 +++++++++++++ .../newrelic/newrelic_component_test.go | 42 ++++++ plugins/outputs/newrelic/newrelic_data.go | 35 +++++ .../outputs/newrelic/newrelic_data_test.go | 35 +++++ plugins/outputs/newrelic/newrelic_tags.go | 45 ++++++ .../outputs/newrelic/newrelic_tags_test.go | 36 +++++ plugins/outputs/newrelic/test_structures.go | 29 ++++ 11 files changed, 444 insertions(+) create mode 100644 plugins/outputs/newrelic/newrelic.go create mode 100644 plugins/outputs/newrelic/newrelic_component.go create mode 100644 plugins/outputs/newrelic/newrelic_component_test.go create mode 100644 plugins/outputs/newrelic/newrelic_data.go create mode 100644 plugins/outputs/newrelic/newrelic_data_test.go create mode 100644 plugins/outputs/newrelic/newrelic_tags.go create mode 100644 plugins/outputs/newrelic/newrelic_tags_test.go create mode 100644 plugins/outputs/newrelic/test_structures.go diff --git a/.gitignore b/.gitignore index 8269337df..4fdf4ae72 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tivan .idea *~ *# +.tags diff --git a/README.md b/README.md index 53e672534..448dbf356 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ want to add support for another service or third-party API. * [opentsdb](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/opentsdb) * [prometheus](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/prometheus_client) * [riemann](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/riemann) +* [newrelic](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/riemann) ## Contributing diff --git a/plugins/outputs/all/all.go b/plugins/outputs/all/all.go index 27f8958fe..39b31f3a9 100644 --- a/plugins/outputs/all/all.go +++ b/plugins/outputs/all/all.go @@ -14,6 +14,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/outputs/kinesis" _ "github.com/influxdata/telegraf/plugins/outputs/librato" _ "github.com/influxdata/telegraf/plugins/outputs/mqtt" + _ "github.com/influxdata/telegraf/plugins/outputs/newrelic" _ "github.com/influxdata/telegraf/plugins/outputs/nsq" _ "github.com/influxdata/telegraf/plugins/outputs/opentsdb" _ "github.com/influxdata/telegraf/plugins/outputs/prometheus_client" diff --git a/plugins/outputs/newrelic/newrelic.go b/plugins/outputs/newrelic/newrelic.go new file mode 100644 index 000000000..cb9e9efea --- /dev/null +++ b/plugins/outputs/newrelic/newrelic.go @@ -0,0 +1,128 @@ +package newrelic + +import ( + "bytes" + "fmt" + "time" + "net/http" + // "io" + + "encoding/json" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/outputs" + "github.com/influxdata/telegraf/internal" +) + +type NewRelic struct { + ApiKey string + GuidBase string + Timeout internal.Duration + + LastWrite time.Time + client *http.Client +} + +var sampleConfig = ` + ## Your NewRelic Api Key + api_key = "XXXX" # required + + ## Guid base + ## This allows you have unique GUIDs for your installation. This will generate + ## a separate "plugin" GUID for each of the inputs that you use. + ## + ## see https://docs.newrelic.com/docs/plugins/plugin-developer-resources/planning-your-plugin/parts-plugin#guid + ## + ## This setting will allow you to "fork" your "plugins", and have your own + ## dashboards and settings for them. + ## The default behaviour is that the original author of the plugin sets up + ## all the dashboards; other users cannot modify them. + ## As it is very hard to provide useful defaults for all possible setup, we + ## instead allow you to make your "own plugin" and modify the dashboards. + ## + ## The drawback is that your GUID must be unique, and that you must setup + ## your own dashboards for everything. + ## + ## TODO: The default for this should be + ## a "proper" GUID that is maintained to have reasonable default + # guid_base = 'my.domain.something.something' # TODO must still be implemented + + ## Metric Type TODO - Not yet implemented + ## + ## Can either be "Component" or "Custom" + ## + ## Component metrics are the default for plugins. They make the metrics + ## available even to free accounts, but with the restrictions mentioned above. + ## + ## Custom metrics don't show up as plugins. They are freely usable in custom + ## dashboards, but you need to have a paid subscription to see the data. + ## + ## Default is "Component" + # metric_type = "Custom" +` +func (nr *NewRelic) Connect() error { + if nr.ApiKey == "" { + return fmt.Errorf("apikey is a required field for newrelic output") + } + + if nr.GuidBase == "" { + nr.GuidBase = "com.influxdata.demo-newrelic-agent" + } + nr.client = &http.Client{ + Timeout: nr.Timeout.Duration, + } + nr.LastWrite = time.Now() + return nil +} + +func (nr *NewRelic) Close() error { + return nil +} + +func (nr *NewRelic) SampleConfig() string { + return sampleConfig +} + +func (nr *NewRelic) Description() string { + return "Send telegraf metrics to NewRelic" +} + +func (nr *NewRelic) PostPluginData(jsonData []byte) error { + req, reqErr := http.NewRequest("POST", "https://platform-api.newrelic.com/platform/v1/metrics", bytes.NewBuffer(jsonData)) + if reqErr != nil { return reqErr } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-License-Key", nr.ApiKey) + + resp, respErr := nr.client.Do(req) + if respErr != nil { return respErr } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode > 209 { + return fmt.Errorf("received bad status code, %d\n", resp.StatusCode) + } + + return nil +} + +func (nr *NewRelic) SendDataPage(dataPage interface{}) error { + cmpJson, err := json.Marshal(dataPage) + if err != nil { return err } + fmt.Println("Sending " + string(cmpJson) + " <") + return nr.PostPluginData(cmpJson) +} + +func (nr *NewRelic) Write(metrics []telegraf.Metric) error { + data := NewRelicData{LastWrite: nr.LastWrite, Hosts: make(map[string][]NewRelicComponent)} + data.AddMetrics(metrics) + + for _, dataPage := range(data.DataSets()) { + nr.SendDataPage(dataPage) + } + nr.LastWrite = time.Now() + return nil +} + + +func init() { + outputs.Add("newrelic", func() telegraf.Output { return &NewRelic{} }) +} diff --git a/plugins/outputs/newrelic/newrelic_component.go b/plugins/outputs/newrelic/newrelic_component.go new file mode 100644 index 000000000..7678746a7 --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_component.go @@ -0,0 +1,91 @@ +package newrelic + +import( + "fmt" + "encoding/json" + "os" + "strings" + "bytes" + "github.com/influxdata/telegraf" +) + +type NewRelicComponent struct { + Duration int + TMetric telegraf.Metric + GuidBase string + tags *NewRelicTags +} + +func (nrc NewRelicComponent) Tags() *NewRelicTags { + if nrc.tags == nil { + nrc.tags = &NewRelicTags{}; + nrc.tags.Fill(nrc.TMetric.Tags()) + } + return nrc.tags +} + +func (nrc NewRelicComponent) Name() string { + return nrc.TMetric.Name() +} + +func metricValue(value interface{}) int { + result := 0 + switch value.(type) { + case int32: + result = int(value.(int32)) + case int64: + result = int(value.(int64)) + case float32: + result = int(value.(float32)) + case float64: + result = int(value.(float64)) + default: + result = 0 + } + return result +} + +func (nrc* NewRelicComponent) MetricName(originalName string) string { + var nameBuffer bytes.Buffer + nameBuffer.WriteString("Component/") + nameBuffer.WriteString(strings.Title(nrc.TMetric.Name())) + nameBuffer.WriteString("/") + nameBuffer.WriteString(strings.Title(originalName)) + tags := nrc.Tags() + for _, key := range tags.SortedKeys { + nameBuffer.WriteString(fmt.Sprintf("/%s-%s", key, tags.GetTag(key))) + } + nameBuffer.WriteString("[Units]") + return nameBuffer.String() +} + +func (nrc *NewRelicComponent) Metrics() map[string]int { + result := make(map[string]int) + for k,v := range(nrc.TMetric.Fields()) { + result[nrc.MetricName(k)] = metricValue(v) + } + return result +} + +func (nrc NewRelicComponent) Hostname() string { + result := nrc.Tags().Hostname + if result == "" { + osname, err := os.Hostname() + if err == nil { result = "unknown" } else { result = osname } + } + return result +} + +func (nrc *NewRelicComponent) Guid() string { + return fmt.Sprintf("%s-%s", nrc.GuidBase, strings.ToLower(nrc.TMetric.Name())) +} + +func (nrc NewRelicComponent) MarshalJSON() ([]byte, error) { + myData := map[string]interface{} { + "name": nrc.Hostname(), + "guid": nrc.Guid(), + "duration": nrc.Duration, + "metrics": nrc.Metrics(), + } + return json.Marshal(myData) +} diff --git a/plugins/outputs/newrelic/newrelic_component_test.go b/plugins/outputs/newrelic/newrelic_component_test.go new file mode 100644 index 000000000..e47bef949 --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_component_test.go @@ -0,0 +1,42 @@ +package newrelic + +import ( + "testing" + // "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/outputs/newrelic" + // "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestName(t *testing.T) { + dm := newrelic.DemoMetric{MyName: "Foo", TagList: newrelic.DemoTagList()} + component := newrelic.NewRelicComponent{TMetric: dm} + require.EqualValues(t, component.Name(), "Foo") +} + +func TestGuid(t *testing.T) { + dm := newrelic.DemoMetric{MyName: "Lulu", TagList: newrelic.DemoTagList()} + component := newrelic.NewRelicComponent{TMetric: dm, GuidBase: "org.betterplace.telegraf-agent"} + require.EqualValues(t, component.Guid(), "org.betterplace.telegraf-agent-lulu") +} + +func TestTags(t *testing.T) { + dm := newrelic.DemoMetric{MyName: "Lulu", TagList: newrelic.DemoTagList()} + component := newrelic.NewRelicComponent{TMetric: dm} + component.Tags() +} + +func TestHostname(t *testing.T) { + tagList := newrelic.DemoTagList() + tagList["host"] = "baba" + dm := newrelic.DemoMetric{MyName: "Lulu", TagList: tagList} + component := newrelic.NewRelicComponent{TMetric: dm} + require.EqualValues(t, component.Hostname(), "baba") +} + +func TestMetricName(t *testing.T) { + tagList := newrelic.DemoTagList() + dm := newrelic.DemoMetric{MyName: "Lulu", TagList: tagList} + component := newrelic.NewRelicComponent{TMetric: dm} + require.EqualValues(t, component.MetricName("fnord"), "Component/Lulu/Fnord/Fluff-naa/Hoof-bar/Zoo-goo[Units]") +} diff --git a/plugins/outputs/newrelic/newrelic_data.go b/plugins/outputs/newrelic/newrelic_data.go new file mode 100644 index 000000000..1c63d69ed --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_data.go @@ -0,0 +1,35 @@ +package newrelic + +import( + "time" + "github.com/influxdata/telegraf" +) + +type NewRelicData struct { + LastWrite time.Time + Hosts map[string][]NewRelicComponent + GuidBase string +} + +func (nrd *NewRelicData) AddMetric(metric telegraf.Metric) { + component := NewRelicComponent{ + Duration: int(time.Since(nrd.LastWrite).Seconds()), + TMetric: metric, + GuidBase: nrd.GuidBase} + host := component.Hostname() + nrd.Hosts[host] = append(nrd.Hosts[host],component) +} + +func (nrd *NewRelicData) AddMetrics(metrics []telegraf.Metric) { + for _, metric := range(metrics) { + nrd.AddMetric(metric) + } +} + +func (nrd *NewRelicData) DataSets() []interface{} { + result := make([]interface{}, 0) + for host, components := range(nrd.Hosts) { + result = append(result, map[string]interface{} { "agent": map[string]string { "host": host, "version": "0.0.1" }, "components": components }) + } + return result +} diff --git a/plugins/outputs/newrelic/newrelic_data_test.go b/plugins/outputs/newrelic/newrelic_data_test.go new file mode 100644 index 000000000..8b535f28d --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_data_test.go @@ -0,0 +1,35 @@ +package newrelic + +import ( + "testing" + "time" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/outputs/newrelic" + // "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" + +) + +func TestAddMetrics(t *testing.T) { + dm := []telegraf.Metric{newrelic.DemoMetric{MyName: "Foo", TagList: newrelic.DemoTagList()}} + data := newrelic.NewRelicData{ + LastWrite: time.Now(), + Hosts: make(map[string][]newrelic.NewRelicComponent), + GuidBase: "org.betterplace.test-foo"} + data.AddMetrics(dm) + require.EqualValues(t, len(data.Hosts), 1) +} + +func TestDataSets(t *testing.T) { + dm := []telegraf.Metric{newrelic.DemoMetric{MyName: "Foo", TagList: newrelic.DemoTagList()}} + data := newrelic.NewRelicData{ + LastWrite: time.Now(), + Hosts: make(map[string][]newrelic.NewRelicComponent), + GuidBase: "org.betterplace.test-foo"} + data.AddMetrics(dm) + sets := data.DataSets() + require.EqualValues(t, len(sets), 1) + set, _ := sets[0].(map[string]interface{}) + agent := set["agent"].(map[string]string) + require.EqualValues(t, agent["host"], "Hulu") +} diff --git a/plugins/outputs/newrelic/newrelic_tags.go b/plugins/outputs/newrelic/newrelic_tags.go new file mode 100644 index 000000000..a310e5854 --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_tags.go @@ -0,0 +1,45 @@ +package newrelic + +import( + "sort" + "strings" +) + +type NewRelicTags struct { + Tags *map[string]string + SortedKeys []string + Hostname string +} + +func TagValue(tagValue string) string { + tagValueParts := strings.Split(tagValue, "/") + var clean_parts []string + for _, part := range(tagValueParts) { + if part != "" { clean_parts = append(clean_parts, part) } + } + if len(clean_parts) > 0 { + tagValue = strings.ToLower(strings.Join(clean_parts, "-")) + } else { + tagValue = "root" + } + return tagValue +} + +func (nrt *NewRelicTags) Fill(originalTags map[string]string) { + nrt.SortedKeys = make([]string, 0, len(originalTags)) + tags := make(map[string]string) + nrt.Tags = &tags + for key, value := range originalTags { + if key != "host" { + nrt.SortedKeys = append(nrt.SortedKeys, key) + (*nrt.Tags)[key] = TagValue(value) + } else { + nrt.Hostname = value + } + } + sort.Strings(nrt.SortedKeys) +} + +func (nrt *NewRelicTags) GetTag(tagKey string) string { + return (*nrt.Tags)[tagKey] +} diff --git a/plugins/outputs/newrelic/newrelic_tags_test.go b/plugins/outputs/newrelic/newrelic_tags_test.go new file mode 100644 index 000000000..957da2937 --- /dev/null +++ b/plugins/outputs/newrelic/newrelic_tags_test.go @@ -0,0 +1,36 @@ +package newrelic + +import ( + "testing" + "github.com/influxdata/telegraf/plugins/outputs/newrelic" + // "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" + +) + +func TestTagValue(t *testing.T) { + require.EqualValues(t, newrelic.TagValue("Hello/World"), "hello-world") + require.EqualValues(t, newrelic.TagValue("/Hello/World"), "hello-world") + require.EqualValues(t, newrelic.TagValue(""), "root") +} + +func TestFillSimple(t *testing.T) { + tags := &newrelic.NewRelicTags{} + tags.Fill(newrelic.DemoTagList()) + require.EqualValues(t, tags.SortedKeys, []string{"Fluff", "Hoof", "Zoo"}) +} + +func TestFillWithHost(t *testing.T) { + tags := &newrelic.NewRelicTags{} + demoList := newrelic.DemoTagList() + demoList["host"] = "hulu" + tags.Fill(demoList) + require.EqualValues(t, tags.SortedKeys, []string{"Fluff", "Hoof", "Zoo"}) + require.EqualValues(t, tags.Hostname, "hulu") +} + +func TestFillAndGetTag(t *testing.T) { + tags := &newrelic.NewRelicTags{} + tags.Fill(newrelic.DemoTagList()) + require.EqualValues(t, tags.GetTag("Zoo"), "goo") +} diff --git a/plugins/outputs/newrelic/test_structures.go b/plugins/outputs/newrelic/test_structures.go new file mode 100644 index 000000000..faf2577a7 --- /dev/null +++ b/plugins/outputs/newrelic/test_structures.go @@ -0,0 +1,29 @@ +package newrelic + +import ( + "time" + "github.com/influxdata/influxdb/client/v2" +) + +func DemoTagList() map[string]string { + return map[string]string{ + "Fluff": "Naa", + "Hoof": "Bar", + "Zoo": "Goo", + "host": "Hulu", + } +} + +type DemoMetric struct { + MyName string + TagList map[string]string +} + +func (dm DemoMetric) Name() string { return dm.MyName } +func (dm DemoMetric) Tags() map[string]string { return dm.TagList } +func (dm DemoMetric) Time() time.Time { return time.Now() } +func (dm DemoMetric) UnixNano() int64 { return 0 } +func (dm DemoMetric) Fields() map[string]interface{} { return nil } +func (dm DemoMetric) String() string { return "StringRepresenation" } +func (dm DemoMetric) PrecisionString(precison string) string { return "PrecisionString" } +func (dm DemoMetric) Point() *client.Point { return nil }