dom/telegraf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

OpenTSDB HTTP output

Browse Source 
closes #1539

First version of http put working

Refactored code to separate http handling from opentsdb module. Added batching support.

Fixed tag cleaning in http output and refactored telnet output.

Removed useless struct.

Fixed current unittest and added a new one.

Added benchmark test to test json serialization. Made sure http client would reuse connection.

Ran go fmt on opentsdb sources.

Updated README file

Removed useHttp in favor of parsing host string to determine the right API to use for sending metrics. Also renamed BatchSize to HttpBatchSize to better convey that it is only used when using Http API.

Updated changelog

Fixed format issues.

Removed TagSet type to make it more explicit.

Fixed unittest after removing TagSet type.

Revert "Updated changelog"

This reverts commit 24dba5520008d876b5a8d266c34a53e8805cc5f5.

Added PR under 1.1 release.

add missing redis metrics

This makes sure that all redis metrics are present without having to use a hard-coded list of what metrics to pull in.
This commit is contained in:
Eric
2016-07-18 15:01:36 -04:00
committed by Cameron Sparr
parent a8f9eb23cc
commit 6e33a6d62f
5 changed files with 373 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
plugins/outputs/opentsdb/opentsdb.go
View File

@@ -3,10 +3,10 @@ package opentsdb
import (
"fmt"
"net"
"net/url"
"sort"
"strconv"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/outputs"
@@ -18,6 +18,8 @@ type OpenTSDB struct {
Host string
Port int
HttpBatchSize int
Debug bool
}
@@ -28,27 +30,41 @@ var sampleConfig = `
## prefix for metrics keys
prefix = "my.specific.prefix."
## Telnet Mode ##
## DNS name of the OpenTSDB server in telnet mode
## DNS name of the OpenTSDB server
## Using "opentsdb.example.com" or "tcp://opentsdb.example.com" will use the
## telnet API. "http://opentsdb.example.com" will use the Http API.
host = "opentsdb.example.com"
## Port of the OpenTSDB server in telnet mode
## Port of the OpenTSDB server
port = 4242
## Number of data points to send to OpenTSDB in Http requests.
## Not used with telnet API.
httpBatchSize = 50
## Debug true - Prints OpenTSDB communication
debug = false
`
type MetricLine struct {
Metric string
Timestamp int64
Value string
Tags string
func ToLineFormat(tags map[string]string) string {
tagsArray := make([]string, len(tags))
index := 0
for k, v := range tags {
tagsArray[index] = fmt.Sprintf("%s=%s", k, v)
index++
}
sort.Strings(tagsArray)
return strings.Join(tagsArray, " ")
}
func (o *OpenTSDB) Connect() error {
// Test Connection to OpenTSDB Server
uri := fmt.Sprintf("%s:%d", o.Host, o.Port)
u, err := url.Parse(o.Host)
if err != nil {
return fmt.Errorf("Error in parsing host url: %s", err.Error())
}
uri := fmt.Sprintf("%s:%d", u.Host, o.Port)
tcpAddr, err := net.ResolveTCPAddr("tcp", uri)
if err != nil {
return fmt.Errorf("OpenTSDB: TCP address cannot be resolved")
@@ -65,10 +81,64 @@ func (o *OpenTSDB) Write(metrics []telegraf.Metric) error {
if len(metrics) == 0 {
return nil
}
now := time.Now()
u, err := url.Parse(o.Host)
if err != nil {
return fmt.Errorf("Error in parsing host url: %s", err.Error())
}
if u.Scheme == "" || u.Scheme == "tcp" {
return o.WriteTelnet(metrics, u)
} else if u.Scheme == "http" {
return o.WriteHttp(metrics, u)
} else {
return fmt.Errorf("Unknown scheme in host parameter.")
}
}
func (o *OpenTSDB) WriteHttp(metrics []telegraf.Metric, u *url.URL) error {
http := openTSDBHttp{
Host: u.Host,
Port: o.Port,
BatchSize: o.HttpBatchSize,
Debug: o.Debug,
}
for _, m := range metrics {
now := m.UnixNano() / 1000000000
tags := cleanTags(m.Tags())
for fieldName, value := range m.Fields() {
metricValue, buildError := buildValue(value)
if buildError != nil {
fmt.Printf("OpenTSDB: %s\n", buildError.Error())
continue
}
metric := &HttpMetric{
Metric: sanitizedChars.Replace(fmt.Sprintf("%s%s_%s",
o.Prefix, m.Name(), fieldName)),
Tags: tags,
Timestamp: now,
Value: metricValue,
}
if err := http.sendDataPoint(metric); err != nil {
return err
}
}
}
if err := http.flush(); err != nil {
return err
}
return nil
}
func (o *OpenTSDB) WriteTelnet(metrics []telegraf.Metric, u *url.URL) error {
// Send Data with telnet / socket communication
uri := fmt.Sprintf("%s:%d", o.Host, o.Port)
uri := fmt.Sprintf("%s:%d", u.Host, o.Port)
tcpAddr, _ := net.ResolveTCPAddr("tcp", uri)
connection, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
@@ -77,9 +147,20 @@ func (o *OpenTSDB) Write(metrics []telegraf.Metric) error {
defer connection.Close()
for _, m := range metrics {
for _, metric := range buildMetrics(m, now, o.Prefix) {
now := m.UnixNano() / 1000000000
tags := ToLineFormat(cleanTags(m.Tags()))
for fieldName, value := range m.Fields() {
metricValue, buildError := buildValue(value)
if buildError != nil {
fmt.Printf("OpenTSDB: %s\n", buildError.Error())
continue
}
messageLine := fmt.Sprintf("put %s %v %s %s\n",
metric.Metric, metric.Timestamp, metric.Value, metric.Tags)
sanitizedChars.Replace(fmt.Sprintf("%s%s_%s", o.Prefix, m.Name(), fieldName)),
now, metricValue, tags)
if o.Debug {
fmt.Print(messageLine)
}
@@ -93,37 +174,12 @@ func (o *OpenTSDB) Write(metrics []telegraf.Metric) error {
return nil
}
func buildTags(mTags map[string]string) []string {
tags := make([]string, len(mTags))
index := 0
for k, v := range mTags {
tags[index] = sanitizedChars.Replace(fmt.Sprintf("%s=%s", k, v))
index++
func cleanTags(tags map[string]string) map[string]string {
tagSet := make(map[string]string, len(tags))
for k, v := range tags {
tagSet[sanitizedChars.Replace(k)] = sanitizedChars.Replace(v)
}
sort.Strings(tags)
return tags
}
func buildMetrics(m telegraf.Metric, now time.Time, prefix string) []*MetricLine {
ret := []*MetricLine{}
for fieldName, value := range m.Fields() {
metric := &MetricLine{
Metric: sanitizedChars.Replace(fmt.Sprintf("%s%s_%s",
prefix, m.Name(), fieldName)),
Timestamp: now.Unix(),
}
metricValue, buildError := buildValue(value)
if buildError != nil {
fmt.Printf("OpenTSDB: %s\n", buildError.Error())
continue
}
metric.Value = metricValue
tagsSlice := buildTags(m.Tags())
metric.Tags = fmt.Sprint(strings.Join(tagsSlice, " "))
ret = append(ret, metric)
}
return ret
return tagSet
}
func buildValue(v interface{}) (string, error) {