Utilizing new client and overhauling Accumulator interface

Fixes #280
Fixes #281
Fixes #289
This commit is contained in:
Cameron Sparr
2015-10-16 16:13:32 -06:00
parent 6263bc2d1b
commit c26ce9c4fe
27 changed files with 498 additions and 550 deletions

View File

@@ -6,7 +6,7 @@ import (
"sync"
"time"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
"github.com/influxdb/telegraf/outputs"
"github.com/streadway/amqp"
)
@@ -82,39 +82,21 @@ func (q *AMQP) Description() string {
return "Configuration for the AMQP server to send metrics to"
}
func (q *AMQP) Write(bp client.BatchPoints) error {
func (q *AMQP) Write(points []*client.Point) error {
q.Lock()
defer q.Unlock()
if len(bp.Points) == 0 {
if len(points) == 0 {
return nil
}
var zero_time time.Time
for _, p := range bp.Points {
for _, p := range points {
// Combine tags from Point and BatchPoints and grab the resulting
// line-protocol output string to write to AMQP
var value, key string
if p.Raw != "" {
value = p.Raw
} else {
for k, v := range bp.Tags {
if p.Tags == nil {
p.Tags = make(map[string]string, len(bp.Tags))
}
p.Tags[k] = v
}
if p.Time == zero_time {
if bp.Time == zero_time {
p.Time = time.Now()
} else {
p.Time = bp.Time
}
}
value = p.MarshalString()
}
value = p.String()
if q.RoutingTag != "" {
if h, ok := p.Tags[q.RoutingTag]; ok {
if h, ok := p.Tags()[q.RoutingTag]; ok {
key = h
}
}

View File

@@ -23,6 +23,6 @@ func TestConnectAndWrite(t *testing.T) {
require.NoError(t, err)
// Verify that we can successfully write data to the amqp broker
err = q.Write(testutil.MockBatchPoints())
err = q.Write(testutil.MockBatchPoints().Points())
require.NoError(t, err)
}

View File

@@ -8,7 +8,7 @@ import (
"net/url"
"sort"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
t "github.com/influxdb/telegraf"
"github.com/influxdb/telegraf/outputs"
)
@@ -59,19 +59,19 @@ func (d *Datadog) Connect() error {
return nil
}
func (d *Datadog) Write(bp client.BatchPoints) error {
if len(bp.Points) == 0 {
func (d *Datadog) Write(points []*client.Point) error {
if len(points) == 0 {
return nil
}
ts := TimeSeries{
Series: make([]*Metric, len(bp.Points)),
Series: make([]*Metric, len(points)),
}
for index, pt := range bp.Points {
for index, pt := range points {
metric := &Metric{
Metric: pt.Measurement,
Tags: buildTags(bp.Tags, pt.Tags),
Metric: pt.Name(),
Tags: buildTags(pt.Tags()),
}
if p, err := buildPoint(bp, pt); err == nil {
if p, err := buildPoint(pt); err == nil {
metric.Points[0] = p
}
ts.Series[index] = metric
@@ -114,13 +114,18 @@ func (d *Datadog) authenticatedUrl() string {
return fmt.Sprintf("%s?%s", d.apiUrl, q.Encode())
}
func buildTags(bpTags map[string]string, ptTags map[string]string) []string {
tags := make([]string, (len(bpTags) + len(ptTags)))
index := 0
for k, v := range bpTags {
tags[index] = fmt.Sprintf("%s:%s", k, v)
index += 1
func buildPoint(pt *client.Point) (Point, error) {
var p Point
if err := p.setValue(pt.Fields()["value"]); err != nil {
return p, fmt.Errorf("unable to extract value from Fields, %s", err.Error())
}
p[0] = float64(pt.Time().Unix())
return p, nil
}
func buildTags(ptTags map[string]string) []string {
tags := make([]string, len(ptTags))
index := 0
for k, v := range ptTags {
tags[index] = fmt.Sprintf("%s:%s", k, v)
index += 1
@@ -129,19 +134,6 @@ func buildTags(bpTags map[string]string, ptTags map[string]string) []string {
return tags
}
func buildPoint(bp client.BatchPoints, pt client.Point) (Point, error) {
var p Point
if err := p.setValue(pt.Fields["value"]); err != nil {
return p, fmt.Errorf("unable to extract value from Fields, %s", err.Error())
}
if pt.Time.IsZero() {
p[0] = float64(bp.Time.Unix())
} else {
p[0] = float64(pt.Time.Unix())
}
return p, nil
}
func (p *Point) setValue(v interface{}) error {
switch d := v.(type) {
case int:

View File

@@ -11,7 +11,7 @@ import (
"github.com/influxdb/telegraf/testutil"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -38,7 +38,7 @@ func TestUriOverride(t *testing.T) {
d.Apikey = "123456"
err := d.Connect()
require.NoError(t, err)
err = d.Write(testutil.MockBatchPoints())
err = d.Write(testutil.MockBatchPoints().Points())
require.NoError(t, err)
}
@@ -57,7 +57,7 @@ func TestBadStatusCode(t *testing.T) {
d.Apikey = "123456"
err := d.Connect()
require.NoError(t, err)
err = d.Write(testutil.MockBatchPoints())
err = d.Write(testutil.MockBatchPoints().Points())
if err == nil {
t.Errorf("error expected but none returned")
} else {
@@ -74,28 +74,24 @@ func TestAuthenticatedUrl(t *testing.T) {
func TestBuildTags(t *testing.T) {
var tagtests = []struct {
bpIn map[string]string
ptIn map[string]string
outTags []string
}{
{
map[string]string{"one": "two"},
map[string]string{"three": "four"},
map[string]string{"one": "two", "three": "four"},
[]string{"one:two", "three:four"},
},
{
map[string]string{"aaa": "bbb"},
map[string]string{},
[]string{"aaa:bbb"},
},
{
map[string]string{},
map[string]string{},
[]string{},
},
}
for _, tt := range tagtests {
tags := buildTags(tt.bpIn, tt.ptIn)
tags := buildTags(tt.ptIn)
if !reflect.DeepEqual(tags, tt.outTags) {
t.Errorf("\nexpected %+v\ngot %+v\n", tt.outTags, tags)
}
@@ -103,92 +99,114 @@ func TestBuildTags(t *testing.T) {
}
func TestBuildPoint(t *testing.T) {
tags := make(map[string]string)
var tagtests = []struct {
bpIn client.BatchPoints
ptIn client.Point
ptIn *client.Point
outPt Point
err error
}{
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test1",
tags,
map[string]interface{}{"value": 0.0},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
0.0,
},
client.Point{
Fields: map[string]interface{}{"value": 0.0},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 0.0},
nil,
},
{
client.BatchPoints{},
client.Point{
Fields: map[string]interface{}{"value": 1.0},
Time: time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test2",
tags,
map[string]interface{}{"value": 1.0},
time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC).Unix()),
1.0,
},
Point{float64(time.Date(2010, time.December, 10, 23, 0, 0, 0, time.UTC).Unix()), 1.0},
nil,
},
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test3",
tags,
map[string]interface{}{"value": 10},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
10.0,
},
client.Point{
Fields: map[string]interface{}{"value": 10},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 10.0},
nil,
},
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test4",
tags,
map[string]interface{}{"value": int32(112345)},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
112345.0,
},
client.Point{
Fields: map[string]interface{}{"value": int32(112345)},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 112345.0},
nil,
},
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test5",
tags,
map[string]interface{}{"value": int64(112345)},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
112345.0,
},
client.Point{
Fields: map[string]interface{}{"value": int64(112345)},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 112345.0},
nil,
},
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test6",
tags,
map[string]interface{}{"value": float32(11234.5)},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
11234.5,
},
client.Point{
Fields: map[string]interface{}{"value": float32(11234.5)},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 11234.5},
nil,
},
{
client.BatchPoints{
Time: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
client.NewPoint(
"test7",
tags,
map[string]interface{}{"value": "11234.5"},
time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
),
Point{
float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()),
11234.5,
},
client.Point{
Fields: map[string]interface{}{"value": "11234.5"},
},
Point{float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).Unix()), 11234.5},
fmt.Errorf("unable to extract value from Fields, undeterminable type"),
},
}
for _, tt := range tagtests {
pt, err := buildPoint(tt.bpIn, tt.ptIn)
pt, err := buildPoint(tt.ptIn)
if err != nil && tt.err == nil {
t.Errorf("unexpected error, %+v\n", err)
t.Errorf("%s: unexpected error, %+v\n", tt.ptIn.Name(), err)
}
if tt.err != nil && err == nil {
t.Errorf("expected an error (%s) but none returned", tt.err.Error())
t.Errorf("%s: expected an error (%s) but none returned", tt.ptIn.Name(), tt.err.Error())
}
if !reflect.DeepEqual(pt, tt.outPt) && tt.err == nil {
t.Errorf("\nexpected %+v\ngot %+v\n", tt.outPt, pt)
t.Errorf("%s: \nexpected %+v\ngot %+v\n", tt.ptIn.Name(), tt.outPt, pt)
}
}
}

View File

@@ -8,7 +8,7 @@ import (
"net/url"
"strings"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
t "github.com/influxdb/telegraf"
"github.com/influxdb/telegraf/outputs"
)
@@ -21,9 +21,10 @@ type InfluxDB struct {
Password string
Database string
UserAgent string
Precision string
Timeout t.Duration
conns []*client.Client
conns []client.Client
}
var sampleConfig = `
@@ -32,6 +33,7 @@ var sampleConfig = `
urls = ["http://localhost:8086"] # required
# The target database for metrics (telegraf will create it if not exists)
database = "telegraf" # required
precision = "s"
# Connection timeout (for the connection with InfluxDB), formatted as a string.
# Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
@@ -63,18 +65,15 @@ func (i *InfluxDB) Connect() error {
urls = append(urls, u)
}
var conns []*client.Client
var conns []client.Client
for _, parsed_url := range urls {
c, err := client.NewClient(client.Config{
URL: *parsed_url,
c := client.NewClient(client.Config{
URL: parsed_url,
Username: i.Username,
Password: i.Password,
UserAgent: i.UserAgent,
Timeout: i.Timeout.Duration,
})
if err != nil {
return err
}
conns = append(conns, c)
}
@@ -113,15 +112,22 @@ func (i *InfluxDB) Description() string {
// Choose a random server in the cluster to write to until a successful write
// occurs, logging each unsuccessful. If all servers fail, return error.
func (i *InfluxDB) Write(bp client.BatchPoints) error {
bp.Database = i.Database
func (i *InfluxDB) Write(points []*client.Point) error {
bp, _ := client.NewBatchPoints(client.BatchPointsConfig{
Database: i.Database,
Precision: i.Precision,
})
for _, point := range points {
bp.AddPoint(point)
}
// This will get set to nil if a successful write occurs
err := errors.New("Could not write to any InfluxDB server in cluster")
p := rand.Perm(len(i.conns))
for _, n := range p {
if _, e := i.conns[n].Write(bp); e != nil {
if e := i.conns[n].Write(bp); e != nil {
log.Println("ERROR: " + e.Error())
} else {
err = nil

View File

@@ -3,10 +3,9 @@ package kafka
import (
"errors"
"fmt"
"time"
"github.com/Shopify/sarama"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
"github.com/influxdb/telegraf/outputs"
)
@@ -52,40 +51,21 @@ func (k *Kafka) Description() string {
return "Configuration for the Kafka server to send metrics to"
}
func (k *Kafka) Write(bp client.BatchPoints) error {
if len(bp.Points) == 0 {
func (k *Kafka) Write(points []*client.Point) error {
if len(points) == 0 {
return nil
}
var zero_time time.Time
for _, p := range bp.Points {
for _, p := range points {
// Combine tags from Point and BatchPoints and grab the resulting
// line-protocol output string to write to Kafka
var value string
if p.Raw != "" {
value = p.Raw
} else {
for k, v := range bp.Tags {
if p.Tags == nil {
p.Tags = make(map[string]string, len(bp.Tags))
}
p.Tags[k] = v
}
if p.Time == zero_time {
if bp.Time == zero_time {
p.Time = time.Now()
} else {
p.Time = bp.Time
}
}
value = p.MarshalString()
}
value := p.String()
m := &sarama.ProducerMessage{
Topic: k.Topic,
Value: sarama.StringEncoder(value),
}
if h, ok := p.Tags[k.RoutingTag]; ok {
if h, ok := p.Tags()[k.RoutingTag]; ok {
m.Key = sarama.StringEncoder(h)
}

View File

@@ -23,6 +23,6 @@ func TestConnectAndWrite(t *testing.T) {
require.NoError(t, err)
// Verify that we can successfully write data to the kafka broker
err = k.Write(testutil.MockBatchPoints())
err = k.Write(testutil.MockBatchPoints().Points())
require.NoError(t, err)
}

View File

@@ -10,7 +10,7 @@ import (
"sync"
paho "git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
t "github.com/influxdb/telegraf"
"github.com/influxdb/telegraf/outputs"
)
@@ -78,35 +78,31 @@ func (m *MQTT) Description() string {
return "Configuration for MQTT server to send metrics to"
}
func (m *MQTT) Write(bp client.BatchPoints) error {
func (m *MQTT) Write(points []*client.Point) error {
m.Lock()
defer m.Unlock()
if len(bp.Points) == 0 {
if len(points) == 0 {
return nil
}
hostname, ok := bp.Tags["host"]
hostname, ok := points[0].Tags()["host"]
if !ok {
hostname = ""
}
for _, p := range bp.Points {
for _, p := range points {
var t []string
if m.TopicPrefix != "" {
t = append(t, m.TopicPrefix)
}
tm := strings.Split(p.Measurement, "_")
tm := strings.Split(p.Name(), "_")
if len(tm) < 2 {
tm = []string{p.Measurement, "stat"}
tm = []string{p.Name(), "stat"}
}
t = append(t, "host", hostname, tm[0], tm[1])
topic := strings.Join(t, "/")
var value string
if p.Raw != "" {
value = p.Raw
} else {
value = getValue(p.Fields["value"])
}
value := p.String()
err := m.publish(topic, value)
if err != nil {
return fmt.Errorf("Could not write to MQTT server, %s", err)
@@ -116,23 +112,6 @@ func (m *MQTT) Write(bp client.BatchPoints) error {
return nil
}
func getValue(v interface{}) string {
var ret string
switch v.(type) {
default:
ret = fmt.Sprintf("%v", v)
case bool:
ret = fmt.Sprintf("%t", v)
case float32, float64:
ret = fmt.Sprintf("%f", v)
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
ret = fmt.Sprintf("%d", v)
case string, []byte:
ret = fmt.Sprintf("%s", v)
}
return ret
}
func (m *MQTT) publish(topic, body string) error {
token := m.Client.Publish(topic, 0, false, body)
token.Wait()

View File

@@ -8,7 +8,7 @@ import (
"strings"
"time"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
"github.com/influxdb/telegraf/outputs"
)
@@ -51,15 +51,15 @@ func (o *OpenTSDB) Connect() error {
return fmt.Errorf("OpenTSDB: TCP address cannot be resolved")
}
connection, err := net.DialTCP("tcp", nil, tcpAddr)
defer connection.Close()
if err != nil {
return fmt.Errorf("OpenTSDB: Telnet connect fail")
}
defer connection.Close()
return nil
}
func (o *OpenTSDB) Write(bp client.BatchPoints) error {
if len(bp.Points) == 0 {
func (o *OpenTSDB) Write(points []*client.Point) error {
if len(points) == 0 {
return nil
}
var timeNow = time.Now()
@@ -70,19 +70,20 @@ func (o *OpenTSDB) Write(bp client.BatchPoints) error {
if err != nil {
return fmt.Errorf("OpenTSDB: Telnet connect fail")
}
for _, pt := range bp.Points {
for _, pt := range points {
metric := &MetricLine{
Metric: fmt.Sprintf("%s%s", o.Prefix, pt.Measurement),
Metric: fmt.Sprintf("%s%s", o.Prefix, pt.Name()),
Timestamp: timeNow.Unix(),
}
metricValue, buildError := buildValue(bp, pt)
metricValue, buildError := buildValue(pt)
if buildError != nil {
fmt.Printf("OpenTSDB: %s\n", buildError.Error())
continue
}
metric.Value = metricValue
tagsSlice := buildTags(bp.Tags, pt.Tags)
tagsSlice := buildTags(pt.Tags())
metric.Tags = fmt.Sprint(strings.Join(tagsSlice, " "))
messageLine := fmt.Sprintf("put %s %v %s %s\n", metric.Metric, metric.Timestamp, metric.Value, metric.Tags)
@@ -99,13 +100,9 @@ func (o *OpenTSDB) Write(bp client.BatchPoints) error {
return nil
}
func buildTags(bpTags map[string]string, ptTags map[string]string) []string {
tags := make([]string, (len(bpTags) + len(ptTags)))
func buildTags(ptTags map[string]string) []string {
tags := make([]string, len(ptTags))
index := 0
for k, v := range bpTags {
tags[index] = fmt.Sprintf("%s=%s", k, v)
index += 1
}
for k, v := range ptTags {
tags[index] = fmt.Sprintf("%s=%s", k, v)
index += 1
@@ -114,9 +111,9 @@ func buildTags(bpTags map[string]string, ptTags map[string]string) []string {
return tags
}
func buildValue(bp client.BatchPoints, pt client.Point) (string, error) {
func buildValue(pt *client.Point) (string, error) {
var retv string
var v = pt.Fields["value"]
var v = pt.Fields()["value"]
switch p := v.(type) {
case int64:
retv = IntToString(int64(p))

View File

@@ -3,47 +3,42 @@ package opentsdb
import (
"reflect"
"testing"
"time"
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
"github.com/influxdb/telegraf/testutil"
"github.com/stretchr/testify/require"
)
func TestBuildTagsTelnet(t *testing.T) {
var tagtests = []struct {
bpIn map[string]string
ptIn map[string]string
outTags []string
}{
{
map[string]string{"one": "two"},
map[string]string{"three": "four"},
map[string]string{"one": "two", "three": "four"},
[]string{"one=two", "three=four"},
},
{
map[string]string{"aaa": "bbb"},
map[string]string{},
[]string{"aaa=bbb"},
},
{
map[string]string{"one": "two"},
map[string]string{"aaa": "bbb"},
map[string]string{"one": "two", "aaa": "bbb"},
[]string{"aaa=bbb", "one=two"},
},
{
map[string]string{},
map[string]string{},
[]string{},
},
}
for _, tt := range tagtests {
tags := buildTags(tt.bpIn, tt.ptIn)
tags := buildTags(tt.ptIn)
if !reflect.DeepEqual(tags, tt.outTags) {
t.Errorf("\nexpected %+v\ngot %+v\n", tt.outTags, tags)
}
}
}
func TestWrite(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
@@ -60,36 +55,24 @@ func TestWrite(t *testing.T) {
require.NoError(t, err)
// Verify that we can successfully write data to OpenTSDB
err = o.Write(testutil.MockBatchPoints())
err = o.Write(testutil.MockBatchPoints().Points())
require.NoError(t, err)
// Verify postive and negative test cases of writing data
var bp client.BatchPoints
bp.Time = time.Now()
bp.Tags = map[string]string{"testkey": "testvalue"}
bp.Points = []client.Point{
{
Measurement: "justametric.float",
Fields: map[string]interface{}{"value": float64(1.0)},
},
{
Measurement: "justametric.int",
Fields: map[string]interface{}{"value": int64(123456789)},
},
{
Measurement: "justametric.uint",
Fields: map[string]interface{}{"value": uint64(123456789012345)},
},
{
Measurement: "justametric.string",
Fields: map[string]interface{}{"value": "Lorem Ipsum"},
},
{
Measurement: "justametric.anotherfloat",
Fields: map[string]interface{}{"value": float64(42.0)},
},
}
err = o.Write(bp)
bp := testutil.MockBatchPoints()
tags := make(map[string]string)
bp.AddPoint(client.NewPoint("justametric.float", tags,
map[string]interface{}{"value": float64(1.0)}))
bp.AddPoint(client.NewPoint("justametric.int", tags,
map[string]interface{}{"value": int64(123456789)}))
bp.AddPoint(client.NewPoint("justametric.uint", tags,
map[string]interface{}{"value": uint64(123456789012345)}))
bp.AddPoint(client.NewPoint("justametric.string", tags,
map[string]interface{}{"value": "Lorem Ipsum"}))
bp.AddPoint(client.NewPoint("justametric.anotherfloat", tags,
map[string]interface{}{"value": float64(42.0)}))
err = o.Write(bp.Points())
require.NoError(t, err)
}

View File

@@ -1,7 +1,7 @@
package outputs
import (
"github.com/influxdb/influxdb/client"
"github.com/influxdb/influxdb/client/v2"
)
type Output interface {
@@ -9,7 +9,7 @@ type Output interface {
Close() error
Description() string
SampleConfig() string
Write(client.BatchPoints) error
Write(points []*client.Point) error
}
type Creator func() Output