Kafka output producer, send telegraf metrics to Kafka brokers

Closes #38

outputs/all/all.go

@@ -3,4 +3,5 @@ package all
 import (
 	_ "github.com/influxdb/telegraf/outputs/datadog"
 	_ "github.com/influxdb/telegraf/outputs/influxdb"
+	_ "github.com/influxdb/telegraf/outputs/kafka"
 )

outputs/datadog/datadog_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/influxdb/telegraf/testutil"
+
 	"github.com/influxdb/influxdb/client"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -25,18 +27,6 @@ func fakeDatadog() *Datadog {
 	return d
 }
 
-func testData() client.BatchPoints {
-	var bp client.BatchPoints
-	bp.Time = time.Now()
-	bp.Tags = map[string]string{"tag1": "value1"}
-	bp.Points = []client.Point{
-		{
-			Fields: map[string]interface{}{"value": 1.0},
-		},
-	}
-	return bp
-}
-
 func TestUriOverride(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -48,7 +38,7 @@ func TestUriOverride(t *testing.T) {
 	d.Apikey = "123456"
 	err := d.Connect()
 	require.NoError(t, err)
-	err = d.Write(testData())
+	err = d.Write(testutil.MockBatchPoints())
 	require.NoError(t, err)
 }
 
@@ -67,7 +57,7 @@ func TestBadStatusCode(t *testing.T) {
 	d.Apikey = "123456"
 	err := d.Connect()
 	require.NoError(t, err)
-	err = d.Write(testData())
+	err = d.Write(testutil.MockBatchPoints())
 	if err == nil {
 		t.Errorf("error expected but none returned")
 	} else {

outputs/kafka/kafka.go

@@ -0,0 +1,91 @@
+package kafka
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/Shopify/sarama"
+	"github.com/influxdb/influxdb/client"
+	"github.com/influxdb/telegraf/outputs"
+)
+
+type Kafka struct {
+	// Kafka brokers to send metrics to
+	Brokers []string
+	// Kafka topic
+	Topic string
+
+	producer sarama.SyncProducer
+}
+
+var sampleConfig = `
+    # URLs of kafka brokers
+    brokers = ["localhost:9092"]
+    # Kafka topic for producer messages
+    topic = "telegraf"
+`
+
+func (k *Kafka) Connect() error {
+	producer, err := sarama.NewSyncProducer(k.Brokers, nil)
+	if err != nil {
+		return err
+	}
+	k.producer = producer
+	return nil
+}
+
+func (k *Kafka) Close() error {
+	return k.producer.Close()
+}
+
+func (k *Kafka) SampleConfig() string {
+	return sampleConfig
+}
+
+func (k *Kafka) Description() string {
+	return "Configuration for the Kafka server to send metrics to"
+}
+
+func (k *Kafka) Write(bp client.BatchPoints) error {
+	if len(bp.Points) == 0 {
+		return nil
+	}
+
+	for _, p := range bp.Points {
+		// Combine tags from Point and BatchPoints and grab the resulting
+		// line-protocol output string to write to Kafka
+		var value string
+		if p.Raw != "" {
+			value = p.Raw
+		} else {
+			for k, v := range bp.Tags {
+				if p.Tags == nil {
+					p.Tags = make(map[string]string, len(bp.Tags))
+				}
+				p.Tags[k] = v
+			}
+			value = p.MarshalString()
+		}
+
+		m := &sarama.ProducerMessage{
+			Topic: k.Topic,
+			Value: sarama.StringEncoder(value),
+		}
+		if h, ok := p.Tags["host"]; ok {
+			m.Key = sarama.StringEncoder(h)
+		}
+
+		_, _, err := k.producer.SendMessage(m)
+		if err != nil {
+			return errors.New(fmt.Sprintf("FAILED to send kafka message: %s\n",
+				err))
+		}
+	}
+	return nil
+}
+
+func init() {
+	outputs.Add("kafka", func() outputs.Output {
+		return &Kafka{}
+	})
+}

outputs/kafka/kafka_test.go

@@ -0,0 +1,28 @@
+package kafka
+
+import (
+	"testing"
+
+	"github.com/influxdb/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectAndWrite(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	brokers := []string{testutil.GetLocalHost() + ":9092"}
+	k := &Kafka{
+		Brokers: brokers,
+		Topic:   "Test",
+	}
+
+	// Verify that we can connect to the Kafka broker
+	err := k.Connect()
+	require.NoError(t, err)
+
+	// Verify that we can successfully write data to the kafka broker
+	err = k.Write(testutil.MockBatchPoints())
+	require.NoError(t, err)
+}