Flush based on buffer size rather than time

this includes:
- Add Accumulator to the Start() function of service inputs
- For message consumer plugins, use the Accumulator to constantly add
  metrics and make Gather a dummy function
- rework unit tests to match this new behavior.
- make "flush_buffer_when_full" a config option that defaults to true

closes #666

plugins/inputs/kafka_consumer/kafka_consumer.go

@@ -1,7 +1,6 @@
 package kafka_consumer
 
 import (
-	"fmt"
 	"log"
 	"strings"
 	"sync"
@@ -19,11 +18,13 @@ type Kafka struct {
 	Topics         []string
 	ZookeeperPeers []string
 	Consumer       *consumergroup.ConsumerGroup
-	MetricBuffer   int
+
+	// Legacy metric buffer support
+	MetricBuffer int
 	// TODO remove PointBuffer, legacy support
 	PointBuffer int
-	Offset      string
 
+	Offset string
 	parser parsers.Parser
 
 	sync.Mutex
@@ -32,9 +33,10 @@ type Kafka struct {
 	in <-chan *sarama.ConsumerMessage
 	// channel for all kafka consumer errors
 	errs <-chan *sarama.ConsumerError
-	// channel for all incoming parsed kafka metrics
-	metricC chan telegraf.Metric
-	done    chan struct{}
+	done chan struct{}
+
+	// keep the accumulator internally:
+	acc telegraf.Accumulator
 
 	// doNotCommitMsgs tells the parser not to call CommitUpTo on the consumer
 	// this is mostly for test purposes, but there may be a use-case for it later.
@@ -48,8 +50,6 @@ var sampleConfig = `
   zookeeper_peers = ["localhost:2181"]
   ### the name of the consumer group
   consumer_group = "telegraf_metrics_consumers"
-  ### Maximum number of metrics to buffer between collection intervals
-  metric_buffer = 100000
   ### Offset (must be either "oldest" or "newest")
   offset = "oldest"
 
@@ -72,11 +72,13 @@ func (k *Kafka) SetParser(parser parsers.Parser) {
 	k.parser = parser
 }
 
-func (k *Kafka) Start() error {
+func (k *Kafka) Start(acc telegraf.Accumulator) error {
 	k.Lock()
 	defer k.Unlock()
 	var consumerErr error
 
+	k.acc = acc
+
 	config := consumergroup.NewConfig()
 	switch strings.ToLower(k.Offset) {
 	case "oldest", "":
@@ -106,13 +108,6 @@ func (k *Kafka) Start() error {
 	}
 
 	k.done = make(chan struct{})
-	if k.PointBuffer == 0 && k.MetricBuffer == 0 {
-		k.MetricBuffer = 100000
-	} else if k.PointBuffer > 0 {
-		// Legacy support of PointBuffer field TODO remove
-		k.MetricBuffer = k.PointBuffer
-	}
-	k.metricC = make(chan telegraf.Metric, k.MetricBuffer)
 
 	// Start the kafka message reader
 	go k.receiver()
@@ -138,14 +133,7 @@ func (k *Kafka) receiver() {
 			}
 
 			for _, metric := range metrics {
-				fmt.Println(string(metric.Name()))
-				select {
-				case k.metricC <- metric:
-					continue
-				default:
-					log.Printf("Kafka Consumer buffer is full, dropping a metric." +
-						" You may want to increase the metric_buffer setting")
-				}
+				k.acc.AddFields(metric.Name(), metric.Fields(), metric.Tags(), metric.Time())
 			}
 
 			if !k.doNotCommitMsgs {
@@ -169,13 +157,6 @@ func (k *Kafka) Stop() {
 }
 
 func (k *Kafka) Gather(acc telegraf.Accumulator) error {
-	k.Lock()
-	defer k.Unlock()
-	nmetrics := len(k.metricC)
-	for i := 0; i < nmetrics; i++ {
-		metric := <-k.metricC
-		acc.AddFields(metric.Name(), metric.Fields(), metric.Tags(), metric.Time())
-	}
 	return nil
 }