Flush based on buffer size rather than time

this includes:
- Add Accumulator to the Start() function of service inputs
- For message consumer plugins, use the Accumulator to constantly add
  metrics and make Gather a dummy function
- rework unit tests to match this new behavior.
- make "flush_buffer_when_full" a config option that defaults to true

closes #666

plugins/inputs/mqtt_consumer/mqtt_consumer_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/testutil"
 
@@ -16,19 +15,15 @@ const (
 	testMsgGraphite = "cpu.load.short.graphite 23422 1454780029"
 	testMsgJSON     = "{\"a\": 5, \"b\": {\"c\": 6}}\n"
 	invalidMsg      = "cpu_load_short,host=server01 1422568543702900257"
-	metricBuffer    = 5
 )
 
 func newTestMQTTConsumer() (*MQTTConsumer, chan mqtt.Message) {
-	in := make(chan mqtt.Message, metricBuffer)
+	in := make(chan mqtt.Message, 100)
 	n := &MQTTConsumer{
-		Topics:       []string{"telegraf"},
-		Servers:      []string{"localhost:1883"},
-		MetricBuffer: metricBuffer,
-		in:           in,
-		done:         make(chan struct{}),
-		metricC:      make(chan telegraf.Metric, metricBuffer),
-		topicC:       make(chan string, metricBuffer),
+		Topics:  []string{"telegraf"},
+		Servers: []string{"localhost:1883"},
+		in:      in,
+		done:    make(chan struct{}),
 	}
 	return n, in
 }
@@ -36,14 +31,16 @@ func newTestMQTTConsumer() (*MQTTConsumer, chan mqtt.Message) {
 // Test that the parser parses NATS messages into metrics
 func TestRunParser(t *testing.T) {
 	n, in := newTestMQTTConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- mqttMsg(testMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	if a := len(n.metricC); a != 1 {
+	if a := acc.NFields(); a != 1 {
 		t.Errorf("got %v, expected %v", a, 1)
 	}
 }
@@ -51,51 +48,34 @@ func TestRunParser(t *testing.T) {
 // Test that the parser ignores invalid messages
 func TestRunParserInvalidMsg(t *testing.T) {
 	n, in := newTestMQTTConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- mqttMsg(invalidMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	if a := len(n.metricC); a != 0 {
+	if a := acc.NFields(); a != 0 {
 		t.Errorf("got %v, expected %v", a, 0)
 	}
 }
 
-// Test that metrics are dropped when we hit the buffer limit
-func TestRunParserRespectsBuffer(t *testing.T) {
-	n, in := newTestMQTTConsumer()
-	defer close(n.done)
-
-	n.parser, _ = parsers.NewInfluxParser()
-	go n.receiver()
-	for i := 0; i < metricBuffer+1; i++ {
-		in <- mqttMsg(testMsg)
-	}
-	time.Sleep(time.Millisecond)
-
-	if a := len(n.metricC); a != metricBuffer {
-		t.Errorf("got %v, expected %v", a, metricBuffer)
-	}
-}
-
 // Test that the parser parses line format messages into metrics
 func TestRunParserAndGather(t *testing.T) {
 	n, in := newTestMQTTConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewInfluxParser()
 	go n.receiver()
 	in <- mqttMsg(testMsg)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "cpu_load_short",
 		map[string]interface{}{"value": float64(23422)})
 }
@@ -103,19 +83,17 @@ func TestRunParserAndGather(t *testing.T) {
 // Test that the parser parses graphite format messages into metrics
 func TestRunParserAndGatherGraphite(t *testing.T) {
 	n, in := newTestMQTTConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewGraphiteParser("_", []string{}, nil)
 	go n.receiver()
 	in <- mqttMsg(testMsgGraphite)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "cpu_load_short_graphite",
 		map[string]interface{}{"value": float64(23422)})
 }
@@ -123,19 +101,17 @@ func TestRunParserAndGatherGraphite(t *testing.T) {
 // Test that the parser parses json format messages into metrics
 func TestRunParserAndGatherJSON(t *testing.T) {
 	n, in := newTestMQTTConsumer()
+	acc := testutil.Accumulator{}
+	n.acc = &acc
 	defer close(n.done)
 
 	n.parser, _ = parsers.NewJSONParser("nats_json_test", []string{}, nil)
 	go n.receiver()
 	in <- mqttMsg(testMsgJSON)
-	time.Sleep(time.Millisecond)
+	time.Sleep(time.Millisecond * 25)
 
-	acc := testutil.Accumulator{}
 	n.Gather(&acc)
 
-	if a := len(acc.Metrics); a != 1 {
-		t.Errorf("got %v, expected %v", a, 1)
-	}
 	acc.AssertContainsFields(t, "nats_json_test",
 		map[string]interface{}{
 			"a":   float64(5),